Skip to contentSkip to conversation

perf/uprobe: Optimize uprobes

PATCH In review Peter Zijlstra 5 participants 45 messages Jul 11, 2024 – Jul 27, 2024

Subsystems (7)
BPF perf events Tracing BPF tracing +3 more subsystems
lore
Share & tools
lore

Peter Zijlstra

author maintainer
#0 lore raw

v2 cover perf/uprobe: Optimize uprobes

Hi!

These patches implement the (S)RCU based proposal to optimize uprobes.

On my c^Htrusty old IVB-EP -- where each (of the 40) CPU calls 'func' in a tight loop:

perf probe -x ./uprobes test=func
perf stat -ae probe_uprobe:test -- sleep 1

perf probe -x ./uprobes test=func%return perf stat -ae probe_uprobe:test__return -- sleep 1

PRE:

4,038,804 probe_uprobe:test
2,356,275 probe_uprobe:test__return

POST:

7,216,579 probe_uprobe:test
6,744,786 probe_uprobe:test__return

(copy-paste FTW, I didn't do new numbers because the fast paths didn't change --
and quick test run shows similar numbers)

Patches also available here:

git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git perf/uprobes

Changes since last time:
- better split with intermediate inc_not_zero()
- fix UPROBE_HANDLER_REMOVE
- restored the lost rcu_assign_pointer()
- avoid lockdep for uretprobe_srcu
- add missing put_uprobe() -> srcu_read_unlock() conversion
- actually initialize return_instance::has_ref
- a few comments
- things I don't remember

author maintainer
#1 lore raw

PATCH v2 1/11 perf/uprobe: Re-indent labels

Remove the silly label indenting.

s/^\ \([[:alnum:]]*\):$/\1:/g

---
 kernel/events/uprobes.c |   26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)
kernel/events/uprobes.c +13 -13
@@ -205,7 +205,7 @@ static int __replace_page(struct vm_area
 	folio_put(old_folio);
 
 	err = 0;
- unlock:
+unlock:
 	mmu_notifier_invalidate_range_end(&range);
 	folio_unlock(old_folio);
 	return err;
@@ -857,7 +857,7 @@ static int prepare_uprobe(struct uprobe
 	smp_wmb(); /* pairs with the smp_rmb() in handle_swbp() */
 	set_bit(UPROBE_COPY_INSN, &uprobe->flags);
 
- out:
+out:
 	up_write(&uprobe->consumer_rwsem);
 
 	return ret;
@@ -965,7 +965,7 @@ build_map_info(struct address_space *map
 	struct map_info *info;
 	int more = 0;
 
- again:
+again:
 	i_mmap_lock_read(mapping);
 	vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {
 		if (!valid_vma(vma, is_register))
@@ -1019,7 +1019,7 @@ build_map_info(struct address_space *map
 	} while (--more);
 
 	goto again;
- out:
+out:
 	while (prev)
 		prev = free_map_info(prev);
 	return curr;
@@ -1068,13 +1068,13 @@ register_for_each_vma(struct uprobe *upr
 				err |= remove_breakpoint(uprobe, mm, info->vaddr);
 		}
 
- unlock:
+unlock:
 		mmap_write_unlock(mm);
- free:
+free:
 		mmput(mm);
 		info = free_map_info(info);
 	}
- out:
+out:
 	percpu_up_write(&dup_mmap_sem);
 	return err;
 }
@@ -1159,7 +1159,7 @@ static int __uprobe_register(struct inod
 	if (!IS_ALIGNED(ref_ctr_offset, sizeof(short)))
 		return -EINVAL;
 
- retry:
+retry:
 	uprobe = alloc_uprobe(inode, offset, ref_ctr_offset);
 	if (!uprobe)
 		return -ENOMEM;
@@ -1468,7 +1468,7 @@ static int xol_add_vma(struct mm_struct
 	ret = 0;
 	/* pairs with get_xol_area() */
 	smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */
- fail:
+fail:
 	mmap_write_unlock(mm);
 
 	return ret;
@@ -1512,7 +1512,7 @@ static struct xol_area *__create_xol_are
 	kfree(area->bitmap);
  free_area:
 	kfree(area);
- out:
+out:
 	return NULL;
 }
 
@@ -1915,7 +1915,7 @@ static void prepare_uretprobe(struct upr
 	utask->return_instances = ri;
 
 	return;
- fail:
+fail:
 	kfree(ri);
 }
 
@@ -2031,7 +2031,7 @@ static int is_trap_at_addr(struct mm_str
 
 	copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
 	put_page(page);
- out:
+out:
 	/* This needs to return true for any variant of the trap insn */
 	return is_trap_insn(&opcode);
 }
@@ -2159,7 +2159,7 @@ static void handle_trampoline(struct pt_
 	utask->return_instances = ri;
 	return;
 
- sigill:
+sigill:
 	uprobe_warn(current, "handle uretprobe, sending SIGILL.");
 	force_sig(SIGILL);
 
#2 lore raw

Re: [PATCH v2 01/11] perf/uprobe: Re-indent labels

On Thu, Jul 11, 2024 at 01:02:36PM +0200, Peter Zijlstra wrote:

SNIP

Quoted 19 lines
kernel/events/uprobes.c +2 -2
@@ -1159,7 +1159,7 @@ static int __uprobe_register(struct inod
 	if (!IS_ALIGNED(ref_ctr_offset, sizeof(short)))
 		return -EINVAL;
 
- retry:
+retry:
 	uprobe = alloc_uprobe(inode, offset, ref_ctr_offset);
 	if (!uprobe)
 		return -ENOMEM;
@@ -1468,7 +1468,7 @@ static int xol_add_vma(struct mm_struct
 	ret = 0;
 	/* pairs with get_xol_area() */
 	smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */
- fail:
+fail:
 	mmap_write_unlock(mm);
 
 	return ret;

kernel/events/uprobes.c static struct xol_area *__create_xol_are · line 1513

Quoted 2 lines
kernel/events/uprobes.c +0 -0
@@ -1512,7 +1512,7 @@ static struct xol_area *__create_xol_are
 	kfree(area->bitmap);
  free_area:

hi,
missed this one and another one few lines before that ;-)

jirka

Quoted 35 lines
kernel/events/uprobes.c +4 -4
 	kfree(area);
- out:
+out:
 	return NULL;
 }
 
@@ -1915,7 +1915,7 @@ static void prepare_uretprobe(struct upr
 	utask->return_instances = ri;
 
 	return;
- fail:
+fail:
 	kfree(ri);
 }
 
@@ -2031,7 +2031,7 @@ static int is_trap_at_addr(struct mm_str
 
 	copy_from_page(page, vaddr, &opcode, UPROBE_SWBP_INSN_SIZE);
 	put_page(page);
- out:
+out:
 	/* This needs to return true for any variant of the trap insn */
 	return is_trap_insn(&opcode);
 }
@@ -2159,7 +2159,7 @@ static void handle_trampoline(struct pt_
 	utask->return_instances = ri;
 	return;
 
- sigill:
+sigill:
 	uprobe_warn(current, "handle uretprobe, sending SIGILL.");
 	force_sig(SIGILL);
 
author maintainer
#3 lore raw

Re: [PATCH v2 01/11] perf/uprobe: Re-indent labels

Quoted 21 lines · Jiri Olsa

On Thu, Jul 11, 2024 at 01:02:36PM +0200, Peter Zijlstra wrote:

SNIP

Quoted 17 lines
kernel/events/uprobes.c +2 -2
@@ -1159,7 +1159,7 @@ static int __uprobe_register(struct inod
 	if (!IS_ALIGNED(ref_ctr_offset, sizeof(short)))
 		return -EINVAL;
 
- retry:
+retry:
 	uprobe = alloc_uprobe(inode, offset, ref_ctr_offset);
 	if (!uprobe)
 		return -ENOMEM;
@@ -1468,7 +1468,7 @@ static int xol_add_vma(struct mm_struct
 	ret = 0;
 	/* pairs with get_xol_area() */
 	smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */
- fail:
+fail:
 	mmap_write_unlock(mm);
 
Quoted 7 lines

kernel/events/uprobes.c static struct xol_area *__create_xol_are · line 1513

Quoted 4 lines
kernel/events/uprobes.c +0 -0
 	return ret;
@@ -1512,7 +1512,7 @@ static struct xol_area *__create_xol_are
 	kfree(area->bitmap);
  free_area:

hi,
missed this one and another one few lines before that ;-)

Bah, _ isn't in [[:alnum:]]. I'll go fix.

author maintainer
#4 lore raw

PATCH v2 2/11 perf/uprobe: Remove spurious whitespace

In reply to Peter Zijlstra
---
 kernel/events/uprobes.c |   12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
kernel/events/uprobes.c +6 -6
@@ -67,7 +67,7 @@ struct uprobe {
 	 * The generic code assumes that it has two members of unknown type
 	 * owned by the arch-specific code:
 	 *
-	 * 	insn -	copy_insn() saves the original instruction here for
+	 *	insn -	copy_insn() saves the original instruction here for
 	 *		arch_uprobe_analyze_insn().
 	 *
 	 *	ixol -	potentially modified instruction to execute out of
@@ -95,18 +95,18 @@ static LIST_HEAD(delayed_uprobe_list);
  * allocated.
  */
 struct xol_area {
-	wait_queue_head_t 		wq;		/* if all slots are busy */
-	atomic_t 			slot_count;	/* number of in-use slots */
-	unsigned long 			*bitmap;	/* 0 = free slot */
+	wait_queue_head_t		wq;		/* if all slots are busy */
+	atomic_t			slot_count;	/* number of in-use slots */
+	unsigned long			*bitmap;	/* 0 = free slot */
 
 	struct vm_special_mapping	xol_mapping;
-	struct page 			*pages[2];
+	struct page			*pages[2];
 	/*
 	 * We keep the vma's vm_start rather than a pointer to the vma
 	 * itself.  The probed process or a naughty kernel module could make
 	 * the vma go away, and we must handle that reasonably gracefully.
 	 */
-	unsigned long 			vaddr;		/* Page(s) of instruction slots */
+	unsigned long			vaddr;		/* Page(s) of instruction slots */
 };
 
 /*
author maintainer
#5 lore raw

PATCH v2 3/11 rbtree: Provide rb_find_rcu() / rb_find_add_rcu()

In reply to Peter Zijlstra

Much like latch_tree, add two RCU methods for the regular RB-tree, which can be used in conjunction with a seqcount to provide lockless lookups.

---
 include/linux/rbtree.h |   67 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
include/linux/rbtree.h +67 -0
@@ -245,6 +245,42 @@ rb_find_add(struct rb_node *node, struct
 }
 
 /**
+ * rb_find_add_rcu() - find equivalent @node in @tree, or add @node
+ * @node: node to look-for / insert
+ * @tree: tree to search / modify
+ * @cmp: operator defining the node order
+ *
+ * Adds a Store-Release for link_node.
+ *
+ * Returns the rb_node matching @node, or NULL when no match is found and @node
+ * is inserted.
+ */
+static __always_inline struct rb_node *
+rb_find_add_rcu(struct rb_node *node, struct rb_root *tree,
+		int (*cmp)(struct rb_node *, const struct rb_node *))
+{
+	struct rb_node **link = &tree->rb_node;
+	struct rb_node *parent = NULL;
+	int c;
+
+	while (*link) {
+		parent = *link;
+		c = cmp(node, parent);
+
+		if (c < 0)
+			link = &parent->rb_left;
+		else if (c > 0)
+			link = &parent->rb_right;
+		else
+			return parent;
+	}
+
+	rb_link_node_rcu(node, parent, link);
+	rb_insert_color(node, tree);
+	return NULL;
+}
+
+/**
  * rb_find() - find @key in tree @tree
  * @key: key to match
  * @tree: tree to search
@@ -268,6 +304,37 @@ rb_find(const void *key, const struct rb
 		else
 			return node;
 	}
+
+	return NULL;
+}
+
+/**
+ * rb_find_rcu() - find @key in tree @tree
+ * @key: key to match
+ * @tree: tree to search
+ * @cmp: operator defining the node order
+ *
+ * Notably, tree descent vs concurrent tree rotations is unsound and can result
+ * in false-negatives.
+ *
+ * Returns the rb_node matching @key or NULL.
+ */
+static __always_inline struct rb_node *
+rb_find_rcu(const void *key, const struct rb_root *tree,
+	    int (*cmp)(const void *key, const struct rb_node *))
+{
+	struct rb_node *node = tree->rb_node;
+
+	while (node) {
+		int c = cmp(key, node);
+
+		if (c < 0)
+			node = rcu_dereference_raw(node->rb_left);
+		else if (c > 0)
+			node = rcu_dereference_raw(node->rb_right);
+		else
+			return node;
+	}
 
 	return NULL;
 }
#6 lore raw

Re: [PATCH v2 03/11] rbtree: Provide rb_find_rcu() / rb_find_add_rcu()

Quoted 20 lines · Peter Zijlstra

Much like latch_tree, add two RCU methods for the regular RB-tree, which can be used in conjunction with a seqcount to provide lockless lookups.

---
 include/linux/rbtree.h |   67 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
include/linux/rbtree.h +3 -0
@@ -245,6 +245,42 @@ rb_find_add(struct rb_node *node, struct
 }
 
 /**
+ * rb_find_add_rcu() - find equivalent @node in @tree, or add @node
+ * @node: node to look-for / insert
+ * @tree: tree to search / modify

include/linux/rbtree.h rb_find_add(struct rb_node *node, struct · line 260

Quoted 10 lines
include/linux/rbtree.h +10 -0
+ * @cmp: operator defining the node order
+ *
+ * Adds a Store-Release for link_node.
+ *
+ * Returns the rb_node matching @node, or NULL when no match is found and @node
+ * is inserted.
+ */
+static __always_inline struct rb_node *
+rb_find_add_rcu(struct rb_node *node, struct rb_root *tree,
+               int (*cmp)(struct rb_node *, const struct rb_node *))

I don't get the point of the RCU version of rb_find_add as RCU itself doesn't provide enough protection for modification of the tree, right? So in uprobes code you do rb_find_add_rcu() under uprobes_treelock + uprobes_seqcount locks. Wouldn't it be just as fine to do plain non-RCU rb_find_add() in that case? After all, you do plain rb_erase under the same set of locks.

So what's the point of this one?

Quoted 22 lines
include/linux/rbtree.h +22 -0
+{
+       struct rb_node **link = &tree->rb_node;
+       struct rb_node *parent = NULL;
+       int c;
+
+       while (*link) {
+               parent = *link;
+               c = cmp(node, parent);
+
+               if (c < 0)
+                       link = &parent->rb_left;
+               else if (c > 0)
+                       link = &parent->rb_right;
+               else
+                       return parent;
+       }
+
+       rb_link_node_rcu(node, parent, link);
+       rb_insert_color(node, tree);
+       return NULL;
+}
+

[...]

author maintainer
#7 lore raw

Re: [PATCH v2 03/11] rbtree: Provide rb_find_rcu() / rb_find_add_rcu()

Quoted 39 lines · Andrii Nakryiko
Quoted 20 lines · Peter Zijlstra

Much like latch_tree, add two RCU methods for the regular RB-tree, which can be used in conjunction with a seqcount to provide lockless lookups.

---
 include/linux/rbtree.h |   67 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
include/linux/rbtree.h +3 -0
@@ -245,6 +245,42 @@ rb_find_add(struct rb_node *node, struct
 }
 
 /**
+ * rb_find_add_rcu() - find equivalent @node in @tree, or add @node
+ * @node: node to look-for / insert
+ * @tree: tree to search / modify

include/linux/rbtree.h rb_find_add(struct rb_node *node, struct · line 260

Quoted 10 lines
include/linux/rbtree.h +10 -0
+ * @cmp: operator defining the node order
+ *
+ * Adds a Store-Release for link_node.
+ *
+ * Returns the rb_node matching @node, or NULL when no match is found and @node
+ * is inserted.
+ */
+static __always_inline struct rb_node *
+rb_find_add_rcu(struct rb_node *node, struct rb_root *tree,
+               int (*cmp)(struct rb_node *, const struct rb_node *))

I don't get the point of the RCU version of rb_find_add as RCU itself doesn't provide enough protection for modification of the tree, right? So in uprobes code you do rb_find_add_rcu() under uprobes_treelock + uprobes_seqcount locks. Wouldn't it be just as fine to do plain non-RCU rb_find_add() in that case? After all, you do plain rb_erase under the same set of locks.

Quoted 1 line

So what's the point of this one?

The store-release when adding it to the tree. Without that it becomes possible to find the entry while the entry itself is incomplete.

Eg. something like:

entry.foo = A
rb_find_add(&entry->node, &my_tree, my_cmp);

vs

rcu_read_lock();
entry = rb_find_rcu(...);
assert(entry->foo == A);

might fail. Because there is nothing ordering the foo store and the rb-node add.

#8 lore raw

Re: [PATCH v2 03/11] rbtree: Provide rb_find_rcu() / rb_find_add_rcu()

Quoted 51 lines · Peter Zijlstra
Quoted 39 lines · Andrii Nakryiko
Quoted 20 lines · Peter Zijlstra

Much like latch_tree, add two RCU methods for the regular RB-tree, which can be used in conjunction with a seqcount to provide lockless lookups.

---
 include/linux/rbtree.h |   67 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)
include/linux/rbtree.h +3 -0
@@ -245,6 +245,42 @@ rb_find_add(struct rb_node *node, struct
 }
 
 /**
+ * rb_find_add_rcu() - find equivalent @node in @tree, or add @node
+ * @node: node to look-for / insert
+ * @tree: tree to search / modify

include/linux/rbtree.h rb_find_add(struct rb_node *node, struct · line 260

Quoted 10 lines
include/linux/rbtree.h +10 -0
+ * @cmp: operator defining the node order
+ *
+ * Adds a Store-Release for link_node.
+ *
+ * Returns the rb_node matching @node, or NULL when no match is found and @node
+ * is inserted.
+ */
+static __always_inline struct rb_node *
+rb_find_add_rcu(struct rb_node *node, struct rb_root *tree,
+               int (*cmp)(struct rb_node *, const struct rb_node *))

I don't get the point of the RCU version of rb_find_add as RCU itself doesn't provide enough protection for modification of the tree, right? So in uprobes code you do rb_find_add_rcu() under uprobes_treelock + uprobes_seqcount locks. Wouldn't it be just as fine to do plain non-RCU rb_find_add() in that case? After all, you do plain rb_erase under the same set of locks.

Quoted 1 line

So what's the point of this one?

The store-release when adding it to the tree. Without that it becomes possible to find the entry while the entry itself is incomplete.

Eg. something like:

entry.foo = A
rb_find_add(&entry->node, &my_tree, my_cmp);

Quoted 10 lines

vs

rcu_read_lock();
entry = rb_find_rcu(...);
assert(entry->foo == A);

might fail. Because there is nothing ordering the foo store and the rb-node add.

Ah, I see, thanks for the explanation. That's what "Adds a Store-Release for link_node." in the comment means, I see.

author maintainer
#9 lore raw

PATCH v2 4/11 perf/uprobe: RCU-ify find_uprobe()

In reply to Peter Zijlstra

With handle_swbp() triggering concurrently on (all) CPUs, tree_lock becomes a bottleneck. Avoid treelock by doing RCU lookups of the uprobe.

---
 kernel/events/uprobes.c |   49 +++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 40 insertions(+), 9 deletions(-)
kernel/events/uprobes.c +40 -9
@@ -40,6 +40,7 @@ static struct rb_root uprobes_tree = RB_
 #define no_uprobe_events()	RB_EMPTY_ROOT(&uprobes_tree)
 
 static DEFINE_RWLOCK(uprobes_treelock);	/* serialize rbtree access */
+static seqcount_rwlock_t uprobes_seqcount = SEQCNT_RWLOCK_ZERO(uprobes_seqcount, &uprobes_treelock);
 
 #define UPROBES_HASH_SZ	13
 /* serialize uprobe->pending_list */
@@ -54,6 +55,7 @@ DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem)
 struct uprobe {
 	struct rb_node		rb_node;	/* node in the rb tree */
 	refcount_t		ref;
+	struct rcu_head		rcu;
 	struct rw_semaphore	register_rwsem;
 	struct rw_semaphore	consumer_rwsem;
 	struct list_head	pending_list;
@@ -587,12 +589,25 @@ set_orig_insn(struct arch_uprobe *auprob
 			*(uprobe_opcode_t *)&auprobe->insn);
 }
 
+static struct uprobe *try_get_uprobe(struct uprobe *uprobe)
+{
+	if (refcount_inc_not_zero(&uprobe->ref))
+		return uprobe;
+	return NULL;
+}
+
 static struct uprobe *get_uprobe(struct uprobe *uprobe)
 {
 	refcount_inc(&uprobe->ref);
 	return uprobe;
 }
 
+static void uprobe_free_rcu(struct rcu_head *rcu)
+{
+	struct uprobe *uprobe = container_of(rcu, struct uprobe, rcu);
+	kfree(uprobe);
+}
+
 static void put_uprobe(struct uprobe *uprobe)
 {
 	if (refcount_dec_and_test(&uprobe->ref)) {
@@ -604,7 +619,7 @@ static void put_uprobe(struct uprobe *up
 		mutex_lock(&delayed_uprobe_lock);
 		delayed_uprobe_remove(uprobe, NULL);
 		mutex_unlock(&delayed_uprobe_lock);
-		kfree(uprobe);
+		call_rcu(&uprobe->rcu, uprobe_free_rcu);
 	}
 }
 
@@ -653,10 +668,10 @@ static struct uprobe *__find_uprobe(stru
 		.inode = inode,
 		.offset = offset,
 	};
-	struct rb_node *node = rb_find(&key, &uprobes_tree, __uprobe_cmp_key);
+	struct rb_node *node = rb_find_rcu(&key, &uprobes_tree, __uprobe_cmp_key);
 
 	if (node)
-		return get_uprobe(__node_2_uprobe(node));
+		return try_get_uprobe(__node_2_uprobe(node));
 
 	return NULL;
 }
@@ -667,20 +682,32 @@ static struct uprobe *__find_uprobe(stru
  */
 static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)
 {
-	struct uprobe *uprobe;
+	unsigned int seq;
 
-	read_lock(&uprobes_treelock);
-	uprobe = __find_uprobe(inode, offset);
-	read_unlock(&uprobes_treelock);
+	guard(rcu)();
 
-	return uprobe;
+	do {
+		seq = read_seqcount_begin(&uprobes_seqcount);
+		struct uprobe *uprobe = __find_uprobe(inode, offset);
+		if (uprobe) {
+			/*
+			 * Lockless RB-tree lookups are prone to false-negatives.
+			 * If they find something, it's good. If they do not find,
+			 * it needs to be validated.
+			 */
+			return uprobe;
+		}
+	} while (read_seqcount_retry(&uprobes_seqcount, seq));
+
+	/* Really didn't find anything. */
+	return NULL;
 }
 
 static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
 {
 	struct rb_node *node;
 
-	node = rb_find_add(&uprobe->rb_node, &uprobes_tree, __uprobe_cmp);
+	node = rb_find_add_rcu(&uprobe->rb_node, &uprobes_tree, __uprobe_cmp);
 	if (node)
 		return get_uprobe(__node_2_uprobe(node));
 
@@ -702,7 +729,9 @@ static struct uprobe *insert_uprobe(stru
 	struct uprobe *u;
 
 	write_lock(&uprobes_treelock);
+	write_seqcount_begin(&uprobes_seqcount);
 	u = __insert_uprobe(uprobe);
+	write_seqcount_end(&uprobes_seqcount);
 	write_unlock(&uprobes_treelock);
 
 	return u;
@@ -936,7 +965,9 @@ static void delete_uprobe(struct uprobe
 		return;
 
 	write_lock(&uprobes_treelock);
+	write_seqcount_begin(&uprobes_seqcount);
 	rb_erase(&uprobe->rb_node, &uprobes_tree);
+	write_seqcount_end(&uprobes_seqcount);
 	write_unlock(&uprobes_treelock);
 	RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */
 	put_uprobe(uprobe);
maintainer Acked-by on v2 4/11
#10 lore raw

Re: [PATCH v2 04/11] perf/uprobe: RCU-ify find_uprobe()

Quoted 4 lines · On Thu, 11 Jul 2024 13:02:39 +0200 Peter Zijlstra <[email protected]> wrote:

With handle_swbp() triggering concurrently on (all) CPUs, tree_lock becomes a bottleneck. Avoid treelock by doing RCU lookups of the uprobe.

Looks good to me.

Thanks,

Quoted 132 lines
---
 kernel/events/uprobes.c |   49 +++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 40 insertions(+), 9 deletions(-)
kernel/events/uprobes.c +40 -9
@@ -40,6 +40,7 @@ static struct rb_root uprobes_tree = RB_
 #define no_uprobe_events()	RB_EMPTY_ROOT(&uprobes_tree)
 
 static DEFINE_RWLOCK(uprobes_treelock);	/* serialize rbtree access */
+static seqcount_rwlock_t uprobes_seqcount = SEQCNT_RWLOCK_ZERO(uprobes_seqcount, &uprobes_treelock);
 
 #define UPROBES_HASH_SZ	13
 /* serialize uprobe->pending_list */
@@ -54,6 +55,7 @@ DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem)
 struct uprobe {
 	struct rb_node		rb_node;	/* node in the rb tree */
 	refcount_t		ref;
+	struct rcu_head		rcu;
 	struct rw_semaphore	register_rwsem;
 	struct rw_semaphore	consumer_rwsem;
 	struct list_head	pending_list;
@@ -587,12 +589,25 @@ set_orig_insn(struct arch_uprobe *auprob
 			*(uprobe_opcode_t *)&auprobe->insn);
 }
 
+static struct uprobe *try_get_uprobe(struct uprobe *uprobe)
+{
+	if (refcount_inc_not_zero(&uprobe->ref))
+		return uprobe;
+	return NULL;
+}
+
 static struct uprobe *get_uprobe(struct uprobe *uprobe)
 {
 	refcount_inc(&uprobe->ref);
 	return uprobe;
 }
 
+static void uprobe_free_rcu(struct rcu_head *rcu)
+{
+	struct uprobe *uprobe = container_of(rcu, struct uprobe, rcu);
+	kfree(uprobe);
+}
+
 static void put_uprobe(struct uprobe *uprobe)
 {
 	if (refcount_dec_and_test(&uprobe->ref)) {
@@ -604,7 +619,7 @@ static void put_uprobe(struct uprobe *up
 		mutex_lock(&delayed_uprobe_lock);
 		delayed_uprobe_remove(uprobe, NULL);
 		mutex_unlock(&delayed_uprobe_lock);
-		kfree(uprobe);
+		call_rcu(&uprobe->rcu, uprobe_free_rcu);
 	}
 }
 
@@ -653,10 +668,10 @@ static struct uprobe *__find_uprobe(stru
 		.inode = inode,
 		.offset = offset,
 	};
-	struct rb_node *node = rb_find(&key, &uprobes_tree, __uprobe_cmp_key);
+	struct rb_node *node = rb_find_rcu(&key, &uprobes_tree, __uprobe_cmp_key);
 
 	if (node)
-		return get_uprobe(__node_2_uprobe(node));
+		return try_get_uprobe(__node_2_uprobe(node));
 
 	return NULL;
 }
@@ -667,20 +682,32 @@ static struct uprobe *__find_uprobe(stru
  */
 static struct uprobe *find_uprobe(struct inode *inode, loff_t offset)
 {
-	struct uprobe *uprobe;
+	unsigned int seq;
 
-	read_lock(&uprobes_treelock);
-	uprobe = __find_uprobe(inode, offset);
-	read_unlock(&uprobes_treelock);
+	guard(rcu)();
 
-	return uprobe;
+	do {
+		seq = read_seqcount_begin(&uprobes_seqcount);
+		struct uprobe *uprobe = __find_uprobe(inode, offset);
+		if (uprobe) {
+			/*
+			 * Lockless RB-tree lookups are prone to false-negatives.
+			 * If they find something, it's good. If they do not find,
+			 * it needs to be validated.
+			 */
+			return uprobe;
+		}
+	} while (read_seqcount_retry(&uprobes_seqcount, seq));
+
+	/* Really didn't find anything. */
+	return NULL;
 }
 
 static struct uprobe *__insert_uprobe(struct uprobe *uprobe)
 {
 	struct rb_node *node;
 
-	node = rb_find_add(&uprobe->rb_node, &uprobes_tree, __uprobe_cmp);
+	node = rb_find_add_rcu(&uprobe->rb_node, &uprobes_tree, __uprobe_cmp);
 	if (node)
 		return get_uprobe(__node_2_uprobe(node));
 
@@ -702,7 +729,9 @@ static struct uprobe *insert_uprobe(stru
 	struct uprobe *u;
 
 	write_lock(&uprobes_treelock);
+	write_seqcount_begin(&uprobes_seqcount);
 	u = __insert_uprobe(uprobe);
+	write_seqcount_end(&uprobes_seqcount);
 	write_unlock(&uprobes_treelock);
 
 	return u;
@@ -936,7 +965,9 @@ static void delete_uprobe(struct uprobe
 		return;
 
 	write_lock(&uprobes_treelock);
+	write_seqcount_begin(&uprobes_seqcount);
 	rb_erase(&uprobe->rb_node, &uprobes_tree);
+	write_seqcount_end(&uprobes_seqcount);
 	write_unlock(&uprobes_treelock);
 	RB_CLEAR_NODE(&uprobe->rb_node); /* for uprobe_is_active() */
 	put_uprobe(uprobe);
Signature · 1 line
Masami Hiramatsu (Google) <[email protected]>
author maintainer
#11 lore raw

PATCH v2 5/11 perf/uprobe: Simplify UPROBE_HANDLER_REMOVE logic

In reply to Peter Zijlstra

Specifically, get rid of the uprobe->consumers re-load, which isn't sound under RCU.

---
 kernel/events/uprobes.c |   17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)
kernel/events/uprobes.c +14 -3
@@ -2101,6 +2101,7 @@ static void handler_chain(struct uprobe
 	struct uprobe_consumer *uc;
 	int remove = UPROBE_HANDLER_REMOVE;
 	bool need_prep = false; /* prepare return uprobe, when needed */
+	bool had_handler = false;
 
 	down_read(&uprobe->register_rwsem);
 	for (uc = uprobe->consumers; uc; uc = uc->next) {
@@ -2115,16 +2116,26 @@ static void handler_chain(struct uprobe
 		if (uc->ret_handler)
 			need_prep = true;
 
+		/*
+		 * A single handler that does not mask out REMOVE, means the
+		 * probe stays.
+		 */
+		had_handler = true;
 		remove &= rc;
 	}
 
+	/*
+	 * If there were no handlers called, nobody asked for it to be removed
+	 * but also nobody got to mask the value. Fix it up.
+	 */
+	if (!had_handler)
+		remove = 0;
+
 	if (need_prep && !remove)
 		prepare_uretprobe(uprobe, regs); /* put bp at return */
 
-	if (remove && uprobe->consumers) {
-		WARN_ON(!uprobe_is_active(uprobe));
+	if (remove)
 		unapply_uprobe(uprobe, current->mm);
-	}
 	up_read(&uprobe->register_rwsem);
 }
 
author maintainer
#12 lore raw

PATCH v2 6/11 perf/uprobe: SRCU-ify uprobe->consumer list

In reply to Peter Zijlstra

With handle_swbp() hitting concurrently on (all) CPUs the uprobe->register_rwsem can get very contended. Add an SRCU instance to cover the consumer list and consumer lifetime.

Since the consumer are externally embedded structures, unregister will have to suffer a synchronize_srcu().

A notably complication is the UPROBE_HANDLER_REMOVE logic which can race against uprobe_register() such that it might want to remove a freshly installer handler that didn't get called. In order to close this hole, a seqcount is added. With that, the removal path can tell if anything changed and bail out of the removal.

---
 kernel/events/uprobes.c |   60 ++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 50 insertions(+), 10 deletions(-)
kernel/events/uprobes.c +50 -10
@@ -26,6 +26,7 @@
 #include <linux/task_work.h>
 #include <linux/shmem_fs.h>
 #include <linux/khugepaged.h>
+#include <linux/srcu.h>
 
 #include <linux/uprobes.h>
 
@@ -49,6 +50,11 @@ static struct mutex uprobes_mmap_mutex[U
 
 DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem);
 
+/*
+ * Covers uprobe->consumers lifetime.
+ */
+DEFINE_STATIC_SRCU(uprobes_srcu);
+
 /* Have a copy of original instruction */
 #define UPROBE_COPY_INSN	0
 
@@ -57,6 +63,7 @@ struct uprobe {
 	refcount_t		ref;
 	struct rcu_head		rcu;
 	struct rw_semaphore	register_rwsem;
+	seqcount_t		register_seq;
 	struct rw_semaphore	consumer_rwsem;
 	struct list_head	pending_list;
 	struct uprobe_consumer	*consumers;
@@ -760,6 +767,7 @@ static struct uprobe *alloc_uprobe(struc
 	uprobe->offset = offset;
 	uprobe->ref_ctr_offset = ref_ctr_offset;
 	init_rwsem(&uprobe->register_rwsem);
+	seqcount_init(&uprobe->register_seq);
 	init_rwsem(&uprobe->consumer_rwsem);
 
 	/* add to uprobes_tree, sorted on inode:offset */
@@ -782,8 +790,8 @@ static struct uprobe *alloc_uprobe(struc
 static void consumer_add(struct uprobe *uprobe, struct uprobe_consumer *uc)
 {
 	down_write(&uprobe->consumer_rwsem);
-	uc->next = uprobe->consumers;
-	uprobe->consumers = uc;
+	WRITE_ONCE(uc->next, uprobe->consumers);
+	rcu_assign_pointer(uprobe->consumers, uc);
 	up_write(&uprobe->consumer_rwsem);
 }
 
@@ -800,7 +808,7 @@ static bool consumer_del(struct uprobe *
 	down_write(&uprobe->consumer_rwsem);
 	for (con = &uprobe->consumers; *con; con = &(*con)->next) {
 		if (*con == uc) {
-			*con = uc->next;
+			WRITE_ONCE(*con, uc->next);
 			ret = true;
 			break;
 		}
@@ -1139,9 +1147,13 @@ void uprobe_unregister(struct inode *ino
 		return;
 
 	down_write(&uprobe->register_rwsem);
+	raw_write_seqcount_begin(&uprobe->register_seq);
 	__uprobe_unregister(uprobe, uc);
+	raw_write_seqcount_end(&uprobe->register_seq);
 	up_write(&uprobe->register_rwsem);
 	put_uprobe(uprobe);
+
+	synchronize_srcu(&uprobes_srcu);
 }
 EXPORT_SYMBOL_GPL(uprobe_unregister);
 
@@ -1204,10 +1216,12 @@ static int __uprobe_register(struct inod
 	down_write(&uprobe->register_rwsem);
 	ret = -EAGAIN;
 	if (likely(uprobe_is_active(uprobe))) {
+		raw_write_seqcount_begin(&uprobe->register_seq);
 		consumer_add(uprobe, uc);
 		ret = register_for_each_vma(uprobe, uc);
 		if (ret)
 			__uprobe_unregister(uprobe, uc);
+		raw_write_seqcount_end(&uprobe->register_seq);
 	}
 	up_write(&uprobe->register_rwsem);
 	put_uprobe(uprobe);
@@ -1250,10 +1264,12 @@ int uprobe_apply(struct inode *inode, lo
 		return ret;
 
 	down_write(&uprobe->register_rwsem);
+	raw_write_seqcount_begin(&uprobe->register_seq);
 	for (con = uprobe->consumers; con && con != uc ; con = con->next)
 		;
 	if (con)
 		ret = register_for_each_vma(uprobe, add ? uc : NULL);
+	raw_write_seqcount_end(&uprobe->register_seq);
 	up_write(&uprobe->register_rwsem);
 	put_uprobe(uprobe);
 
@@ -2096,15 +2112,23 @@ static struct uprobe *find_active_uprobe
 	return uprobe;
 }
 
+#define for_each_consumer_rcu(pos, head) \
+	for (pos = rcu_dereference_raw(head); pos; \
+	     pos = rcu_dereference_raw(pos->next))
+
 static void handler_chain(struct uprobe *uprobe, struct pt_regs *regs)
 {
 	struct uprobe_consumer *uc;
 	int remove = UPROBE_HANDLER_REMOVE;
 	bool need_prep = false; /* prepare return uprobe, when needed */
 	bool had_handler = false;
+	unsigned int seq;
 
-	down_read(&uprobe->register_rwsem);
-	for (uc = uprobe->consumers; uc; uc = uc->next) {
+	guard(srcu)(&uprobes_srcu);
+
+	seq = raw_read_seqcount_begin(&uprobe->register_seq);
+
+	for_each_consumer_rcu(uc, uprobe->consumers) {
 		int rc = 0;
 
 		if (uc->handler) {
@@ -2134,9 +2158,25 @@ static void handler_chain(struct uprobe
 	if (need_prep && !remove)
 		prepare_uretprobe(uprobe, regs); /* put bp at return */
 
-	if (remove)
+	if (remove) {
+		/*
+		 * Removing uprobes is a slow path, after all, the more probes
+		 * you remove, the less probe hits you get.
+		 *
+		 * This needs to serialize against uprobe_register(), such that
+		 * if the above RCU iteration missed a new handler that
+		 * would've liked to keep the probe, we don't go uninstall the
+		 * probe after it already ran register_for_each_vma().
+		 *
+		 * The rwsem ensures exclusivity against uprobe_register()
+		 * while the seqcount will avoid the removal if anything has
+		 * changed since we started.
+		 */
+		guard(rwsem_read)(&uprobe->register_rwsem);
+		if (read_seqcount_retry(&uprobe->register_seq, seq))
+			return;
 		unapply_uprobe(uprobe, current->mm);
-	up_read(&uprobe->register_rwsem);
+	}
 }
 
 static void
@@ -2145,12 +2185,12 @@ handle_uretprobe_chain(struct return_ins
 	struct uprobe *uprobe = ri->uprobe;
 	struct uprobe_consumer *uc;
 
-	down_read(&uprobe->register_rwsem);
-	for (uc = uprobe->consumers; uc; uc = uc->next) {
+	guard(srcu)(&uprobes_srcu);
+
+	for_each_consumer_rcu(uc, uprobe->consumers) {
 		if (uc->ret_handler)
 			uc->ret_handler(uc, ri->func, regs);
 	}
-	up_read(&uprobe->register_rwsem);
 }
 
 static struct return_instance *find_next_ret_chain(struct return_instance *ri)
#13 lore raw

Re: [PATCH v2 06/11] perf/uprobe: SRCU-ify uprobe->consumer list

+ bpf@vger, please cc bpf ML for the next revision, these changes are very relevant there as well, thanks

Quoted 19 lines · Peter Zijlstra

With handle_swbp() hitting concurrently on (all) CPUs the uprobe->register_rwsem can get very contended. Add an SRCU instance to cover the consumer list and consumer lifetime.

Since the consumer are externally embedded structures, unregister will have to suffer a synchronize_srcu().

A notably complication is the UPROBE_HANDLER_REMOVE logic which can race against uprobe_register() such that it might want to remove a freshly installer handler that didn't get called. In order to close this hole, a seqcount is added. With that, the removal path can tell if anything changed and bail out of the removal.

---
 kernel/events/uprobes.c |   60 ++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 50 insertions(+), 10 deletions(-)

[...]

kernel/events/uprobes.c static bool consumer_del(struct uprobe * · line 811

Quoted 6 lines
kernel/events/uprobes.c +1 -1
@@ -800,7 +808,7 @@ static bool consumer_del(struct uprobe *
        down_write(&uprobe->consumer_rwsem);
        for (con = &uprobe->consumers; *con; con = &(*con)->next) {
                if (*con == uc) {
-                       *con = uc->next;
+                       WRITE_ONCE(*con, uc->next);

I have a dumb and mechanical question.

Above in consumer_add() you are doing WRITE_ONCE() for uc->next assignment, but rcu_assign_pointer() for uprobe->consumers. Here, you are doing WRITE_ONCE() for the same operation, if it so happens that uc == *con == uprobe->consumers. So is rcu_assign_pointer() necessary in consumer_addr()? If yes, we should have it here as well, no? And if not, why bother with it in consumer_add()?

Quoted 16 lines
kernel/events/uprobes.c +4 -0
                        ret = true;
                        break;
                }
@@ -1139,9 +1147,13 @@ void uprobe_unregister(struct inode *ino
                return;
 
        down_write(&uprobe->register_rwsem);
+       raw_write_seqcount_begin(&uprobe->register_seq);
        __uprobe_unregister(uprobe, uc);
+       raw_write_seqcount_end(&uprobe->register_seq);
        up_write(&uprobe->register_rwsem);
        put_uprobe(uprobe);
+
+       synchronize_srcu(&uprobes_srcu);
 }
 EXPORT_SYMBOL_GPL(uprobe_unregister);

[...]

author maintainer
#14 lore raw

Re: [PATCH v2 06/11] perf/uprobe: SRCU-ify uprobe->consumer list

Quoted 35 lines · Andrii Nakryiko

+ bpf@vger, please cc bpf ML for the next revision, these changes are very relevant there as well, thanks

Quoted 19 lines · Peter Zijlstra

With handle_swbp() hitting concurrently on (all) CPUs the uprobe->register_rwsem can get very contended. Add an SRCU instance to cover the consumer list and consumer lifetime.

Since the consumer are externally embedded structures, unregister will have to suffer a synchronize_srcu().

A notably complication is the UPROBE_HANDLER_REMOVE logic which can race against uprobe_register() such that it might want to remove a freshly installer handler that didn't get called. In order to close this hole, a seqcount is added. With that, the removal path can tell if anything changed and bail out of the removal.

---
 kernel/events/uprobes.c |   60 ++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 50 insertions(+), 10 deletions(-)

[...]

kernel/events/uprobes.c static bool consumer_del(struct uprobe * · line 811

Quoted 6 lines
kernel/events/uprobes.c +1 -1
@@ -800,7 +808,7 @@ static bool consumer_del(struct uprobe *
        down_write(&uprobe->consumer_rwsem);
        for (con = &uprobe->consumers; *con; con = &(*con)->next) {
                if (*con == uc) {
-                       *con = uc->next;
+                       WRITE_ONCE(*con, uc->next);

I have a dumb and mechanical question.

Quoted 6 lines

Above in consumer_add() you are doing WRITE_ONCE() for uc->next assignment, but rcu_assign_pointer() for uprobe->consumers. Here, you are doing WRITE_ONCE() for the same operation, if it so happens that uc == *con == uprobe->consumers. So is rcu_assign_pointer() necessary in consumer_addr()? If yes, we should have it here as well, no? And if not, why bother with it in consumer_add()?

add is a publish and needs to ensure all stores to the object are ordered before the object is linked in. Note that rcu_assign_pointer() is basically a fancy way of writing smp_store_release().

del otoh does not publish, it removes and doesn't need ordering.

It does however need to ensure the pointer write itself isn't torn. That is, without the WRITE_ONCE() the compiler is free to do byte stores in order to update the 8 byte pointer value (assuming 64bit). This is pretty dumb, but very much permitted by C and also utterly fatal in the case where an RCU iteration comes by and reads a half-half pointer value.

Quoted 18 lines
Quoted 16 lines
kernel/events/uprobes.c +4 -0
                        ret = true;
                        break;
                }
@@ -1139,9 +1147,13 @@ void uprobe_unregister(struct inode *ino
                return;
 
        down_write(&uprobe->register_rwsem);
+       raw_write_seqcount_begin(&uprobe->register_seq);
        __uprobe_unregister(uprobe, uc);
+       raw_write_seqcount_end(&uprobe->register_seq);
        up_write(&uprobe->register_rwsem);
        put_uprobe(uprobe);
+
+       synchronize_srcu(&uprobes_srcu);
 }
 EXPORT_SYMBOL_GPL(uprobe_unregister);

[...]

#15 lore raw

Re: [PATCH v2 06/11] perf/uprobe: SRCU-ify uprobe->consumer list

Quoted 48 lines · Peter Zijlstra
Quoted 35 lines · Andrii Nakryiko

+ bpf@vger, please cc bpf ML for the next revision, these changes are very relevant there as well, thanks

Quoted 19 lines · Peter Zijlstra

With handle_swbp() hitting concurrently on (all) CPUs the uprobe->register_rwsem can get very contended. Add an SRCU instance to cover the consumer list and consumer lifetime.

Since the consumer are externally embedded structures, unregister will have to suffer a synchronize_srcu().

A notably complication is the UPROBE_HANDLER_REMOVE logic which can race against uprobe_register() such that it might want to remove a freshly installer handler that didn't get called. In order to close this hole, a seqcount is added. With that, the removal path can tell if anything changed and bail out of the removal.

---
 kernel/events/uprobes.c |   60 ++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 50 insertions(+), 10 deletions(-)

[...]

kernel/events/uprobes.c static bool consumer_del(struct uprobe * · line 811

Quoted 6 lines
kernel/events/uprobes.c +1 -1
@@ -800,7 +808,7 @@ static bool consumer_del(struct uprobe *
        down_write(&uprobe->consumer_rwsem);
        for (con = &uprobe->consumers; *con; con = &(*con)->next) {
                if (*con == uc) {
-                       *con = uc->next;
+                       WRITE_ONCE(*con, uc->next);

I have a dumb and mechanical question.

Quoted 6 lines

Above in consumer_add() you are doing WRITE_ONCE() for uc->next assignment, but rcu_assign_pointer() for uprobe->consumers. Here, you are doing WRITE_ONCE() for the same operation, if it so happens that uc == *con == uprobe->consumers. So is rcu_assign_pointer() necessary in consumer_addr()? If yes, we should have it here as well, no? And if not, why bother with it in consumer_add()?

add is a publish and needs to ensure all stores to the object are ordered before the object is linked in. Note that rcu_assign_pointer() is basically a fancy way of writing smp_store_release().

Quoted 9 lines

del otoh does not publish, it removes and doesn't need ordering.

It does however need to ensure the pointer write itself isn't torn. That is, without the WRITE_ONCE() the compiler is free to do byte stores in order to update the 8 byte pointer value (assuming 64bit). This is pretty dumb, but very much permitted by C and also utterly fatal in the case where an RCU iteration comes by and reads a half-half pointer value.

Thanks for elaborating, very helpful! It's basically the same idea as with rb_find_add_rcu(), got it.

Quoted 18 lines
Quoted 18 lines
Quoted 16 lines
kernel/events/uprobes.c +4 -0
                        ret = true;
                        break;
                }
@@ -1139,9 +1147,13 @@ void uprobe_unregister(struct inode *ino
                return;
 
        down_write(&uprobe->register_rwsem);
+       raw_write_seqcount_begin(&uprobe->register_seq);
        __uprobe_unregister(uprobe, uc);
+       raw_write_seqcount_end(&uprobe->register_seq);
        up_write(&uprobe->register_rwsem);
        put_uprobe(uprobe);
+
+       synchronize_srcu(&uprobes_srcu);
 }
 EXPORT_SYMBOL_GPL(uprobe_unregister);

[...]

author maintainer
#16 lore raw

PATCH v2 7/11 perf/uprobe: Split uprobe_unregister()

In reply to Peter Zijlstra

With uprobe_unregister() having grown a synchronize_srcu(), it becomes fairly slow to call. Esp. since both users of this API call it in a loop.

Peel off the sync_srcu() and do it once, after the loop.

---
 include/linux/uprobes.h     |    8 ++++++--
 kernel/events/uprobes.c     |    8 ++++++--
 kernel/trace/bpf_trace.c    |    6 ++++--
 kernel/trace/trace_uprobe.c |    6 +++++-
 4 files changed, 21 insertions(+), 7 deletions(-)
include/linux/uprobes.h +6 -2
@@ -113,7 +113,8 @@ extern int uprobe_write_opcode(struct ar
 extern int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc);
 extern int uprobe_register_refctr(struct inode *inode, loff_t offset, loff_t ref_ctr_offset, struct uprobe_consumer *uc);
 extern int uprobe_apply(struct inode *inode, loff_t offset, struct uprobe_consumer *uc, bool);
-extern void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc);
+extern void uprobe_unregister_nosync(struct inode *inode, loff_t offset, struct uprobe_consumer *uc);
+extern void uprobe_unregister_sync(void);
 extern int uprobe_mmap(struct vm_area_struct *vma);
 extern void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 extern void uprobe_start_dup_mmap(void);
@@ -163,7 +164,10 @@ uprobe_apply(struct inode *inode, loff_t
 	return -ENOSYS;
 }
 static inline void
-uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
+uprobe_unregister_nosync(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
+{
+}
+static inline void uprobes_unregister_sync(void)
 {
 }
 static inline int uprobe_mmap(struct vm_area_struct *vma)
kernel/events/uprobes.c +6 -2
@@ -1138,7 +1138,7 @@ __uprobe_unregister(struct uprobe *uprob
  * @offset: offset from the start of the file.
  * @uc: identify which probe if multiple probes are colocated.
  */
-void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
+void uprobe_unregister_nosync(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
 {
 	struct uprobe *uprobe;
 
@@ -1152,10 +1152,14 @@ void uprobe_unregister(struct inode *ino
 	raw_write_seqcount_end(&uprobe->register_seq);
 	up_write(&uprobe->register_rwsem);
 	put_uprobe(uprobe);
+}
+EXPORT_SYMBOL_GPL(uprobe_unregister_nosync);
 
+void uprobe_unregister_sync(void)
+{
 	synchronize_srcu(&uprobes_srcu);
 }
-EXPORT_SYMBOL_GPL(uprobe_unregister);
+EXPORT_SYMBOL_GPL(uprobe_unregister_sync);
 
 /*
  * __uprobe_register - register a probe
kernel/trace/bpf_trace.c +4 -2
@@ -3181,9 +3181,11 @@ static void bpf_uprobe_unregister(struct
 	u32 i;
 
 	for (i = 0; i < cnt; i++) {
-		uprobe_unregister(d_real_inode(path->dentry), uprobes[i].offset,
-				  &uprobes[i].consumer);
+		uprobe_unregister_nosync(d_real_inode(path->dentry), uprobes[i].offset,
+					 &uprobes[i].consumer);
 	}
+	if (cnt)
+		uprobe_unregister_sync();
 }
 
 static void bpf_uprobe_multi_link_release(struct bpf_link *link)
kernel/trace/trace_uprobe.c +5 -1
@@ -1104,6 +1104,7 @@ static int trace_uprobe_enable(struct tr
 static void __probe_event_disable(struct trace_probe *tp)
 {
 	struct trace_uprobe *tu;
+	bool sync = false;
 
 	tu = container_of(tp, struct trace_uprobe, tp);
 	WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));
@@ -1112,9 +1113,12 @@ static void __probe_event_disable(struct
 		if (!tu->inode)
 			continue;
 
-		uprobe_unregister(tu->inode, tu->offset, &tu->consumer);
+		uprobe_unregister_nosync(tu->inode, tu->offset, &tu->consumer);
+		sync = true;
 		tu->inode = NULL;
 	}
+	if (sync)
+		uprobe_unregister_sync();
 }
 
 static int probe_event_enable(struct trace_event_call *call,
Acked-by on v2 7/11
#17 lore raw

Re: [PATCH v2 07/11] perf/uprobe: Split uprobe_unregister()

+ bpf

Quoted 7 lines · Peter Zijlstra

With uprobe_unregister() having grown a synchronize_srcu(), it becomes fairly slow to call. Esp. since both users of this API call it in a loop.

Peel off the sync_srcu() and do it once, after the loop.

Quoted 9 lines
---
 include/linux/uprobes.h     |    8 ++++++--
 kernel/events/uprobes.c     |    8 ++++++--
 kernel/trace/bpf_trace.c    |    6 ++++--
 kernel/trace/trace_uprobe.c |    6 +++++-
 4 files changed, 21 insertions(+), 7 deletions(-)

BPF side of things looks good:

Quoted 93 lines
include/linux/uprobes.h +6 -2
@@ -113,7 +113,8 @@ extern int uprobe_write_opcode(struct ar
 extern int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc);
 extern int uprobe_register_refctr(struct inode *inode, loff_t offset, loff_t ref_ctr_offset, struct uprobe_consumer *uc);
 extern int uprobe_apply(struct inode *inode, loff_t offset, struct uprobe_consumer *uc, bool);
-extern void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc);
+extern void uprobe_unregister_nosync(struct inode *inode, loff_t offset, struct uprobe_consumer *uc);
+extern void uprobe_unregister_sync(void);
 extern int uprobe_mmap(struct vm_area_struct *vma);
 extern void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end);
 extern void uprobe_start_dup_mmap(void);
@@ -163,7 +164,10 @@ uprobe_apply(struct inode *inode, loff_t
        return -ENOSYS;
 }
 static inline void
-uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
+uprobe_unregister_nosync(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
+{
+}
+static inline void uprobes_unregister_sync(void)
 {
 }
 static inline int uprobe_mmap(struct vm_area_struct *vma)
kernel/events/uprobes.c +6 -2
@@ -1138,7 +1138,7 @@ __uprobe_unregister(struct uprobe *uprob
  * @offset: offset from the start of the file.
  * @uc: identify which probe if multiple probes are colocated.
  */
-void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
+void uprobe_unregister_nosync(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
 {
        struct uprobe *uprobe;
 
@@ -1152,10 +1152,14 @@ void uprobe_unregister(struct inode *ino
        raw_write_seqcount_end(&uprobe->register_seq);
        up_write(&uprobe->register_rwsem);
        put_uprobe(uprobe);
+}
+EXPORT_SYMBOL_GPL(uprobe_unregister_nosync);
 
+void uprobe_unregister_sync(void)
+{
        synchronize_srcu(&uprobes_srcu);
 }
-EXPORT_SYMBOL_GPL(uprobe_unregister);
+EXPORT_SYMBOL_GPL(uprobe_unregister_sync);
 
 /*
  * __uprobe_register - register a probe
kernel/trace/bpf_trace.c +4 -2
@@ -3181,9 +3181,11 @@ static void bpf_uprobe_unregister(struct
        u32 i;
 
        for (i = 0; i < cnt; i++) {
-               uprobe_unregister(d_real_inode(path->dentry), uprobes[i].offset,
-                                 &uprobes[i].consumer);
+               uprobe_unregister_nosync(d_real_inode(path->dentry), uprobes[i].offset,
+                                        &uprobes[i].consumer);
        }
+       if (cnt)
+               uprobe_unregister_sync();
 }
 
 static void bpf_uprobe_multi_link_release(struct bpf_link *link)
kernel/trace/trace_uprobe.c +5 -1
@@ -1104,6 +1104,7 @@ static int trace_uprobe_enable(struct tr
 static void __probe_event_disable(struct trace_probe *tp)
 {
        struct trace_uprobe *tu;
+       bool sync = false;
 
        tu = container_of(tp, struct trace_uprobe, tp);
        WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));
@@ -1112,9 +1113,12 @@ static void __probe_event_disable(struct
                if (!tu->inode)
                        continue;
 
-               uprobe_unregister(tu->inode, tu->offset, &tu->consumer);
+               uprobe_unregister_nosync(tu->inode, tu->offset, &tu->consumer);
+               sync = true;
                tu->inode = NULL;
        }
+       if (sync)
+               uprobe_unregister_sync();
 }
 
 static int probe_event_enable(struct trace_event_call *call,
author maintainer
#18 lore raw

PATCH v2 8/11 perf/uprobe: Convert (some) uprobe->refcount to SRCU

In reply to Peter Zijlstra

With handle_swbp() hitting concurrently on (all) CPUs, potentially on the same uprobe, the uprobe->refcount can get *very* hot. Move the struct uprobe lifetime into uprobes_srcu such that it covers both the uprobe and the uprobe->consumers list.

With this, handle_swbp() can use a single large SRCU critical section to avoid taking a refcount on the uprobe for it's duration.

Notably, the single-step and uretprobe paths need a reference that leaves handle_swbp() and will, for now, still use ->refcount.

---
 kernel/events/uprobes.c |   68 ++++++++++++++++++++++++++++--------------------
 1 file changed, 41 insertions(+), 27 deletions(-)
kernel/events/uprobes.c +41 -27
@@ -51,7 +51,7 @@ static struct mutex uprobes_mmap_mutex[U
 DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem);
 
 /*
- * Covers uprobe->consumers lifetime.
+ * Covers uprobe->consumers lifetime as well as struct uprobe.
  */
 DEFINE_STATIC_SRCU(uprobes_srcu);
 
@@ -626,7 +626,7 @@ static void put_uprobe(struct uprobe *up
 		mutex_lock(&delayed_uprobe_lock);
 		delayed_uprobe_remove(uprobe, NULL);
 		mutex_unlock(&delayed_uprobe_lock);
-		call_rcu(&uprobe->rcu, uprobe_free_rcu);
+		call_srcu(&uprobes_srcu, &uprobe->rcu, uprobe_free_rcu);
 	}
 }
 
@@ -678,7 +678,7 @@ static struct uprobe *__find_uprobe(stru
 	struct rb_node *node = rb_find_rcu(&key, &uprobes_tree, __uprobe_cmp_key);
 
 	if (node)
-		return try_get_uprobe(__node_2_uprobe(node));
+		return __node_2_uprobe(node);
 
 	return NULL;
 }
@@ -691,7 +691,7 @@ static struct uprobe *find_uprobe(struct
 {
 	unsigned int seq;
 
-	guard(rcu)();
+	lockdep_assert(srcu_read_lock_held(&uprobes_srcu));
 
 	do {
 		seq = read_seqcount_begin(&uprobes_seqcount);
@@ -1142,6 +1142,8 @@ void uprobe_unregister_nosync(struct ino
 {
 	struct uprobe *uprobe;
 
+	guard(srcu)(&uprobes_srcu);
+
 	uprobe = find_uprobe(inode, offset);
 	if (WARN_ON(!uprobe))
 		return;
@@ -1151,7 +1153,6 @@ void uprobe_unregister_nosync(struct ino
 	__uprobe_unregister(uprobe, uc);
 	raw_write_seqcount_end(&uprobe->register_seq);
 	up_write(&uprobe->register_rwsem);
-	put_uprobe(uprobe);
 }
 EXPORT_SYMBOL_GPL(uprobe_unregister_nosync);
 
@@ -1263,6 +1264,8 @@ int uprobe_apply(struct inode *inode, lo
 	struct uprobe_consumer *con;
 	int ret = -ENOENT;
 
+	guard(srcu)(&uprobes_srcu);
+
 	uprobe = find_uprobe(inode, offset);
 	if (WARN_ON(!uprobe))
 		return ret;
@@ -1275,7 +1278,6 @@ int uprobe_apply(struct inode *inode, lo
 		ret = register_for_each_vma(uprobe, add ? uc : NULL);
 	raw_write_seqcount_end(&uprobe->register_seq);
 	up_write(&uprobe->register_rwsem);
-	put_uprobe(uprobe);
 
 	return ret;
 }
@@ -1929,10 +1931,14 @@ static void prepare_uretprobe(struct upr
 	if (!ri)
 		return;
 
+	ri->uprobe = try_get_uprobe(uprobe);
+	if (!ri->uprobe)
+		goto err_mem;
+
 	trampoline_vaddr = get_trampoline_vaddr();
 	orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
 	if (orig_ret_vaddr == -1)
-		goto fail;
+		goto err_uprobe;
 
 	/* drop the entries invalidated by longjmp() */
 	chained = (orig_ret_vaddr == trampoline_vaddr);
@@ -1950,12 +1956,11 @@ static void prepare_uretprobe(struct upr
 			 * attack from user-space.
 			 */
 			uprobe_warn(current, "handle tail call");
-			goto fail;
+			goto err_uprobe;
 		}
 		orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
 	}
 
-	ri->uprobe = get_uprobe(uprobe);
 	ri->func = instruction_pointer(regs);
 	ri->stack = user_stack_pointer(regs);
 	ri->orig_ret_vaddr = orig_ret_vaddr;
@@ -1966,7 +1971,10 @@ static void prepare_uretprobe(struct upr
 	utask->return_instances = ri;
 
 	return;
-fail:
+
+err_uprobe:
+	uprobe_put(ri->uprobe);
+err_mem:
 	kfree(ri);
 }
 
@@ -1982,22 +1990,31 @@ pre_ssout(struct uprobe *uprobe, struct
 	if (!utask)
 		return -ENOMEM;
 
+	utask->active_uprobe = try_get_uprobe(uprobe);
+	if (!utask->active_uprobe)
+		return -ESRCH;
+
 	xol_vaddr = xol_get_insn_slot(uprobe);
-	if (!xol_vaddr)
-		return -ENOMEM;
+	if (!xol_vaddr) {
+		err = -ENOMEM;
+		goto err_uprobe;
+	}
 
 	utask->xol_vaddr = xol_vaddr;
 	utask->vaddr = bp_vaddr;
 
 	err = arch_uprobe_pre_xol(&uprobe->arch, regs);
-	if (unlikely(err)) {
-		xol_free_insn_slot(current);
-		return err;
-	}
+	if (unlikely(err))
+		goto err_xol;
 
-	utask->active_uprobe = uprobe;
 	utask->state = UTASK_SSTEP;
 	return 0;
+
+err_xol:
+	xol_free_insn_slot(current);
+err_uprobe:
+	put_uprobe(utask->active_uprobe);
+	return err;
 }
 
 /*
@@ -2128,7 +2145,7 @@ static void handler_chain(struct uprobe
 	bool had_handler = false;
 	unsigned int seq;
 
-	guard(srcu)(&uprobes_srcu);
+	lockdep_assert(srcu_read_lock_held(&uprobes_srcu));
 
 	seq = raw_read_seqcount_begin(&uprobe->register_seq);
 
@@ -2276,6 +2293,8 @@ static void handle_swbp(struct pt_regs *
 	if (bp_vaddr == get_trampoline_vaddr())
 		return handle_trampoline(regs);
 
+	guard(srcu)(&uprobes_srcu);
+
 	uprobe = find_active_uprobe(bp_vaddr, &is_swbp);
 	if (!uprobe) {
 		if (is_swbp > 0) {
@@ -2304,7 +2323,7 @@ static void handle_swbp(struct pt_regs *
 	 * new and not-yet-analyzed uprobe at the same address, restart.
 	 */
 	if (unlikely(!test_bit(UPROBE_COPY_INSN, &uprobe->flags)))
-		goto out;
+		return;
 
 	/*
 	 * Pairs with the smp_wmb() in prepare_uprobe().
@@ -2317,22 +2336,17 @@ static void handle_swbp(struct pt_regs *
 
 	/* Tracing handlers use ->utask to communicate with fetch methods */
 	if (!get_utask())
-		goto out;
+		return;
 
 	if (arch_uprobe_ignore(&uprobe->arch, regs))
-		goto out;
+		return;
 
 	handler_chain(uprobe, regs);
 
 	if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
-		goto out;
-
-	if (!pre_ssout(uprobe, regs, bp_vaddr))
 		return;
 
-	/* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */
-out:
-	put_uprobe(uprobe);
+	pre_ssout(uprobe, regs, bp_vaddr);
 }
 
 /*
#19 lore raw

Re: [PATCH v2 08/11] perf/uprobe: Convert (some) uprobe->refcount to SRCU

On Thu, Jul 11, 2024 at 01:02:43PM +0200, Peter Zijlstra wrote:

SNIP

Quoted 11 lines
kernel/events/uprobes.c +2 -2
 	/* Tracing handlers use ->utask to communicate with fetch methods */
 	if (!get_utask())
-		goto out;
+		return;
 
 	if (arch_uprobe_ignore(&uprobe->arch, regs))
-		goto out;
+		return;
 
 	handler_chain(uprobe, regs);
 

kernel/events/uprobes.c static void handle_swbp(struct pt_regs * · line 2349

Quoted 10 lines
kernel/events/uprobes.c +1 -6
 	if (arch_uprobe_skip_sstep(&uprobe->arch, regs))
-		goto out;
-
-	if (!pre_ssout(uprobe, regs, bp_vaddr))
 		return;
 
-	/* arch_uprobe_skip_sstep() succeeded, or restart if can't singlestep */
-out:
-	put_uprobe(uprobe);
+	pre_ssout(uprobe, regs, bp_vaddr);

pre_ssout could now return void

jirka

Quoted 5 lines

}

/*

#20 lore raw

Re: [PATCH v2 08/11] perf/uprobe: Convert (some) uprobe->refcount to SRCU

In reply to Peter Zijlstra

+ bpf

Quoted 22 lines · Peter Zijlstra

With handle_swbp() hitting concurrently on (all) CPUs, potentially on the same uprobe, the uprobe->refcount can get *very* hot. Move the struct uprobe lifetime into uprobes_srcu such that it covers both the uprobe and the uprobe->consumers list.

With this, handle_swbp() can use a single large SRCU critical section to avoid taking a refcount on the uprobe for it's duration.

Notably, the single-step and uretprobe paths need a reference that leaves handle_swbp() and will, for now, still use ->refcount.

---
 kernel/events/uprobes.c |   68 ++++++++++++++++++++++++++++--------------------
 1 file changed, 41 insertions(+), 27 deletions(-)
kernel/events/uprobes.c +0 -0
@@ -51,7 +51,7 @@ static struct mutex uprobes_mmap_mutex[U
 DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem);
 

[...]

Quoted 12 lines
kernel/events/uprobes.c +5 -2
@@ -1982,22 +1990,31 @@ pre_ssout(struct uprobe *uprobe, struct
        if (!utask)
                return -ENOMEM;
 
+       utask->active_uprobe = try_get_uprobe(uprobe);
+       if (!utask->active_uprobe)
+               return -ESRCH;
+
        xol_vaddr = xol_get_insn_slot(uprobe);
-       if (!xol_vaddr)
-               return -ENOMEM;
+       if (!xol_vaddr) {

kernel/events/uprobes.c pre_ssout(struct uprobe *uprobe, struct · line 1994

Quoted 10 lines
kernel/events/uprobes.c +3 -2
+               err = -ENOMEM;
+               goto err_uprobe;
+       }
 
        utask->xol_vaddr = xol_vaddr;
        utask->vaddr = bp_vaddr;
 
        err = arch_uprobe_pre_xol(&uprobe->arch, regs);
-       if (unlikely(err)) {
-               xol_free_insn_slot(current);

let's keep this here, because you later remove err_uprobe part and err_xol is only jumped to from here; it's better to just drop err_xol and err_uprobe altogether and keep the original xol_free_insn_slot() here.

kernel/events/uprobes.c pre_ssout(struct uprobe *uprobe, struct · line 2016

Quoted 13 lines
kernel/events/uprobes.c +7 -3
-               return err;
-       }
+       if (unlikely(err))
+               goto err_xol;
 
-       utask->active_uprobe = uprobe;
        utask->state = UTASK_SSTEP;
        return 0;
+
+err_xol:
+       xol_free_insn_slot(current);
+err_uprobe:
+       put_uprobe(utask->active_uprobe);

utask->active_uprobe = NULL;

let's not leave garbage in utask (even if you remove this later anyways)

Quoted 4 lines
kernel/events/uprobes.c +1 -0
+       return err;
 }
 
 /*

[...]

author maintainer
#21 lore raw

PATCH v2 9/11 srcu: Add __srcu_clone_read_lock()

In reply to Peter Zijlstra

In order to support carrying an srcu_read_lock() section across fork, where both the parent and child process will do: srcu_read_unlock(), it is needed to account for the extra decrement with an extra increment at fork time.

---
 include/linux/srcu.h     |    1 +
 include/linux/srcutiny.h |   10 ++++++++++
 kernel/rcu/srcutree.c    |    5 +++++
 3 files changed, 16 insertions(+)
include/linux/srcu.h +1 -0
@@ -55,6 +55,7 @@ void call_srcu(struct srcu_struct *ssp,
 		void (*func)(struct rcu_head *head));
 void cleanup_srcu_struct(struct srcu_struct *ssp);
 int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp);
+void __srcu_clone_read_lock(struct srcu_struct *ssp, int idx);
 void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);
 void synchronize_srcu(struct srcu_struct *ssp);
 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp);
include/linux/srcutiny.h +10 -0
@@ -71,6 +71,16 @@ static inline int __srcu_read_lock(struc
 	return idx;
 }
 
+static inline void __srcu_clone_read_lock(struct srcu_struct *ssp, int idx)
+{
+	int newval;
+
+	preempt_disable();  // Needed for PREEMPT_AUTO
+	newval = READ_ONCE(ssp->srcu_lock_nesting[idx]) + 1;
+	WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval);
+	preempt_enable();
+}
+
 static inline void synchronize_srcu_expedited(struct srcu_struct *ssp)
 {
 	synchronize_srcu(ssp);
kernel/rcu/srcutree.c +5 -0
@@ -720,6 +720,11 @@ int __srcu_read_lock(struct srcu_struct
 }
 EXPORT_SYMBOL_GPL(__srcu_read_lock);
 
+void __srcu_clone_read_lock(struct srcu_struct *ssp, int idx)
+{
+	this_cpu_inc(ssp->sda->srcu_lock_count[idx].counter);
+}
+
 /*
  * Removes the count for the old reader from the appropriate per-CPU
  * element of the srcu_struct.  Note that this may well be a different
author maintainer
#22 lore raw

PATCH v2 10/11 perf/uprobe: Convert single-step and uretprobe to SRCU

In reply to Peter Zijlstra

Both single-step and uretprobes take a refcount on struct uprobe in handle_swbp() in order to ensure struct uprobe stays extant until a next trap.

Since uprobe_unregister() only cares about the uprobe_consumer life-time, and these intra-trap sections can be arbitrarily large, create a second SRCU domain to cover these.

Notably, a uretprobe with a registered return_instance that never triggers -- because userspace -- will currently pin the return_instance and related uprobe until the task dies. With this convertion to SRCU this behaviour will inhibit freeing of all uprobes.

---
 include/linux/uprobes.h |    2 +
 kernel/events/uprobes.c |   60 +++++++++++++++++++++++-------------------------
 2 files changed, 31 insertions(+), 31 deletions(-)
include/linux/uprobes.h +2 -0
@@ -78,6 +78,7 @@ struct uprobe_task {
 
 	struct return_instance		*return_instances;
 	unsigned int			depth;
+	unsigned int			active_srcu_idx;
 };
 
 struct return_instance {
@@ -86,6 +87,7 @@ struct return_instance {
 	unsigned long		stack;		/* stack pointer */
 	unsigned long		orig_ret_vaddr; /* original return address */
 	bool			chained;	/* true, if instance is nested */
+	int			srcu_idx;
 
 	struct return_instance	*next;		/* keep as stack */
 };
kernel/events/uprobes.c +29 -31

156 lines not on this page · raw message

maintainer
#23 lore raw

Re: [PATCH v2 10/11] perf/uprobe: Convert single-step and uretprobe to SRCU

I'll try to actually apply the whole series and read the code tomorrow. Right now I can't understand this change... Just one question for now.

kernel/events/uprobes.c static void prepare_uretprobe(struct upr · line 1969

Quoted 13 lines · Peter Zijlstra
kernel/events/uprobes.c +3 -1
@@ -1956,11 +1960,13 @@ static void prepare_uretprobe(struct upr
 			 * attack from user-space.
 			 */
 			uprobe_warn(current, "handle tail call");
-			goto err_uprobe;
+			goto err_mem;
 		}
 		orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
 	}
 
+	ri->srcu_idx = __srcu_read_lock(&uretprobes_srcu);
+	ri->uprobe = uprobe;

It seems that, if we race with _unregister, this __srcu_read_lock() can happen after call_srcu(uprobes_srcu, uprobe, uprobe_free_stage1) was already called...

In this case read_lock "has no effect" in that uprobe_free_stage1() can run before free_ret_instance() does srcu_read_unlock(ri->srcu_idx).

Perhaps it is fine, uprobe_free_stage1() does another call_srcu(), but somehow I got lost.

Could you re-check this logic? Most probably I missed something, but still...

Oleg.

author maintainer
#24 lore raw

Re: [PATCH v2 10/11] perf/uprobe: Convert single-step and uretprobe to SRCU

Quoted 22 lines · Oleg Nesterov

I'll try to actually apply the whole series and read the code tomorrow. Right now I can't understand this change... Just one question for now.

kernel/events/uprobes.c static void prepare_uretprobe(struct upr · line 1969

Quoted 13 lines · Peter Zijlstra
kernel/events/uprobes.c +3 -1
@@ -1956,11 +1960,13 @@ static void prepare_uretprobe(struct upr
 			 * attack from user-space.
 			 */
 			uprobe_warn(current, "handle tail call");
-			goto err_uprobe;
+			goto err_mem;
 		}
 		orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
 	}
 
+	ri->srcu_idx = __srcu_read_lock(&uretprobes_srcu);
+	ri->uprobe = uprobe;

It seems that, if we race with _unregister, this __srcu_read_lock() can happen after call_srcu(uprobes_srcu, uprobe, uprobe_free_stage1) was already called...

Quoted 7 lines

In this case read_lock "has no effect" in that uprobe_free_stage1() can run before free_ret_instance() does srcu_read_unlock(ri->srcu_idx).

Perhaps it is fine, uprobe_free_stage1() does another call_srcu(), but somehow I got lost.

Could you re-check this logic? Most probably I missed something, but still...

  handle_swbp()
    guard(srcu)(&uprobes_srcu);
    handle_chain();
      prepare_uretprobe()
        __srcu_read_lock(&uretprobe_srcu);

vs

uprobe_free_stage2
kfree(uprobe)

  uprobe_free_stage1
    call_srcu(&uretprobe_srcu, &uprobe->rcu, uprobe_free_stage2);

  put_uprobe()
    if (refcount_dec_and_test)
      call_srcu(&uprobes_srcu, &uprobe->rcu, uprobe_free_stage1);

So my thinking was since we take uretprobe_srcu *inside* uprobe_srcu, this reference must be visible before we execute stage1, and as such stage2 cannot complete prematurely.

maintainer
#25 lore raw

Re: [PATCH v2 10/11] perf/uprobe: Convert single-step and uretprobe to SRCU

Quoted 10 lines · Peter Zijlstra
  uprobe_free_stage1
    call_srcu(&uretprobe_srcu, &uprobe->rcu, uprobe_free_stage2);

  put_uprobe()
    if (refcount_dec_and_test)
      call_srcu(&uprobes_srcu, &uprobe->rcu, uprobe_free_stage1);

So my thinking was since we take uretprobe_srcu *inside* uprobe_srcu,

Ah, indeed! Somehow misread the change in put_uprobe() as if it uses call_rcu(uretprobe_srcu).

Thanks,

Oleg.

#26 lore raw

Re: [PATCH v2 10/11] perf/uprobe: Convert single-step and uretprobe to SRCU

In reply to Peter Zijlstra

+ bpf

Quoted 117 lines · Peter Zijlstra

Both single-step and uretprobes take a refcount on struct uprobe in handle_swbp() in order to ensure struct uprobe stays extant until a next trap.

Since uprobe_unregister() only cares about the uprobe_consumer life-time, and these intra-trap sections can be arbitrarily large, create a second SRCU domain to cover these.

Notably, a uretprobe with a registered return_instance that never triggers -- because userspace -- will currently pin the return_instance and related uprobe until the task dies. With this convertion to SRCU this behaviour will inhibit freeing of all uprobes.

---
 include/linux/uprobes.h |    2 +
 kernel/events/uprobes.c |   60 +++++++++++++++++++++++-------------------------
 2 files changed, 31 insertions(+), 31 deletions(-)
include/linux/uprobes.h +2 -0
@@ -78,6 +78,7 @@ struct uprobe_task {
 
        struct return_instance          *return_instances;
        unsigned int                    depth;
+       unsigned int                    active_srcu_idx;
 };
 
 struct return_instance {
@@ -86,6 +87,7 @@ struct return_instance {
        unsigned long           stack;          /* stack pointer */
        unsigned long           orig_ret_vaddr; /* original return address */
        bool                    chained;        /* true, if instance is nested */
+       int                     srcu_idx;
 
        struct return_instance  *next;          /* keep as stack */
 };
kernel/events/uprobes.c +19 -11
@@ -54,6 +54,15 @@ DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem)
  * Covers uprobe->consumers lifetime as well as struct uprobe.
  */
 DEFINE_STATIC_SRCU(uprobes_srcu);
+/*
+ * Covers return_instance->uprobe and utask->active_uprobe. Separate from
+ * uprobe_srcu because uprobe_unregister() doesn't need to wait for this
+ * and these lifetimes can be fairly long.
+ *
+ * Notably, these sections span userspace and as such use
+ * __srcu_read_{,un}lock() to elide lockdep.
+ */
+DEFINE_STATIC_SRCU(uretprobes_srcu);
 
 /* Have a copy of original instruction */
 #define UPROBE_COPY_INSN       0
@@ -596,25 +605,24 @@ set_orig_insn(struct arch_uprobe *auprob
                        *(uprobe_opcode_t *)&auprobe->insn);
 }
 
-static struct uprobe *try_get_uprobe(struct uprobe *uprobe)
-{
-       if (refcount_inc_not_zero(&uprobe->ref))
-               return uprobe;
-       return NULL;
-}
-
 static struct uprobe *get_uprobe(struct uprobe *uprobe)
 {
        refcount_inc(&uprobe->ref);
        return uprobe;
 }
 
-static void uprobe_free_rcu(struct rcu_head *rcu)
+static void uprobe_free_stage2(struct rcu_head *rcu)
 {
        struct uprobe *uprobe = container_of(rcu, struct uprobe, rcu);
        kfree(uprobe);
 }
 
+static void uprobe_free_stage1(struct rcu_head *rcu)
+{
+       struct uprobe *uprobe = container_of(rcu, struct uprobe, rcu);
+       call_srcu(&uretprobes_srcu, &uprobe->rcu, uprobe_free_stage2);
+}
+
 static void put_uprobe(struct uprobe *uprobe)
 {
        if (refcount_dec_and_test(&uprobe->ref)) {
@@ -626,7 +634,7 @@ static void put_uprobe(struct uprobe *up
                mutex_lock(&delayed_uprobe_lock);
                delayed_uprobe_remove(uprobe, NULL);
                mutex_unlock(&delayed_uprobe_lock);
-               call_srcu(&uprobes_srcu, &uprobe->rcu, uprobe_free_rcu);
+               call_srcu(&uprobes_srcu, &uprobe->rcu, uprobe_free_stage1);
        }
 }
 
@@ -1753,7 +1761,7 @@ unsigned long uprobe_get_trap_addr(struc
 static struct return_instance *free_ret_instance(struct return_instance *ri)
 {
        struct return_instance *next = ri->next;
-       put_uprobe(ri->uprobe);
+       __srcu_read_unlock(&uretprobes_srcu, ri->srcu_idx);
        kfree(ri);
        return next;
 }
@@ -1771,7 +1779,7 @@ void uprobe_free_utask(struct task_struc
                return;
 
        if (utask->active_uprobe)
-               put_uprobe(utask->active_uprobe);
+               __srcu_read_unlock(&uretprobes_srcu, utask->active_srcu_idx);
 
        ri = utask->return_instances;
        while (ri)

kernel/events/uprobes.c static int dup_utask(struct task_struct · line 1825

Quoted 5 lines
kernel/events/uprobes.c +1 -1
@@ -1814,7 +1822,7 @@ static int dup_utask(struct task_struct
                        return -ENOMEM;
 
                *n = *o;
-               get_uprobe(n->uprobe);
+               __srcu_clone_read_lock(&uretprobes_srcu, n->srcu_idx);

do we need to add this __srcu_clone_read_lock hack just to avoid taking a refcount in dup_utask (i.e., on process fork)? This is not that frequent and performance-sensitive case, so it seems like it should be fine to take refcount and avoid doing srcu_read_unlock() in a new process. Just like the case with long-running uretprobes where you convert SRCU lock into refcount.

Quoted 88 lines
kernel/events/uprobes.c +9 -19
                n->next = NULL;
 
                *p = n;
@@ -1931,14 +1939,10 @@ static void prepare_uretprobe(struct upr
        if (!ri)
                return;
 
-       ri->uprobe = try_get_uprobe(uprobe);
-       if (!ri->uprobe)
-               goto err_mem;
-
        trampoline_vaddr = get_trampoline_vaddr();
        orig_ret_vaddr = arch_uretprobe_hijack_return_addr(trampoline_vaddr, regs);
        if (orig_ret_vaddr == -1)
-               goto err_uprobe;
+               goto err_mem;
 
        /* drop the entries invalidated by longjmp() */
        chained = (orig_ret_vaddr == trampoline_vaddr);
@@ -1956,11 +1960,13 @@ static void prepare_uretprobe(struct upr
                         * attack from user-space.
                         */
                        uprobe_warn(current, "handle tail call");
-                       goto err_uprobe;
+                       goto err_mem;
                }
                orig_ret_vaddr = utask->return_instances->orig_ret_vaddr;
        }
 
+       ri->srcu_idx = __srcu_read_lock(&uretprobes_srcu);
+       ri->uprobe = uprobe;
        ri->func = instruction_pointer(regs);
        ri->stack = user_stack_pointer(regs);
        ri->orig_ret_vaddr = orig_ret_vaddr;
@@ -1972,8 +1978,6 @@ static void prepare_uretprobe(struct upr
 
        return;
 
-err_uprobe:
-       uprobe_put(ri->uprobe);
 err_mem:
        kfree(ri);
 }
@@ -1990,15 +1994,9 @@ pre_ssout(struct uprobe *uprobe, struct
        if (!utask)
                return -ENOMEM;
 
-       utask->active_uprobe = try_get_uprobe(uprobe);
-       if (!utask->active_uprobe)
-               return -ESRCH;
-
        xol_vaddr = xol_get_insn_slot(uprobe);
-       if (!xol_vaddr) {
-               err = -ENOMEM;
-               goto err_uprobe;
-       }
+       if (!xol_vaddr)
+               return -ENOMEM;
 
        utask->xol_vaddr = xol_vaddr;
        utask->vaddr = bp_vaddr;
@@ -2007,13 +2005,13 @@ pre_ssout(struct uprobe *uprobe, struct
        if (unlikely(err))
                goto err_xol;
 
+       utask->active_srcu_idx = __srcu_read_lock(&uretprobes_srcu);
+       utask->active_uprobe = uprobe;
        utask->state = UTASK_SSTEP;
        return 0;
 
 err_xol:
        xol_free_insn_slot(current);
-err_uprobe:
-       put_uprobe(utask->active_uprobe);
        return err;
 }
 
@@ -2366,7 +2364,7 @@ static void handle_singlestep(struct upr
        else
                WARN_ON_ONCE(1);
 
-       put_uprobe(uprobe);
+       __srcu_read_unlock(&uretprobes_srcu, utask->active_srcu_idx);
        utask->active_uprobe = NULL;
        utask->state = UTASK_RUNNING;
        xol_free_insn_slot(current);
author maintainer
#27 lore raw

Re: [PATCH v2 10/11] perf/uprobe: Convert single-step and uretprobe to SRCU

Quoted 13 lines · Andrii Nakryiko

kernel/events/uprobes.c static int dup_utask(struct task_struct · line 1825

Quoted 6 lines
kernel/events/uprobes.c +1 -1
@@ -1814,7 +1822,7 @@ static int dup_utask(struct task_struct
                        return -ENOMEM;
 
                *n = *o;
-               get_uprobe(n->uprobe);
+               __srcu_clone_read_lock(&uretprobes_srcu, n->srcu_idx);

do we need to add this __srcu_clone_read_lock hack just to avoid taking a refcount in dup_utask (i.e., on process fork)? This is not that frequent and performance-sensitive case, so it seems like it should be fine to take refcount and avoid doing srcu_read_unlock() in a new process. Just like the case with long-running uretprobes where you convert SRCU lock into refcount.

Yes, I suppose that is now possible too. But it makes the patches harder to split. Let me ponder that after I get it to pass your stress thing.

author maintainer
#28 lore raw

PATCH v2 11/11 perf/uprobe: Add uretprobe timer

In reply to Peter Zijlstra

In order to put a bound on the uretprobe_srcu critical section, add a timer to uprobe_task. Upon every RI added or removed the timer is pushed forward to now + 1s. If the timer were ever to fire, it would convert the SRCU 'reference' to a refcount reference if possible.

---
 include/linux/uprobes.h |    8 +++++
 kernel/events/uprobes.c |   67 ++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 69 insertions(+), 6 deletions(-)
include/linux/uprobes.h +7 -1
@@ -15,6 +15,7 @@
 #include <linux/rbtree.h>
 #include <linux/types.h>
 #include <linux/wait.h>
+#include <linux/timer.h>
 
 struct vm_area_struct;
 struct mm_struct;
@@ -79,6 +80,10 @@ struct uprobe_task {
 	struct return_instance		*return_instances;
 	unsigned int			depth;
 	unsigned int			active_srcu_idx;
+
+	struct timer_list		ri_timer;
+	struct callback_head		ri_task_work;
+	struct task_struct		*task;
 };
 
 struct return_instance {
@@ -86,7 +91,8 @@ struct return_instance {
 	unsigned long		func;
 	unsigned long		stack;		/* stack pointer */
 	unsigned long		orig_ret_vaddr; /* original return address */
-	bool			chained;	/* true, if instance is nested */
+	u8			chained;	/* true, if instance is nested */
+	u8			has_ref;
 	int			srcu_idx;
 
 	struct return_instance	*next;		/* keep as stack */
kernel/events/uprobes.c +62 -5

133 lines not on this page · raw message

maintainer
#29 lore raw

Re: [PATCH v2 11/11] perf/uprobe: Add uretprobe timer

Not sure I read this patch correctly, but at first glance it looks suspicious..

kernel/events/uprobes.c void uprobe_free_utask(struct task_struc · line 1822

Quoted 6 lines · Peter Zijlstra
kernel/events/uprobes.c +5 -0
+static void return_instance_timer(struct timer_list *timer)
+{
+	struct uprobe_task *utask = container_of(timer, struct uprobe_task, ri_timer);
+	task_work_add(utask->task, &utask->ri_task_work, TWA_SIGNAL);
+}

What if utask->task sleeps in TASK_STOPPED/TASK_TRACED state before return from the ret-probed function?

In this case it won't react to TWA_SIGNAL until debugger or SIGCONT wakes it up.

---------------------------------------------------------------------------
And it seems that even task_work_add() itself is not safe...

Suppose we have 2 ret-probed functions

	void f2() { ... }
	void f1() { ...; f2(); }

A task T calls f1(), hits the bp, and calls prepare_uretprobe() which does

	mod_timer(&utask->ri_timer, jiffies + HZ);

Then later it calls f2() and the pending timer expires after it enters the kernel, but before the next prepare_uretprobe() -> mod_timer().

In this case ri_task_work is already queued and the timer is pending again.

Now. Even if T goes to the exit_to_user_mode_loop() path immediately, in theory nothing can guarantee that it will call get_signal/task_work_run in less than 1 second, it can be preempted.

But T can sleep in xol_take_insn_slot() before return from handle_swbp(), and this is not so theoretical.

Oleg.

author maintainer
#30 lore raw

Re: [PATCH v2 11/11] perf/uprobe: Add uretprobe timer

Quoted 14 lines · Oleg Nesterov

Not sure I read this patch correctly, but at first glance it looks suspicious..

kernel/events/uprobes.c void uprobe_free_utask(struct task_struc · line 1822

Quoted 6 lines · Peter Zijlstra
kernel/events/uprobes.c +5 -0
+static void return_instance_timer(struct timer_list *timer)
+{
+	struct uprobe_task *utask = container_of(timer, struct uprobe_task, ri_timer);
+	task_work_add(utask->task, &utask->ri_task_work, TWA_SIGNAL);
+}

What if utask->task sleeps in TASK_STOPPED/TASK_TRACED state before return from the ret-probed function?

Quoted 2 lines

In this case it won't react to TWA_SIGNAL until debugger or SIGCONT wakes it up.

Or FROZEN etc.. Yeah.

Quoted 8 lines

---------------------------------------------------------------------------
And it seems that even task_work_add() itself is not safe...

Suppose we have 2 ret-probed functions

	void f2() { ... }
	void f1() { ...; f2(); }
Quoted 8 lines

A task T calls f1(), hits the bp, and calls prepare_uretprobe() which does

	mod_timer(&utask->ri_timer, jiffies + HZ);

Then later it calls f2() and the pending timer expires after it enters the kernel, but before the next prepare_uretprobe() -> mod_timer().

In this case ri_task_work is already queued and the timer is pending again.

You're saying we can hit a double enqueue, right? Yeah, that's a problem. But that can be fairly easily rectified.

Quoted 6 lines

Now. Even if T goes to the exit_to_user_mode_loop() path immediately, in theory nothing can guarantee that it will call get_signal/task_work_run in less than 1 second, it can be preempted.

But T can sleep in xol_take_insn_slot() before return from handle_swbp(), and this is not so theoretical.

So the assumption is that kernel code makes forward progress. If we get preempted, we'll get scheduled again. If the machine is so overloaded this takes more than a second, stretching the SRCU period is the least of your problems.

Same with sleeps, it'll get a wakeup.

The only thing that is out of our control is userspace. And yes, I had not considered STOPPED/TRACED/FROZEN.

So the reason I did that task_work is because the return_instance list is strictly current, so a random timer cannot safely poke at it. And barring those pesky states, it does as desired.

Let me ponder that a little, I *can* make it work, but all 'solutions' I've come up with so far are really rather vile.

author maintainer
#31 lore raw

Re: [PATCH v2 11/11] perf/uprobe: Add uretprobe timer

Quoted 2 lines · Peter Zijlstra

Let me ponder that a little, I *can* make it work, but all 'solutions' I've come up with so far are really rather vile.

This is the least horrible solution I could come up with...

---

include/linux/uprobes.h +1 -0
@@ -83,6 +83,7 @@ struct uprobe_task {
 
 	struct timer_list		ri_timer;
 	struct callback_head		ri_task_work;
+	bool				ri_work_pending;
 	struct task_struct		*task;
 };
 
kernel/events/uprobes.c +35 -2
@@ -1797,9 +1797,8 @@ void uprobe_free_utask(struct task_struc
 	t->utask = NULL;
 }
 
-static void return_instance_task_work(struct callback_head *head)
+static void __return_instance_put(struct uprobe_task *utask)
 {
-	struct uprobe_task *utask = container_of(head, struct uprobe_task, ri_task_work);
 	struct return_instance *ri;
 
 	for (ri = utask->return_instances; ri; ri = ri->next) {
@@ -1815,9 +1814,43 @@ static void return_instance_task_work(st
 	}
 }
 
+static void return_instance_task_work(struct callback_head *head)
+{
+	struct uprobe_task *utask = container_of(head, struct uprobe_task, ri_task_work);
+	utask->ri_work_pending = false;
+	__return_instance_put(utask);
+}
+
+static int return_instance_blocked(struct task_struct *p, void *arg)
+{
+	unsigned int state = READ_ONCE(p->__state);
+
+	if (state == TASK_RUNNING || state == TASK_WAKING)
+		return 0;
+
+	smp_rmb();
+	if (p->on_rq)
+		return 0;
+
+	/*
+	 * Per __task_needs_rq_locked() we now have: !p->on_cpu and only hold
+	 * p->pi_lock, and can consiter the task fully blocked.
+	 */
+
+	__return_instance_put(p->utask);
+	return 1;
+}
+
 static void return_instance_timer(struct timer_list *timer)
 {
 	struct uprobe_task *utask = container_of(timer, struct uprobe_task, ri_timer);
+	if (utask->ri_work_pending)
+		return;
+
+	if (task_call_func(utask->task, return_instance_blocked, NULL))
+		return;
+
+	utask->ri_work_pending = true;
 	task_work_add(utask->task, &utask->ri_task_work, TWA_SIGNAL);
 }
 
author maintainer
#32 lore raw

Re: [PATCH v2 11/11] perf/uprobe: Add uretprobe timer

Quoted 49 lines · Peter Zijlstra
Quoted 2 lines · Peter Zijlstra

Let me ponder that a little, I *can* make it work, but all 'solutions' I've come up with so far are really rather vile.

This is the least horrible solution I could come up with...

---

include/linux/uprobes.h +1 -0
@@ -83,6 +83,7 @@ struct uprobe_task {
 
 	struct timer_list		ri_timer;
 	struct callback_head		ri_task_work;
+	bool				ri_work_pending;
 	struct task_struct		*task;
 };
 
kernel/events/uprobes.c +15 -2
@@ -1797,9 +1797,8 @@ void uprobe_free_utask(struct task_struc
 	t->utask = NULL;
 }
 
-static void return_instance_task_work(struct callback_head *head)
+static void __return_instance_put(struct uprobe_task *utask)
 {
-	struct uprobe_task *utask = container_of(head, struct uprobe_task, ri_task_work);
 	struct return_instance *ri;
 
 	for (ri = utask->return_instances; ri; ri = ri->next) {
@@ -1815,9 +1814,43 @@ static void return_instance_task_work(st
 	}
 }
 
+static void return_instance_task_work(struct callback_head *head)
+{
+	struct uprobe_task *utask = container_of(head, struct uprobe_task, ri_task_work);
+	utask->ri_work_pending = false;
+	__return_instance_put(utask);
+}
+
+static int return_instance_blocked(struct task_struct *p, void *arg)
+{
+	unsigned int state = READ_ONCE(p->__state);
+
+	if (state == TASK_RUNNING || state == TASK_WAKING)
+		return 0;
+

kernel/events/uprobes.c static void return_instance_task_work(st · line 1840

Quoted 10 lines
kernel/events/uprobes.c +10 -0
+	smp_rmb();
+	if (p->on_rq)
+		return 0;
+
+	/*
+	 * Per __task_needs_rq_locked() we now have: !p->on_cpu and only hold
+	 * p->pi_lock, and can consiter the task fully blocked.
+	 */
+
+	__return_instance_put(p->utask);

PREEMPT_RT might not like this though, doing the full RI iteration under a raw_spinlock_t...

I just can't think of anything saner just now. Oh well, let me go make dinner, perhaps something will come to me.

#33 lore raw

Re: [PATCH v2 11/11] perf/uprobe: Add uretprobe timer

In reply to Peter Zijlstra

+ bpf

Quoted 34 lines · Peter Zijlstra

In order to put a bound on the uretprobe_srcu critical section, add a timer to uprobe_task. Upon every RI added or removed the timer is pushed forward to now + 1s. If the timer were ever to fire, it would convert the SRCU 'reference' to a refcount reference if possible.

---
 include/linux/uprobes.h |    8 +++++
 kernel/events/uprobes.c |   67 ++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 69 insertions(+), 6 deletions(-)
include/linux/uprobes.h +5 -0
@@ -15,6 +15,7 @@
 #include <linux/rbtree.h>
 #include <linux/types.h>
 #include <linux/wait.h>
+#include <linux/timer.h>
 
 struct vm_area_struct;
 struct mm_struct;
@@ -79,6 +80,10 @@ struct uprobe_task {
        struct return_instance          *return_instances;
        unsigned int                    depth;
        unsigned int                    active_srcu_idx;
+
+       struct timer_list               ri_timer;
+       struct callback_head            ri_task_work;
+       struct task_struct              *task;
 };
 
 struct return_instance {

include/linux/uprobes.h struct return_instance { · line 95

Quoted 6 lines
include/linux/uprobes.h +2 -1
@@ -86,7 +91,8 @@ struct return_instance {
        unsigned long           func;
        unsigned long           stack;          /* stack pointer */
        unsigned long           orig_ret_vaddr; /* original return address */
-       bool                    chained;        /* true, if instance is nested */
+       u8                      chained;        /* true, if instance is nested */
+       u8                      has_ref;

Why bool -> u8 switch? You don't touch chained, so why change its type? And for has_ref you interchangeably use 0 and true for the same field. Let's stick to bool as there is nothing wrong with it?

Quoted 3 lines
include/linux/uprobes.h +0 -0
        int                     srcu_idx;
 
        struct return_instance  *next;          /* keep as stack */

[...]

Quoted 9 lines
include/linux/uprobes.h +4 -1
@@ -1822,13 +1864,20 @@ static int dup_utask(struct task_struct
                        return -ENOMEM;
 
                *n = *o;
-               __srcu_clone_read_lock(&uretprobes_srcu, n->srcu_idx);
+               if (n->uprobe) {
+                       if (n->has_ref)
+                               get_uprobe(n->uprobe);
+                       else

include/linux/uprobes.h static int dup_utask(struct task_struct · line 1880

Quoted 10 lines
include/linux/uprobes.h +4 -0
+                               __srcu_clone_read_lock(&uretprobes_srcu, n->srcu_idx);
+               }
                n->next = NULL;
 
                *p = n;
                p = &n->next;
                n_utask->depth++;
        }
+       if (n_utask->return_instances)
+               mod_timer(&n_utask->ri_timer, jiffies + HZ);

let's add #define for HZ, so it's adjusted in just one place (instead of 3 as it is right now)

Also, we can have up to 64 levels of uretprobe nesting, so, technically, the user can cause a delay of 64 seconds in total. Maybe let's use something smaller than a full second? After all, if the user-space function has high latency, then this refcount congestion is much less of a problem. I'd set it to something like 50-100 ms for starters.

Quoted 44 lines
include/linux/uprobes.h +9 -1
        return 0;
 }
@@ -1967,6 +2016,7 @@ static void prepare_uretprobe(struct upr
 
        ri->srcu_idx = __srcu_read_lock(&uretprobes_srcu);
        ri->uprobe = uprobe;
+       ri->has_ref = 0;
        ri->func = instruction_pointer(regs);
        ri->stack = user_stack_pointer(regs);
        ri->orig_ret_vaddr = orig_ret_vaddr;
@@ -1976,6 +2026,8 @@ static void prepare_uretprobe(struct upr
        ri->next = utask->return_instances;
        utask->return_instances = ri;
 
+       mod_timer(&utask->ri_timer, jiffies + HZ);
+
        return;
 
 err_mem:
@@ -2204,6 +2256,9 @@ handle_uretprobe_chain(struct return_ins
        struct uprobe *uprobe = ri->uprobe;
        struct uprobe_consumer *uc;
 
+       if (!uprobe)
+               return;
+
        guard(srcu)(&uprobes_srcu);
 
        for_each_consumer_rcu(uc, uprobe->consumers) {
@@ -2250,8 +2305,10 @@ static void handle_trampoline(struct pt_
 
                instruction_pointer_set(regs, ri->orig_ret_vaddr);
                do {
-                       if (valid)
+                       if (valid) {
                                handle_uretprobe_chain(ri, regs);
+                               mod_timer(&utask->ri_timer, jiffies + HZ);
+                       }
                        ri = free_ret_instance(ri);
                        utask->depth--;
                } while (ri != next);
author maintainer
#34 lore raw

Re: [PATCH v2 11/11] perf/uprobe: Add uretprobe timer

Quoted 35 lines · Andrii Nakryiko

+ bpf

Quoted 32 lines · Peter Zijlstra

In order to put a bound on the uretprobe_srcu critical section, add a timer to uprobe_task. Upon every RI added or removed the timer is pushed forward to now + 1s. If the timer were ever to fire, it would convert the SRCU 'reference' to a refcount reference if possible.

---
 include/linux/uprobes.h |    8 +++++
 kernel/events/uprobes.c |   67 ++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 69 insertions(+), 6 deletions(-)
include/linux/uprobes.h +5 -0
@@ -15,6 +15,7 @@
 #include <linux/rbtree.h>
 #include <linux/types.h>
 #include <linux/wait.h>
+#include <linux/timer.h>
 
 struct vm_area_struct;
 struct mm_struct;
@@ -79,6 +80,10 @@ struct uprobe_task {
        struct return_instance          *return_instances;
        unsigned int                    depth;
        unsigned int                    active_srcu_idx;
+
+       struct timer_list               ri_timer;
+       struct callback_head            ri_task_work;
+       struct task_struct              *task;
 };
 
Quoted 12 lines

include/linux/uprobes.h struct return_instance { · line 95

Quoted 8 lines
include/linux/uprobes.h +2 -1
 struct return_instance {
@@ -86,7 +91,8 @@ struct return_instance {
        unsigned long           func;
        unsigned long           stack;          /* stack pointer */
        unsigned long           orig_ret_vaddr; /* original return address */
-       bool                    chained;        /* true, if instance is nested */
+       u8                      chained;        /* true, if instance is nested */
+       u8                      has_ref;

Why bool -> u8 switch? You don't touch chained, so why change its type? And for has_ref you interchangeably use 0 and true for the same field. Let's stick to bool as there is nothing wrong with it?

sizeof(_Bool) is implementation defined. It is 1 for x86_64, but there are platforms where it ends up begin 4 (some PowerPC ABIs among others. I didn't want to grow this structure for no reason.

Quoted 19 lines
Quoted 3 lines
include/linux/uprobes.h +0 -0
        int                     srcu_idx;
 
        struct return_instance  *next;          /* keep as stack */

[...]

Quoted 13 lines
include/linux/uprobes.h +6 -1
@@ -1822,13 +1864,20 @@ static int dup_utask(struct task_struct
                        return -ENOMEM;
 
                *n = *o;
-               __srcu_clone_read_lock(&uretprobes_srcu, n->srcu_idx);
+               if (n->uprobe) {
+                       if (n->has_ref)
+                               get_uprobe(n->uprobe);
+                       else
+                               __srcu_clone_read_lock(&uretprobes_srcu, n->srcu_idx);
+               }
                n->next = NULL;
 
Quoted 9 lines

include/linux/uprobes.h static int dup_utask(struct task_struct · line 1880

Quoted 6 lines
include/linux/uprobes.h +2 -0
                *p = n;
                p = &n->next;
                n_utask->depth++;
        }
+       if (n_utask->return_instances)
+               mod_timer(&n_utask->ri_timer, jiffies + HZ);

let's add #define for HZ, so it's adjusted in just one place (instead of 3 as it is right now)

Can do I suppose.

Quoted 6 lines

Also, we can have up to 64 levels of uretprobe nesting, so, technically, the user can cause a delay of 64 seconds in total. Maybe let's use something smaller than a full second? After all, if the user-space function has high latency, then this refcount congestion is much less of a problem. I'd set it to something like 50-100 ms for starters.

Before you know it we'll have a sysctl :/ But sure, we can do something shorter.

#35 lore raw

Re: [PATCH v2 11/11] perf/uprobe: Add uretprobe timer

Quoted 50 lines · Peter Zijlstra
Quoted 35 lines · Andrii Nakryiko

+ bpf

Quoted 32 lines · Peter Zijlstra

In order to put a bound on the uretprobe_srcu critical section, add a timer to uprobe_task. Upon every RI added or removed the timer is pushed forward to now + 1s. If the timer were ever to fire, it would convert the SRCU 'reference' to a refcount reference if possible.

---
 include/linux/uprobes.h |    8 +++++
 kernel/events/uprobes.c |   67 ++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 69 insertions(+), 6 deletions(-)
include/linux/uprobes.h +5 -0
@@ -15,6 +15,7 @@
 #include <linux/rbtree.h>
 #include <linux/types.h>
 #include <linux/wait.h>
+#include <linux/timer.h>
 
 struct vm_area_struct;
 struct mm_struct;
@@ -79,6 +80,10 @@ struct uprobe_task {
        struct return_instance          *return_instances;
        unsigned int                    depth;
        unsigned int                    active_srcu_idx;
+
+       struct timer_list               ri_timer;
+       struct callback_head            ri_task_work;
+       struct task_struct              *task;
 };
 
Quoted 12 lines

include/linux/uprobes.h struct return_instance { · line 95

Quoted 8 lines
include/linux/uprobes.h +2 -1
 struct return_instance {
@@ -86,7 +91,8 @@ struct return_instance {
        unsigned long           func;
        unsigned long           stack;          /* stack pointer */
        unsigned long           orig_ret_vaddr; /* original return address */
-       bool                    chained;        /* true, if instance is nested */
+       u8                      chained;        /* true, if instance is nested */
+       u8                      has_ref;

Why bool -> u8 switch? You don't touch chained, so why change its type? And for has_ref you interchangeably use 0 and true for the same field. Let's stick to bool as there is nothing wrong with it?

Quoted 3 lines

sizeof(_Bool) is implementation defined. It is 1 for x86_64, but there are platforms where it ends up begin 4 (some PowerPC ABIs among others. I didn't want to grow this structure for no reason.

There are tons of bools in the kernel, surprised that we (kernel makefiles) don't do anything on PowerPC to keep it consistent with the rest of the world. Oh well, it just kind of looks off when there is a mix of 0 and true used for the same field.

Quoted 30 lines
Quoted 20 lines
Quoted 4 lines
include/linux/uprobes.h +0 -0
        int                     srcu_idx;
 
        struct return_instance  *next;          /* keep as stack */

[...]

Quoted 13 lines
include/linux/uprobes.h +6 -1
@@ -1822,13 +1864,20 @@ static int dup_utask(struct task_struct
                        return -ENOMEM;
 
                *n = *o;
-               __srcu_clone_read_lock(&uretprobes_srcu, n->srcu_idx);
+               if (n->uprobe) {
+                       if (n->has_ref)
+                               get_uprobe(n->uprobe);
+                       else
+                               __srcu_clone_read_lock(&uretprobes_srcu, n->srcu_idx);
+               }
                n->next = NULL;
 
Quoted 9 lines

include/linux/uprobes.h static int dup_utask(struct task_struct · line 1880

Quoted 6 lines
include/linux/uprobes.h +2 -0
                *p = n;
                p = &n->next;
                n_utask->depth++;
        }
+       if (n_utask->return_instances)
+               mod_timer(&n_utask->ri_timer, jiffies + HZ);

let's add #define for HZ, so it's adjusted in just one place (instead of 3 as it is right now)

Quoted 1 line

Can do I suppose.

thanks!

Quoted 10 lines
Quoted 7 lines

Also, we can have up to 64 levels of uretprobe nesting, so, technically, the user can cause a delay of 64 seconds in total. Maybe let's use something smaller than a full second? After all, if the user-space function has high latency, then this refcount congestion is much less of a problem. I'd set it to something like 50-100 ms for starters.

Before you know it we'll have a sysctl :/ But sure, we can do something shorter.

:) let's hope we won't need sysctl (I don't think we will, FWIW)

#36 lore raw
In reply to Peter Zijlstra
Quoted 32 lines · Peter Zijlstra

Hi!

These patches implement the (S)RCU based proposal to optimize uprobes.

On my c^Htrusty old IVB-EP -- where each (of the 40) CPU calls 'func' in a tight loop:

perf probe -x ./uprobes test=func
perf stat -ae probe_uprobe:test -- sleep 1

perf probe -x ./uprobes test=func%return perf stat -ae probe_uprobe:test__return -- sleep 1

PRE:

4,038,804 probe_uprobe:test
2,356,275 probe_uprobe:test__return

POST:

7,216,579 probe_uprobe:test
6,744,786 probe_uprobe:test__return

(copy-paste FTW, I didn't do new numbers because the fast paths didn't change --
and quick test run shows similar numbers)

Patches also available here:

git://git.kernel.org/pub/scm/linux/kernel/git/peterz/queue.git perf/uprobes

Quoted 11 lines

Changes since last time:
- better split with intermediate inc_not_zero()
- fix UPROBE_HANDLER_REMOVE
- restored the lost rcu_assign_pointer()
- avoid lockdep for uretprobe_srcu
- add missing put_uprobe() -> srcu_read_unlock() conversion
- actually initialize return_instance::has_ref
- a few comments
- things I don't remember

Hey Peter!

Thanks for the v2, I plan to look at it more thoroughly tomorrow. But meanwhile I spent a good chunk of today to write an uprobes stress-test, so we can validate that we are not regressing anything (yes, I don't trust lockless code and people in general ;)

Anyways, if you'd like to use it, it's at [0]. All you should need to build and run it is:

  $ cd examples/c
  $ make -j$(nproc) uprobe-stress
  $ sudo ./uprobe-stress -tN -aM -mP -fR

N, M, P, R are number of threads dedicated to one of four functions of the stress test: triggering user space functions (N), attaching/detaching various random subsets of uprobes (M), mmap()ing parts of executable with uprobes (P), and forking the process and triggering uprobes for a little bit (R). The idea is to test various timings and interleavings of uprobe-related logic.

You should only need not-too-old Clang to build everything (Clang 12+ should work, I believe). But do let me know if you run into troubles.

I did run this stress test for a little while on current bpf-next/master with no issues detected (yay!).

But then I also ran it on Linux built from perf/uprobes branch (these patches), and after a few seconds I see that there is no more attachment/detachment happening. Eventually I got splats, which you can see in [1]. I used sudo ./uprobe-stress -a10 -t5 -m5 -f3 command to run it inside my QEMU image.

So there is still something off, hopefully this will help to debug and hammer out any remaining kinks. Thanks!

[0] https://github.com/libbpf/libbpf-bootstrap/commit/2f88cef90f9728ec8c7bee7bd48fdbcf197806c3
[1] https://gist.github.com/anakryiko/f761690addf7aa5f08caec95fda9ef1a

author maintainer
#37 lore raw
Quoted 2 lines · Andrii Nakryiko

You should only need not-too-old Clang to build everything (Clang 12+ should work, I believe). But do let me know if you run into troubles.

A quick look at the thing shows me it's full of BPF gunk :/

Which means, I probably also need a kernel with BPF and BTF and all that other gunk on -- which I don't have on any of my machines.

Esp. the BTF stuff is a force kill on all my builds as it completely destroys build times.

Quoted 5 lines

But then I also ran it on Linux built from perf/uprobes branch (these patches), and after a few seconds I see that there is no more attachment/detachment happening. Eventually I got splats, which you can see in [1]. I used sudo ./uprobe-stress -a10 -t5 -m5 -f3 command to run it inside my QEMU image.

OK, I'll see if I can make it work.

author maintainer
#38 lore raw
In reply to Andrii Nakryiko
Quoted 8 lines · Andrii Nakryiko

Anyways, if you'd like to use it, it's at [0]. All you should need to build and run it is:

  $ cd examples/c
  $ make -j$(nproc) uprobe-stress
  $ sudo ./uprobe-stress -tN -aM -mP -fR

[0] https://github.com/libbpf/libbpf-bootstrap/commit/2f88cef90f9728ec8c7bee7bd48fdbcf197806c3

So, I cannot clone that [0] URL I had to click around github shite for a while to figure out wtf to clone and where your commit lives, turns out it is:

	$ git clone https://github.com/libbpf/libbpf-bootstrap.git
	$ cd libbpf-bootstrap
	$ git checkout uprobe-stress

But then I do the above and I'm greeted with:

root@ivb-ep:/usr/src/libbpf-bootstrap/examples/c# make -j40
  MKDIR    .output
  MKDIR    .output/libbpf
  MKDIR    bpftool
  LIB      libbpf.a
  BPFTOOL  bpftool/bootstrap/bpftool
make[1]: *** /usr/src/libbpf-bootstrap/libbpf/src: No such file or directory.  Stop.
make: *** [Makefile:87: /usr/src/libbpf-bootstrap/examples/c/.output/libbpf.a] Error 2
make: *** Waiting for unfinished jobs....
make[1]: *** /usr/src/libbpf-bootstrap/bpftool/src: No such file or directory.  Stop.
make: *** [Makefile:95: /usr/src/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/bpftool] Error 2

Now what ?!?

BPF is ever such unusable shite :/ It's very near as bad as qemu.

#39 lore raw
Quoted 10 lines · Peter Zijlstra
Quoted 6 lines · Andrii Nakryiko

Anyways, if you'd like to use it, it's at [0]. All you should need to build and run it is:

  $ cd examples/c
  $ make -j$(nproc) uprobe-stress
  $ sudo ./uprobe-stress -tN -aM -mP -fR
Quoted 5 lines
Quoted 1 line

[0] https://github.com/libbpf/libbpf-bootstrap/commit/2f88cef90f9728ec8c7bee7bd48fdbcf197806c3

So, I cannot clone that [0] URL I had to click around github shite for a while to figure out wtf to clone and where your commit lives, turns out it is:

Sorry, my bad, it's just "normal" Github stuff, but yes, I also find it quite confusing, so should have been more explicit that this is uprobe-stress branch in libbpf/libbpf-bootstrap Github repo.

Quoted 4 lines
        $ git clone https://github.com/libbpf/libbpf-bootstrap.git
        $ cd libbpf-bootstrap
        $ git checkout uprobe-stress

Yes, sorry, as I said, I should have been more thorough in my instructions. You did a great job figuring all the above out, the next step is to make sure all the git submodules are checked out, so, in addition to the above you are just missing git submodule initialization:

$ git submodule update --init --recursive
$ cd examples/c
$ make -j%(nproc) uprobe-stress

NOTE: Don't run just make, because it will build all the examples, which have extra dependencies, and we don't want to go over that with you :) Stick to make uprobe-stress to build just an uprobe-stress binary.

The only extra thing that you might need (if you don't have that already on your build machine) is development versions of libelf and zlib, as dependencies of libbpf (that would be elfutils-libelf-devel and zlib-devel packages in Fedora, I think). libbpf-bootstrap is trying to be as self-contained and dependency free as possible (which is also why git submodules).

You were worried about BTF. This tool doesn't need BTF and it should work without it enabled in kernel config (but if you do want BTF generation to succeed, I think the only kernel build requirement for that is up-to-date pahole from dwarves package).

As for the kernel config, I don't think you need anything BPF-specific beyond:

CONFIG_BPF=y
CONFIG_BPF_EVENTS=y
CONFIG_BPF_SYSCALL=y

But just in case, we maintain a list of kernel configuration that *all of BPF selftests* require (see [0]), so worst case just append that to your config (but really, above three is probably all you need, assuming you have all the non-BPF perf/tracing/uprobe configs enabled).

[0] https://github.com/torvalds/linux/blob/master/tools/testing/selftests/bpf/config

Quoted 16 lines

But then I do the above and I'm greeted with:

root@ivb-ep:/usr/src/libbpf-bootstrap/examples/c# make -j40
  MKDIR    .output
  MKDIR    .output/libbpf
  MKDIR    bpftool
  LIB      libbpf.a
  BPFTOOL  bpftool/bootstrap/bpftool
make[1]: *** /usr/src/libbpf-bootstrap/libbpf/src: No such file or directory.  Stop.
make: *** [Makefile:87: /usr/src/libbpf-bootstrap/examples/c/.output/libbpf.a] Error 2
make: *** Waiting for unfinished jobs....
make[1]: *** /usr/src/libbpf-bootstrap/bpftool/src: No such file or directory.  Stop.
make: *** [Makefile:95: /usr/src/libbpf-bootstrap/examples/c/.output/bpftool/bootstrap/bpftool] Error 2
Quoted 4 lines

Now what ?!?

BPF is ever such unusable shite :/ It's very near as bad as qemu.

Sorry, not going into discussions about this :) I do agree initial setup is not straightforward, and this libbpf-bootstrap repo is actually an attempt to let users start quickly by doing a minimal and more-or-less straightforward BPF setup (which is why I built uprobe-stress on top of libbpf-bootstrap setup instead of BPF selftests; that would be even more painful process for you which I didn't want to force you through, see above about selftests/bpf/config).

But just keep in mind that using BPF here isn't some sort of random choice just because I tend to work mostly with BPF. BPF is the only interface to multi-uprobe attachment, it's a lightweight and low-overhead way to know whether uprobes are triggered (I do memory-mapped sharing of per-cpu counters between BPF/kernel and user space, no extra syscall overhead, blocking, or unnecessary memory ordering is added). And in general, whether you like it or not, most people would never care to use uprobes if not for BPF as an interface into that.

I'm not trying to convert you into BPF or anything like that, but this has to use BPF for end-to-end testing.

But you are close, please don't give up!

author maintainer
#40 lore raw
In reply to Andrii Nakryiko
Quoted 5 lines · Andrii Nakryiko

But then I also ran it on Linux built from perf/uprobes branch (these patches), and after a few seconds I see that there is no more attachment/detachment happening. Eventually I got splats, which you can see in [1]. I used sudo ./uprobe-stress -a10 -t5 -m5 -f3 command to run it inside my QEMU image.

So them git voodoo incantations did work and I got it built. I'm running that exact same line above (minus the sudo, because test box only has a root account I think) on real hardware.

I'm now ~100 periods in and wondering what 'eventually' means...

Also, this is a 2 socket, 10 core per socket, 2 threads per core ivybridge thing, are those parameters sufficient?

#41 lore raw
Quoted 9 lines · Peter Zijlstra
Quoted 5 lines · Andrii Nakryiko

But then I also ran it on Linux built from perf/uprobes branch (these patches), and after a few seconds I see that there is no more attachment/detachment happening. Eventually I got splats, which you can see in [1]. I used sudo ./uprobe-stress -a10 -t5 -m5 -f3 command to run it inside my QEMU image.

Quoted 5 lines

So them git voodoo incantations did work and I got it built. I'm running that exact same line above (minus the sudo, because test box only has a root account I think) on real hardware.

I'm now ~100 periods in and wondering what 'eventually' means...

So I was running in a qemu set up with 16 cores on top of bare metal's 80 core CPU (Intel(R) Xeon(R) Gold 6138 CPU @ 2.00GHz). I just tried it again, and I can reproduce it within first few periods:

WORKING HARD!..

PERIOD #1 STATS:

FUNC CALLS               919632
UPROBE HITS              706351
URETPROBE HITS           641679

ATTACHED LINKS 951
ATTACHED UPROBES 2421
ATTACHED URETPROBES 2343
MMAP CALLS 33533
FORKS CALLS 241

PERIOD #2 STATS:
FUNC CALLS 11444
UPROBE HITS 14320
URETPROBE HITS 9896

ATTACHED LINKS               26
ATTACHED UPROBES             75
ATTACHED URETPROBES          61

MMAP CALLS 39093
FORKS CALLS 14

PERIOD #3 STATS:

FUNC CALLS                  230
UPROBE HITS                 152
URETPROBE HITS              145
ATTACHED LINKS                2
ATTACHED UPROBES              2
ATTACHED URETPROBES           2

MMAP CALLS 39121
FORKS CALLS 0

PERIOD #4 STATS:

FUNC CALLS                    0
UPROBE HITS                   0
URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 39010
FORKS CALLS 0

You can see in the second period all the numbers drop and by period #4 (which is about 20 seconds in) anything but mmap()ing stops. When I said "eventually" I meant about a minute tops, however long it takes to do soft lockup detection, 23 seconds this time.

So it should be very fast.

Note that I'm running with debug kernel configuration (see [0] for full kernel config), here are debug-related settings, in case that makes a difference:

$ cat ~/linux-build/default/.config | rg -i debug | rg -v '^#'
CONFIG_X86_DEBUGCTLMSR=y
CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
CONFIG_BLK_DEBUG_FS=y
CONFIG_PNP_DEBUG_MESSAGES=y
CONFIG_AIC7XXX_DEBUG_MASK=0
CONFIG_AIC79XX_DEBUG_MASK=0
CONFIG_SCSI_MVSAS_DEBUG=y
CONFIG_DM_DEBUG=y
CONFIG_MLX4_DEBUG=y
CONFIG_USB_SERIAL_DEBUG=m
CONFIG_INFINIBAND_MTHCA_DEBUG=y
CONFIG_INFINIBAND_IPOIB_DEBUG=y
CONFIG_INFINIBAND_IPOIB_DEBUG_DATA=y
CONFIG_CIFS_DEBUG=y
CONFIG_DLM_DEBUG=y
CONFIG_DEBUG_BUGVERBOSE=y
CONFIG_DEBUG_KERNEL=y
CONFIG_DEBUG_INFO=y
CONFIG_DEBUG_INFO_DWARF4=y
CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
CONFIG_DEBUG_INFO_BTF=y
CONFIG_DEBUG_INFO_BTF_MODULES=y
CONFIG_DEBUG_FS=y
CONFIG_DEBUG_FS_ALLOW_ALL=y
CONFIG_ARCH_HAS_DEBUG_WX=y
CONFIG_HAVE_DEBUG_KMEMLEAK=y
CONFIG_ARCH_HAS_DEBUG_VM_PGTABLE=y
CONFIG_ARCH_HAS_DEBUG_VIRTUAL=y
CONFIG_SCHED_DEBUG=y
CONFIG_DEBUG_PREEMPT=y
CONFIG_LOCK_DEBUGGING_SUPPORT=y
CONFIG_DEBUG_RT_MUTEXES=y
CONFIG_DEBUG_SPINLOCK=y
CONFIG_DEBUG_MUTEXES=y
CONFIG_DEBUG_WW_MUTEX_SLOWPATH=y
CONFIG_DEBUG_RWSEMS=y
CONFIG_DEBUG_LOCK_ALLOC=y
CONFIG_DEBUG_LOCKDEP=y
CONFIG_DEBUG_ATOMIC_SLEEP=y
CONFIG_DEBUG_IRQFLAGS=y
CONFIG_X86_DEBUG_FPU=y
CONFIG_FAULT_INJECTION_DEBUG_FS=y

[0] https://gist.github.com/anakryiko/97a023a95b30fb0fe607ff743433e64b

Quoted 3 lines

Also, this is a 2 socket, 10 core per socket, 2 threads per core ivybridge thing, are those parameters sufficient?

Should be, I guess? It might be VM vs bare metal differences, though. I'll try to run this on bare metal with more production-like kernel configuration to see if I can still trigger this. Will let you know the results when I get them.

#42 lore raw
Quoted 121 lines · Andrii Nakryiko
Quoted 9 lines · Peter Zijlstra
Quoted 5 lines · Andrii Nakryiko

But then I also ran it on Linux built from perf/uprobes branch (these patches), and after a few seconds I see that there is no more attachment/detachment happening. Eventually I got splats, which you can see in [1]. I used sudo ./uprobe-stress -a10 -t5 -m5 -f3 command to run it inside my QEMU image.

Quoted 5 lines

So them git voodoo incantations did work and I got it built. I'm running that exact same line above (minus the sudo, because test box only has a root account I think) on real hardware.

I'm now ~100 periods in and wondering what 'eventually' means...

So I was running in a qemu set up with 16 cores on top of bare metal's 80 core CPU (Intel(R) Xeon(R) Gold 6138 CPU @ 2.00GHz). I just tried it again, and I can reproduce it within first few periods:

WORKING HARD!..

PERIOD #1 STATS:

FUNC CALLS               919632
UPROBE HITS              706351
URETPROBE HITS           641679

ATTACHED LINKS 951
ATTACHED UPROBES 2421
ATTACHED URETPROBES 2343
MMAP CALLS 33533
FORKS CALLS 241

PERIOD #2 STATS:
FUNC CALLS 11444
UPROBE HITS 14320
URETPROBE HITS 9896

ATTACHED LINKS               26
ATTACHED UPROBES             75
ATTACHED URETPROBES          61

MMAP CALLS 39093
FORKS CALLS 14

PERIOD #3 STATS:

FUNC CALLS                  230
UPROBE HITS                 152
URETPROBE HITS              145
ATTACHED LINKS                2
ATTACHED UPROBES              2
ATTACHED URETPROBES           2

MMAP CALLS 39121
FORKS CALLS 0

PERIOD #4 STATS:

FUNC CALLS                    0
UPROBE HITS                   0
URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 39010
FORKS CALLS 0

You can see in the second period all the numbers drop and by period #4 (which is about 20 seconds in) anything but mmap()ing stops. When I said "eventually" I meant about a minute tops, however long it takes to do soft lockup detection, 23 seconds this time.

So it should be very fast.

Note that I'm running with debug kernel configuration (see [0] for full kernel config), here are debug-related settings, in case that makes a difference:

$ cat ~/linux-build/default/.config | rg -i debug | rg -v '^#'
CONFIG_X86_DEBUGCTLMSR=y
CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
CONFIG_BLK_DEBUG_FS=y
CONFIG_PNP_DEBUG_MESSAGES=y
CONFIG_AIC7XXX_DEBUG_MASK=0
CONFIG_AIC79XX_DEBUG_MASK=0
CONFIG_SCSI_MVSAS_DEBUG=y
CONFIG_DM_DEBUG=y
CONFIG_MLX4_DEBUG=y
CONFIG_USB_SERIAL_DEBUG=m
CONFIG_INFINIBAND_MTHCA_DEBUG=y
CONFIG_INFINIBAND_IPOIB_DEBUG=y
CONFIG_INFINIBAND_IPOIB_DEBUG_DATA=y
CONFIG_CIFS_DEBUG=y
CONFIG_DLM_DEBUG=y
CONFIG_DEBUG_BUGVERBOSE=y
CONFIG_DEBUG_KERNEL=y
CONFIG_DEBUG_INFO=y
CONFIG_DEBUG_INFO_DWARF4=y
CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
CONFIG_DEBUG_INFO_BTF=y
CONFIG_DEBUG_INFO_BTF_MODULES=y
CONFIG_DEBUG_FS=y
CONFIG_DEBUG_FS_ALLOW_ALL=y
CONFIG_ARCH_HAS_DEBUG_WX=y
CONFIG_HAVE_DEBUG_KMEMLEAK=y
CONFIG_ARCH_HAS_DEBUG_VM_PGTABLE=y
CONFIG_ARCH_HAS_DEBUG_VIRTUAL=y
CONFIG_SCHED_DEBUG=y
CONFIG_DEBUG_PREEMPT=y
CONFIG_LOCK_DEBUGGING_SUPPORT=y
CONFIG_DEBUG_RT_MUTEXES=y
CONFIG_DEBUG_SPINLOCK=y
CONFIG_DEBUG_MUTEXES=y
CONFIG_DEBUG_WW_MUTEX_SLOWPATH=y
CONFIG_DEBUG_RWSEMS=y
CONFIG_DEBUG_LOCK_ALLOC=y
CONFIG_DEBUG_LOCKDEP=y
CONFIG_DEBUG_ATOMIC_SLEEP=y
CONFIG_DEBUG_IRQFLAGS=y
CONFIG_X86_DEBUG_FPU=y
CONFIG_FAULT_INJECTION_DEBUG_FS=y

[0] https://gist.github.com/anakryiko/97a023a95b30fb0fe607ff743433e64b

Quoted 1 line
Quoted 7 lines
Quoted 2 lines

Also, this is a 2 socket, 10 core per socket, 2 threads per core ivybridge thing, are those parameters sufficient?

Should be, I guess? It might be VM vs bare metal differences, though. I'll try to run this on bare metal with more production-like kernel configuration to see if I can still trigger this. Will let you know the results when I get them.

Ok, so I ran it on bare metal host with production config. I didn't really bother to specify parameters (so just one thread for everything, the default):

# ./uprobe-stress
WORKING HARD!..

PERIOD #1 STATS:
FUNC CALLS 2959843
UPROBE HITS 1001312
URETPROBE HITS 0
ATTACHED LINKS 6
ATTACHED UPROBES 28
ATTACHED URETPROBES 0
MMAP CALLS 8143
FORKS CALLS 301

PERIOD #2 STATS:
FUNC CALLS 0
UPROBE HITS 822826

URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 8006
FORKS CALLS 270

PERIOD #3 STATS:
FUNC CALLS 0
UPROBE HITS 889534

URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 8004
FORKS CALLS 288

PERIOD #4 STATS:
FUNC CALLS 0
UPROBE HITS 886506

URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 8120
FORKS CALLS 285

PERIOD #5 STATS:
FUNC CALLS 0
UPROBE HITS 804556

URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 7131
FORKS CALLS 263
^C
EXITING...

Message from [email protected] at Jul 15 11:06:33 ...
kernel:[ 2194.334618] watchdog: BUG: soft lockup - CPU#71 stuck for
48s! [uprobe-stress:69900]

It was weird on the very first period (no uretprobes, small amount of attachments). And sure enough (gmail will reformat below in the garbage, so [0] has the splat with the original formatting).

[0] https://gist.github.com/anakryiko/3e3ddcccc5ea3ca70ce90b5491485fdc

I also keep getting:

Message from [email protected] at Jul 15 11:09:41 ...
kernel:[ 2382.334088] watchdog: BUG: soft lockup - CPU#71 stuck for
223s! [uprobe-stress:69900]

so it's not just a temporary slowdown

[ 2166.893057] rcu: INFO: rcu_sched self-detected stall on CPU
[ 2166.904199] rcu:     71-....: (20999 ticks this GP)
idle=2c84/1/0x4000000000000000 softirq=30158/30158 fqs=8110
[ 2166.923810] rcu:              hardirqs   softirqs   csw/system
[ 2166.934939] rcu:      number:        0        183            0
[ 2166.946064] rcu:     cputime:       60          0        10438
==> 10549(ms)
[ 2166.959969] rcu:     (t=21065 jiffies g=369217 q=207850 ncpus=80)
[ 2166.971619] CPU: 71 PID: 69900 Comm: uprobe-stress Tainted: G S
     E      6.10.0-rc7-00071-g9423ae8ef6ff #62
[ 2166.992275] Hardware name: Quanta Tioga Pass Single Side
01-0032211004/Tioga Pass Single Side, BIOS F08_3A24 05/13/2020
[ 2167.013804] RIP: 0010:uprobe_notify_resume+0x622/0xe20
[ 2167.024064] Code: 8d 9d c0 00 00 00 48 89 df 4c 89 e6 e8 d7 f9 ff
ff 84 c0 0f 85 c6 06 00 00 48 89 5c 24 20 41 8b 6d 58 40 f6 c5 01 74
23 f3 90 <eb> f2 83 7c 24 18 00 48 8b 44 24 10 0f 8e 71 01 00 00 bf 05
00 00
[ 2167.061543] RSP: 0000:ffffc9004a49fe78 EFLAGS: 00000202
[ 2167.071973] RAX: 0000000000000000 RBX: ffff88a11d307fc0 RCX: ffff88a120752c40
[ 2167.086223] RDX: 00000000000042ec RSI: ffffc9004a49ff58 RDI: ffff88a11d307fc0
[ 2167.100472] RBP: 0000000000000003 R08: ffff88a12516e500 R09: ffff88a12516f208
[ 2167.114717] R10: 00000000004042ec R11: 000000000000000f R12: ffffc9004a49ff58
[ 2167.128967] R13: ffff88a11d307f00 R14: 00000000004042ec R15: ffff88a09042e000
[ 2167.143213] FS:  00007fd252000640(0000) GS:ffff88bfffbc0000(0000)
knlGS:0000000000000000
[ 2167.159368] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 2167.170843] CR2: 00007fd244000b60 CR3: 000000209090b001 CR4: 00000000007706f0
[ 2167.185091] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 2167.199340] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 2167.213586] PKRU: 55555554
[ 2167.218994] Call Trace:
[ 2167.223883]  <IRQ>
[ 2167.227905]  ? rcu_dump_cpu_stacks+0x77/0xd0
[ 2167.236433]  ? print_cpu_stall+0x150/0x2a0
[ 2167.244615]  ? rcu_sched_clock_irq+0x319/0x490
[ 2167.253487]  ? update_process_times+0x71/0xa0
[ 2167.262191]  ? tick_nohz_handler+0xc0/0x100
[ 2167.270544]  ? tick_setup_sched_timer+0x170/0x170
[ 2167.279937]  ? __hrtimer_run_queues+0xe3/0x250
[ 2167.288815]  ? hrtimer_interrupt+0xf0/0x390
[ 2167.297168]  ? __sysvec_apic_timer_interrupt+0x47/0x110
[ 2167.307602]  ? sysvec_apic_timer_interrupt+0x68/0x80
[ 2167.317519]  </IRQ>
[ 2167.321710]  <TASK>
[ 2167.325905]  ? asm_sysvec_apic_timer_interrupt+0x16/0x20
[ 2167.336517]  ? uprobe_notify_resume+0x622/0xe20
[ 2167.345565]  ? uprobe_notify_resume+0x609/0xe20
[ 2167.354612]  ? __se_sys_futex+0xf3/0x180
[ 2167.362445]  ? arch_uprobe_exception_notify+0x29/0x40
[ 2167.372533]  ? notify_die+0x51/0xb0
[ 2167.379503]  irqentry_exit_to_user_mode+0x7f/0xd0
[ 2167.388896]  asm_exc_int3+0x35/0x40
[ 2167.395862] RIP: 0033:0x4042ec
[ 2167.401966] Code: fc 8b 45 fc 89 c7 e8 6f 07 00 00 83 c0 01 c9 c3
cc 48 89 e5 48 83 ec 10 89 7d fc 8b 45 fc 89 c7 e8 55 07 00 00 83 c0
01 c9 c3 <cc> 48 89 e5 48 83 ec 10 89 7d fc 8b 45 fc 89 c7 e8 3b 07 00
00 83
[ 2167.439439] RSP: 002b:00007fd251fff8a8 EFLAGS: 00000206
[ 2167.449874] RAX: 00000000004042ec RBX: 00007fd252000640 RCX: 000000000000001c
[ 2167.464122] RDX: 0000000000000033 RSI: 0000000000000064 RDI: 0000000000000033
[ 2167.478368] RBP: 00007fd251fff8d0 R08: 00007fd2523fa234 R09: 00007fd2523fa280
[ 2167.492617] R10: 0000000000000000 R11: 0000000000000246 R12: 00007fd252000640
[ 2167.506866] R13: 0000000000000016 R14: 00007fd252289930 R15: 0000000000000000
[ 2167.521117]  </TASK>
#43 lore raw
Quoted 266 lines · Andrii Nakryiko
Quoted 121 lines · Andrii Nakryiko
Quoted 9 lines · Peter Zijlstra
Quoted 5 lines · Andrii Nakryiko

But then I also ran it on Linux built from perf/uprobes branch (these patches), and after a few seconds I see that there is no more attachment/detachment happening. Eventually I got splats, which you can see in [1]. I used sudo ./uprobe-stress -a10 -t5 -m5 -f3 command to run it inside my QEMU image.

Quoted 5 lines

So them git voodoo incantations did work and I got it built. I'm running that exact same line above (minus the sudo, because test box only has a root account I think) on real hardware.

I'm now ~100 periods in and wondering what 'eventually' means...

So I was running in a qemu set up with 16 cores on top of bare metal's 80 core CPU (Intel(R) Xeon(R) Gold 6138 CPU @ 2.00GHz). I just tried it again, and I can reproduce it within first few periods:

WORKING HARD!..

PERIOD #1 STATS:

FUNC CALLS               919632
UPROBE HITS              706351
URETPROBE HITS           641679

ATTACHED LINKS 951
ATTACHED UPROBES 2421
ATTACHED URETPROBES 2343
MMAP CALLS 33533
FORKS CALLS 241

PERIOD #2 STATS:
FUNC CALLS 11444
UPROBE HITS 14320
URETPROBE HITS 9896

ATTACHED LINKS               26
ATTACHED UPROBES             75
ATTACHED URETPROBES          61

MMAP CALLS 39093
FORKS CALLS 14

PERIOD #3 STATS:

FUNC CALLS                  230
UPROBE HITS                 152
URETPROBE HITS              145
ATTACHED LINKS                2
ATTACHED UPROBES              2
ATTACHED URETPROBES           2

MMAP CALLS 39121
FORKS CALLS 0

PERIOD #4 STATS:

FUNC CALLS                    0
UPROBE HITS                   0
URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 39010
FORKS CALLS 0

You can see in the second period all the numbers drop and by period #4 (which is about 20 seconds in) anything but mmap()ing stops. When I said "eventually" I meant about a minute tops, however long it takes to do soft lockup detection, 23 seconds this time.

So it should be very fast.

Note that I'm running with debug kernel configuration (see [0] for full kernel config), here are debug-related settings, in case that makes a difference:

$ cat ~/linux-build/default/.config | rg -i debug | rg -v '^#'
CONFIG_X86_DEBUGCTLMSR=y
CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
CONFIG_BLK_DEBUG_FS=y
CONFIG_PNP_DEBUG_MESSAGES=y
CONFIG_AIC7XXX_DEBUG_MASK=0
CONFIG_AIC79XX_DEBUG_MASK=0
CONFIG_SCSI_MVSAS_DEBUG=y
CONFIG_DM_DEBUG=y
CONFIG_MLX4_DEBUG=y
CONFIG_USB_SERIAL_DEBUG=m
CONFIG_INFINIBAND_MTHCA_DEBUG=y
CONFIG_INFINIBAND_IPOIB_DEBUG=y
CONFIG_INFINIBAND_IPOIB_DEBUG_DATA=y
CONFIG_CIFS_DEBUG=y
CONFIG_DLM_DEBUG=y
CONFIG_DEBUG_BUGVERBOSE=y
CONFIG_DEBUG_KERNEL=y
CONFIG_DEBUG_INFO=y
CONFIG_DEBUG_INFO_DWARF4=y
CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
CONFIG_DEBUG_INFO_BTF=y
CONFIG_DEBUG_INFO_BTF_MODULES=y
CONFIG_DEBUG_FS=y
CONFIG_DEBUG_FS_ALLOW_ALL=y
CONFIG_ARCH_HAS_DEBUG_WX=y
CONFIG_HAVE_DEBUG_KMEMLEAK=y
CONFIG_ARCH_HAS_DEBUG_VM_PGTABLE=y
CONFIG_ARCH_HAS_DEBUG_VIRTUAL=y
CONFIG_SCHED_DEBUG=y
CONFIG_DEBUG_PREEMPT=y
CONFIG_LOCK_DEBUGGING_SUPPORT=y
CONFIG_DEBUG_RT_MUTEXES=y
CONFIG_DEBUG_SPINLOCK=y
CONFIG_DEBUG_MUTEXES=y
CONFIG_DEBUG_WW_MUTEX_SLOWPATH=y
CONFIG_DEBUG_RWSEMS=y
CONFIG_DEBUG_LOCK_ALLOC=y
CONFIG_DEBUG_LOCKDEP=y
CONFIG_DEBUG_ATOMIC_SLEEP=y
CONFIG_DEBUG_IRQFLAGS=y
CONFIG_X86_DEBUG_FPU=y
CONFIG_FAULT_INJECTION_DEBUG_FS=y

[0] https://gist.github.com/anakryiko/97a023a95b30fb0fe607ff743433e64b

Quoted 1 line
Quoted 7 lines
Quoted 2 lines

Also, this is a 2 socket, 10 core per socket, 2 threads per core ivybridge thing, are those parameters sufficient?

Should be, I guess? It might be VM vs bare metal differences, though. I'll try to run this on bare metal with more production-like kernel configuration to see if I can still trigger this. Will let you know the results when I get them.

Ok, so I ran it on bare metal host with production config. I didn't really bother to specify parameters (so just one thread for everything, the default):

# ./uprobe-stress
WORKING HARD!..

PERIOD #1 STATS:
FUNC CALLS 2959843
UPROBE HITS 1001312
URETPROBE HITS 0
ATTACHED LINKS 6
ATTACHED UPROBES 28
ATTACHED URETPROBES 0
MMAP CALLS 8143
FORKS CALLS 301

PERIOD #2 STATS:
FUNC CALLS 0
UPROBE HITS 822826

URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 8006
FORKS CALLS 270

PERIOD #3 STATS:
FUNC CALLS 0
UPROBE HITS 889534

URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 8004
FORKS CALLS 288

PERIOD #4 STATS:
FUNC CALLS 0
UPROBE HITS 886506

URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 8120
FORKS CALLS 285

PERIOD #5 STATS:
FUNC CALLS 0
UPROBE HITS 804556

URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 7131
FORKS CALLS 263
^C
EXITING...

Message from [email protected] at Jul 15 11:06:33 ...
kernel:[ 2194.334618] watchdog: BUG: soft lockup - CPU#71 stuck for
48s! [uprobe-stress:69900]

It was weird on the very first period (no uretprobes, small amount of attachments). And sure enough (gmail will reformat below in the garbage, so [0] has the splat with the original formatting).

[0] https://gist.github.com/anakryiko/3e3ddcccc5ea3ca70ce90b5491485fdc

I also keep getting:

Message from [email protected] at Jul 15 11:09:41 ...
kernel:[ 2382.334088] watchdog: BUG: soft lockup - CPU#71 stuck for
223s! [uprobe-stress:69900]

so it's not just a temporary slowdown

[ 2166.893057] rcu: INFO: rcu_sched self-detected stall on CPU
[ 2166.904199] rcu:     71-....: (20999 ticks this GP)
idle=2c84/1/0x4000000000000000 softirq=30158/30158 fqs=8110
[ 2166.923810] rcu:              hardirqs   softirqs   csw/system
[ 2166.934939] rcu:      number:        0        183            0
[ 2166.946064] rcu:     cputime:       60          0        10438
==> 10549(ms)
[ 2166.959969] rcu:     (t=21065 jiffies g=369217 q=207850 ncpus=80)
[ 2166.971619] CPU: 71 PID: 69900 Comm: uprobe-stress Tainted: G S
     E      6.10.0-rc7-00071-g9423ae8ef6ff #62
[ 2166.992275] Hardware name: Quanta Tioga Pass Single Side
01-0032211004/Tioga Pass Single Side, BIOS F08_3A24 05/13/2020
[ 2167.013804] RIP: 0010:uprobe_notify_resume+0x622/0xe20
[ 2167.024064] Code: 8d 9d c0 00 00 00 48 89 df 4c 89 e6 e8 d7 f9 ff
ff 84 c0 0f 85 c6 06 00 00 48 89 5c 24 20 41 8b 6d 58 40 f6 c5 01 74
23 f3 90 <eb> f2 83 7c 24 18 00 48 8b 44 24 10 0f 8e 71 01 00 00 bf 05
00 00
[ 2167.061543] RSP: 0000:ffffc9004a49fe78 EFLAGS: 00000202
[ 2167.071973] RAX: 0000000000000000 RBX: ffff88a11d307fc0 RCX: ffff88a120752c40
[ 2167.086223] RDX: 00000000000042ec RSI: ffffc9004a49ff58 RDI: ffff88a11d307fc0
[ 2167.100472] RBP: 0000000000000003 R08: ffff88a12516e500 R09: ffff88a12516f208
[ 2167.114717] R10: 00000000004042ec R11: 000000000000000f R12: ffffc9004a49ff58
[ 2167.128967] R13: ffff88a11d307f00 R14: 00000000004042ec R15: ffff88a09042e000
[ 2167.143213] FS:  00007fd252000640(0000) GS:ffff88bfffbc0000(0000)
knlGS:0000000000000000
[ 2167.159368] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 2167.170843] CR2: 00007fd244000b60 CR3: 000000209090b001 CR4: 00000000007706f0
[ 2167.185091] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 2167.199340] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 2167.213586] PKRU: 55555554
[ 2167.218994] Call Trace:
[ 2167.223883]  <IRQ>
[ 2167.227905]  ? rcu_dump_cpu_stacks+0x77/0xd0
[ 2167.236433]  ? print_cpu_stall+0x150/0x2a0
[ 2167.244615]  ? rcu_sched_clock_irq+0x319/0x490
[ 2167.253487]  ? update_process_times+0x71/0xa0
[ 2167.262191]  ? tick_nohz_handler+0xc0/0x100
[ 2167.270544]  ? tick_setup_sched_timer+0x170/0x170
[ 2167.279937]  ? __hrtimer_run_queues+0xe3/0x250
[ 2167.288815]  ? hrtimer_interrupt+0xf0/0x390
[ 2167.297168]  ? __sysvec_apic_timer_interrupt+0x47/0x110
[ 2167.307602]  ? sysvec_apic_timer_interrupt+0x68/0x80
[ 2167.317519]  </IRQ>
[ 2167.321710]  <TASK>
[ 2167.325905]  ? asm_sysvec_apic_timer_interrupt+0x16/0x20
[ 2167.336517]  ? uprobe_notify_resume+0x622/0xe20
[ 2167.345565]  ? uprobe_notify_resume+0x609/0xe20
[ 2167.354612]  ? __se_sys_futex+0xf3/0x180
[ 2167.362445]  ? arch_uprobe_exception_notify+0x29/0x40
[ 2167.372533]  ? notify_die+0x51/0xb0
[ 2167.379503]  irqentry_exit_to_user_mode+0x7f/0xd0
[ 2167.388896]  asm_exc_int3+0x35/0x40
[ 2167.395862] RIP: 0033:0x4042ec
[ 2167.401966] Code: fc 8b 45 fc 89 c7 e8 6f 07 00 00 83 c0 01 c9 c3

cc 48 89 e5 48 83 ec 10 89 7d fc 8b 45 fc 89 c7 e8 55 07 00 00 83 c0 01 c9 c3 <cc> 48 89 e5 48 83 ec 10 89 7d fc 8b 45 fc 89 c7 e8 3b 07 00

Quoted 8 lines

00 83

[ 2167.439439] RSP: 002b:00007fd251fff8a8 EFLAGS: 00000206
[ 2167.449874] RAX: 00000000004042ec RBX: 00007fd252000640 RCX: 000000000000001c
[ 2167.464122] RDX: 0000000000000033 RSI: 0000000000000064 RDI: 0000000000000033
[ 2167.478368] RBP: 00007fd251fff8d0 R08: 00007fd2523fa234 R09: 00007fd2523fa280
[ 2167.492617] R10: 0000000000000000 R11: 0000000000000246 R12: 00007fd252000640
[ 2167.506866] R13: 0000000000000016 R14: 00007fd252289930 R15: 0000000000000000
[ 2167.521117]  </TASK>

Peter,

Did you manage to reproduce this?

#44 lore raw
Quoted 278 lines · Andrii Nakryiko
Quoted 266 lines · Andrii Nakryiko
Quoted 121 lines · Andrii Nakryiko
Quoted 9 lines · Peter Zijlstra
Quoted 5 lines · Andrii Nakryiko

But then I also ran it on Linux built from perf/uprobes branch (these patches), and after a few seconds I see that there is no more attachment/detachment happening. Eventually I got splats, which you can see in [1]. I used sudo ./uprobe-stress -a10 -t5 -m5 -f3 command to run it inside my QEMU image.

Quoted 5 lines

So them git voodoo incantations did work and I got it built. I'm running that exact same line above (minus the sudo, because test box only has a root account I think) on real hardware.

I'm now ~100 periods in and wondering what 'eventually' means...

So I was running in a qemu set up with 16 cores on top of bare metal's 80 core CPU (Intel(R) Xeon(R) Gold 6138 CPU @ 2.00GHz). I just tried it again, and I can reproduce it within first few periods:

WORKING HARD!..

PERIOD #1 STATS:

FUNC CALLS               919632
UPROBE HITS              706351
URETPROBE HITS           641679

ATTACHED LINKS 951
ATTACHED UPROBES 2421
ATTACHED URETPROBES 2343
MMAP CALLS 33533
FORKS CALLS 241

PERIOD #2 STATS:
FUNC CALLS 11444
UPROBE HITS 14320
URETPROBE HITS 9896

ATTACHED LINKS               26
ATTACHED UPROBES             75
ATTACHED URETPROBES          61

MMAP CALLS 39093
FORKS CALLS 14

PERIOD #3 STATS:

FUNC CALLS                  230
UPROBE HITS                 152
URETPROBE HITS              145
ATTACHED LINKS                2
ATTACHED UPROBES              2
ATTACHED URETPROBES           2

MMAP CALLS 39121
FORKS CALLS 0

PERIOD #4 STATS:

FUNC CALLS                    0
UPROBE HITS                   0
URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 39010
FORKS CALLS 0

You can see in the second period all the numbers drop and by period #4 (which is about 20 seconds in) anything but mmap()ing stops. When I said "eventually" I meant about a minute tops, however long it takes to do soft lockup detection, 23 seconds this time.

So it should be very fast.

Note that I'm running with debug kernel configuration (see [0] for full kernel config), here are debug-related settings, in case that makes a difference:

$ cat ~/linux-build/default/.config | rg -i debug | rg -v '^#'
CONFIG_X86_DEBUGCTLMSR=y
CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
CONFIG_BLK_DEBUG_FS=y
CONFIG_PNP_DEBUG_MESSAGES=y
CONFIG_AIC7XXX_DEBUG_MASK=0
CONFIG_AIC79XX_DEBUG_MASK=0
CONFIG_SCSI_MVSAS_DEBUG=y
CONFIG_DM_DEBUG=y
CONFIG_MLX4_DEBUG=y
CONFIG_USB_SERIAL_DEBUG=m
CONFIG_INFINIBAND_MTHCA_DEBUG=y
CONFIG_INFINIBAND_IPOIB_DEBUG=y
CONFIG_INFINIBAND_IPOIB_DEBUG_DATA=y
CONFIG_CIFS_DEBUG=y
CONFIG_DLM_DEBUG=y
CONFIG_DEBUG_BUGVERBOSE=y
CONFIG_DEBUG_KERNEL=y
CONFIG_DEBUG_INFO=y
CONFIG_DEBUG_INFO_DWARF4=y
CONFIG_DEBUG_INFO_COMPRESSED_NONE=y
CONFIG_DEBUG_INFO_BTF=y
CONFIG_DEBUG_INFO_BTF_MODULES=y
CONFIG_DEBUG_FS=y
CONFIG_DEBUG_FS_ALLOW_ALL=y
CONFIG_ARCH_HAS_DEBUG_WX=y
CONFIG_HAVE_DEBUG_KMEMLEAK=y
CONFIG_ARCH_HAS_DEBUG_VM_PGTABLE=y
CONFIG_ARCH_HAS_DEBUG_VIRTUAL=y
CONFIG_SCHED_DEBUG=y
CONFIG_DEBUG_PREEMPT=y
CONFIG_LOCK_DEBUGGING_SUPPORT=y
CONFIG_DEBUG_RT_MUTEXES=y
CONFIG_DEBUG_SPINLOCK=y
CONFIG_DEBUG_MUTEXES=y
CONFIG_DEBUG_WW_MUTEX_SLOWPATH=y
CONFIG_DEBUG_RWSEMS=y
CONFIG_DEBUG_LOCK_ALLOC=y
CONFIG_DEBUG_LOCKDEP=y
CONFIG_DEBUG_ATOMIC_SLEEP=y
CONFIG_DEBUG_IRQFLAGS=y
CONFIG_X86_DEBUG_FPU=y
CONFIG_FAULT_INJECTION_DEBUG_FS=y

[0] https://gist.github.com/anakryiko/97a023a95b30fb0fe607ff743433e64b

Quoted 1 line
Quoted 7 lines
Quoted 2 lines

Also, this is a 2 socket, 10 core per socket, 2 threads per core ivybridge thing, are those parameters sufficient?

Should be, I guess? It might be VM vs bare metal differences, though. I'll try to run this on bare metal with more production-like kernel configuration to see if I can still trigger this. Will let you know the results when I get them.

Ok, so I ran it on bare metal host with production config. I didn't really bother to specify parameters (so just one thread for everything, the default):

# ./uprobe-stress
WORKING HARD!..

PERIOD #1 STATS:
FUNC CALLS 2959843
UPROBE HITS 1001312
URETPROBE HITS 0
ATTACHED LINKS 6
ATTACHED UPROBES 28
ATTACHED URETPROBES 0
MMAP CALLS 8143
FORKS CALLS 301

PERIOD #2 STATS:
FUNC CALLS 0
UPROBE HITS 822826

URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 8006
FORKS CALLS 270

PERIOD #3 STATS:
FUNC CALLS 0
UPROBE HITS 889534

URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 8004
FORKS CALLS 288

PERIOD #4 STATS:
FUNC CALLS 0
UPROBE HITS 886506

URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 8120
FORKS CALLS 285

PERIOD #5 STATS:
FUNC CALLS 0
UPROBE HITS 804556

URETPROBE HITS                0
ATTACHED LINKS                0
ATTACHED UPROBES              0
ATTACHED URETPROBES           0

MMAP CALLS 7131
FORKS CALLS 263
^C
EXITING...

Message from [email protected] at Jul 15 11:06:33 ...
kernel:[ 2194.334618] watchdog: BUG: soft lockup - CPU#71 stuck for
48s! [uprobe-stress:69900]

It was weird on the very first period (no uretprobes, small amount of attachments). And sure enough (gmail will reformat below in the garbage, so [0] has the splat with the original formatting).

[0] https://gist.github.com/anakryiko/3e3ddcccc5ea3ca70ce90b5491485fdc

I also keep getting:

Message from [email protected] at Jul 15 11:09:41 ...
kernel:[ 2382.334088] watchdog: BUG: soft lockup - CPU#71 stuck for
223s! [uprobe-stress:69900]

so it's not just a temporary slowdown

[ 2166.893057] rcu: INFO: rcu_sched self-detected stall on CPU
[ 2166.904199] rcu:     71-....: (20999 ticks this GP)
idle=2c84/1/0x4000000000000000 softirq=30158/30158 fqs=8110
[ 2166.923810] rcu:              hardirqs   softirqs   csw/system
[ 2166.934939] rcu:      number:        0        183            0
[ 2166.946064] rcu:     cputime:       60          0        10438
==> 10549(ms)
[ 2166.959969] rcu:     (t=21065 jiffies g=369217 q=207850 ncpus=80)
[ 2166.971619] CPU: 71 PID: 69900 Comm: uprobe-stress Tainted: G S
     E      6.10.0-rc7-00071-g9423ae8ef6ff #62
[ 2166.992275] Hardware name: Quanta Tioga Pass Single Side
01-0032211004/Tioga Pass Single Side, BIOS F08_3A24 05/13/2020
[ 2167.013804] RIP: 0010:uprobe_notify_resume+0x622/0xe20
[ 2167.024064] Code: 8d 9d c0 00 00 00 48 89 df 4c 89 e6 e8 d7 f9 ff
ff 84 c0 0f 85 c6 06 00 00 48 89 5c 24 20 41 8b 6d 58 40 f6 c5 01 74
23 f3 90 <eb> f2 83 7c 24 18 00 48 8b 44 24 10 0f 8e 71 01 00 00 bf 05
00 00
[ 2167.061543] RSP: 0000:ffffc9004a49fe78 EFLAGS: 00000202
[ 2167.071973] RAX: 0000000000000000 RBX: ffff88a11d307fc0 RCX: ffff88a120752c40
[ 2167.086223] RDX: 00000000000042ec RSI: ffffc9004a49ff58 RDI: ffff88a11d307fc0
[ 2167.100472] RBP: 0000000000000003 R08: ffff88a12516e500 R09: ffff88a12516f208
[ 2167.114717] R10: 00000000004042ec R11: 000000000000000f R12: ffffc9004a49ff58
[ 2167.128967] R13: ffff88a11d307f00 R14: 00000000004042ec R15: ffff88a09042e000
[ 2167.143213] FS:  00007fd252000640(0000) GS:ffff88bfffbc0000(0000)
knlGS:0000000000000000
[ 2167.159368] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 2167.170843] CR2: 00007fd244000b60 CR3: 000000209090b001 CR4: 00000000007706f0
[ 2167.185091] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 2167.199340] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 2167.213586] PKRU: 55555554
[ 2167.218994] Call Trace:
[ 2167.223883]  <IRQ>
[ 2167.227905]  ? rcu_dump_cpu_stacks+0x77/0xd0
[ 2167.236433]  ? print_cpu_stall+0x150/0x2a0
[ 2167.244615]  ? rcu_sched_clock_irq+0x319/0x490
[ 2167.253487]  ? update_process_times+0x71/0xa0
[ 2167.262191]  ? tick_nohz_handler+0xc0/0x100
[ 2167.270544]  ? tick_setup_sched_timer+0x170/0x170
[ 2167.279937]  ? __hrtimer_run_queues+0xe3/0x250
[ 2167.288815]  ? hrtimer_interrupt+0xf0/0x390
[ 2167.297168]  ? __sysvec_apic_timer_interrupt+0x47/0x110
[ 2167.307602]  ? sysvec_apic_timer_interrupt+0x68/0x80
[ 2167.317519]  </IRQ>
[ 2167.321710]  <TASK>
[ 2167.325905]  ? asm_sysvec_apic_timer_interrupt+0x16/0x20
[ 2167.336517]  ? uprobe_notify_resume+0x622/0xe20
[ 2167.345565]  ? uprobe_notify_resume+0x609/0xe20
[ 2167.354612]  ? __se_sys_futex+0xf3/0x180
[ 2167.362445]  ? arch_uprobe_exception_notify+0x29/0x40
[ 2167.372533]  ? notify_die+0x51/0xb0
[ 2167.379503]  irqentry_exit_to_user_mode+0x7f/0xd0
[ 2167.388896]  asm_exc_int3+0x35/0x40
[ 2167.395862] RIP: 0033:0x4042ec
[ 2167.401966] Code: fc 8b 45 fc 89 c7 e8 6f 07 00 00 83 c0 01 c9 c3

cc 48 89 e5 48 83 ec 10 89 7d fc 8b 45 fc 89 c7 e8 55 07 00 00 83 c0 01 c9 c3 <cc> 48 89 e5 48 83 ec 10 89 7d fc 8b 45 fc 89 c7 e8 3b 07 00

Quoted 8 lines

00 83

[ 2167.439439] RSP: 002b:00007fd251fff8a8 EFLAGS: 00000206
[ 2167.449874] RAX: 00000000004042ec RBX: 00007fd252000640 RCX: 000000000000001c
[ 2167.464122] RDX: 0000000000000033 RSI: 0000000000000064 RDI: 0000000000000033
[ 2167.478368] RBP: 00007fd251fff8d0 R08: 00007fd2523fa234 R09: 00007fd2523fa280
[ 2167.492617] R10: 0000000000000000 R11: 0000000000000246 R12: 00007fd252000640
[ 2167.506866] R13: 0000000000000016 R14: 00007fd252289930 R15: 0000000000000000
[ 2167.521117]  </TASK>
Quoted 3 lines

Peter,

Did you manage to reproduce this?

Ping.