uprobes: kill xol_area->slot_count
Subsystems (3)
Oleg Nesterov
maintainercover uprobes: kill xol_area->slot_count
Hello,
On top of
[PATCH 0/7] uprobes: deuglify xol_get_insn_slot/xol_free_insn_slot paths https://lore.kernel.org/all/[email protected]/
To me this change looks like a simplification, but perhaps it can also be considered as optimization.
Yes, in the contended case xol_get_slot_nr() will be called twice, the second time after prepare_to_wait_event(), but I don't think it can hurt in the slow path when the caller is likely going to sleep.
Oleg.
---
kernel/events/uprobes.c | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-)
Oleg Nesterov
maintainerPATCH 1/2 uprobes: kill xol_area->slot_count
Add the new helper, xol_get_slot_nr() which doesfind_first_zero_bit() + test_and_set_bit().
xol_take_insn_slot() can wait for the "xol_get_slot_nr() < UINSNS_PER_PAGE" event instead of "area->slot_count < UINSNS_PER_PAGE".
So we can kill area->slot_count and avoid atomic_inc() + atomic_dec(), this simplifies the code and can slightly improve the performance.
--- kernel/events/uprobes.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-)
kernel/events/uprobes.c +15 -14
@@ -100,7 +100,6 @@ static LIST_HEAD(delayed_uprobe_list);*/ struct xol_area {wait_queue_head_t wq;/* if all slots are busy */ -atomic_t slot_count;/* number of in-use slots */ unsigned long *bitmap;/* 0 = free slot */ struct page *page;
@@ -1559,7 +1558,6 @@ static struct xol_area *__create_xol_area(unsigned long vaddr)init_waitqueue_head (&area->wq);/* Reserve the 1st slot for get_trampoline_vaddr() */ set_bit (0 , area->bitmap); -atomic_set (&area->slot_count,1 ); insns =arch_uprobe_trampoline (&insns_size);arch_uprobe_copy_ixol (area->page,0 , insns, insns_size);
@@ -1632,24 +1630,28 @@ void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm) } } +static unsigned long xol_get_slot_nr (struct xol_area *area) +{ +unsigned long slot_nr; + + slot_nr =find_first_zero_bit (area->bitmap,UINSNS_PER_PAGE ); +if (slot_nr <UINSNS_PER_PAGE ) { +if (!test_and_set_bit (slot_nr, area->bitmap)) +return slot_nr; + } + +return UINSNS_PER_PAGE ; +} +/* * - search for a free slot. */ static unsigned long xol_take_insn_slot (struct xol_area *area) { -unsigned int slot_nr; +unsigned long slot_nr; -for (;;) { - slot_nr =find_first_zero_bit (area->bitmap,UINSNS_PER_PAGE ); -if (slot_nr <UINSNS_PER_PAGE ) { -if (!test_and_set_bit (slot_nr, area->bitmap)) -break ; -continue ; - } -wait_event (area->wq, (atomic_read (&area->slot_count) <UINSNS_PER_PAGE )); - } +wait_event (area->wq, (slot_nr =xol_get_slot_nr (area)) <UINSNS_PER_PAGE ); -atomic_inc (&area->slot_count);return area->vaddr + slot_nr *UPROBE_XOL_SLOT_BYTES ; }
@@ -1685,7 +1687,6 @@ static void xol_free_insn_slot(struct uprobe_task *utask) slot_nr = offset /UPROBE_XOL_SLOT_BYTES ;clear_bit (slot_nr, area->bitmap); -atomic_dec (&area->slot_count);smp_mb__after_atomic ();/* pairs with prepare_to_wait() */ if (waitqueue_active (&area->wq))wake_up (&area->wq);
Signature · 1 line
2.25.1.362.g51ebf55
The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 7a166094bd2b1c084fd215747f9cd05a853d66c9 Gitweb: https://git.kernel.org/tip/7a166094bd2b1c084fd215747f9cd05a853d66c9 Author: Oleg Nesterov <[email protected]> AuthorDate: Tue, 01 Oct 2024 16:24:59 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 07 Oct 2024 09:28:45 +02:00
uprobes: kill xol_area->slot_count
Add the new helper, xol_get_slot_nr() which doesfind_first_zero_bit() + test_and_set_bit().
xol_take_insn_slot() can wait for the "xol_get_slot_nr() < UINSNS_PER_PAGE" event instead of "area->slot_count < UINSNS_PER_PAGE".
So we can kill area->slot_count and avoid atomic_inc() + atomic_dec(), this simplifies the code and can slightly improve the performance.
--- kernel/events/uprobes.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-)
kernel/events/uprobes.c +15 -14
@@ -99,7 +99,6 @@ static LIST_HEAD(delayed_uprobe_list);*/ struct xol_area {wait_queue_head_t wq;/* if all slots are busy */ -atomic_t slot_count;/* number of in-use slots */ unsigned long *bitmap;/* 0 = free slot */ struct page *page;
@@ -1556,7 +1555,6 @@ static struct xol_area *__create_xol_area(unsigned long vaddr)init_waitqueue_head (&area->wq);/* Reserve the 1st slot for get_trampoline_vaddr() */ set_bit (0 , area->bitmap); -atomic_set (&area->slot_count,1 ); insns =arch_uprobe_trampoline (&insns_size);arch_uprobe_copy_ixol (area->page,0 , insns, insns_size);
@@ -1629,24 +1627,28 @@ void uprobe_dup_mmap(struct mm_struct *oldmm, struct mm_struct *newmm) } } +static unsigned long xol_get_slot_nr (struct xol_area *area) +{ +unsigned long slot_nr; + + slot_nr =find_first_zero_bit (area->bitmap,UINSNS_PER_PAGE ); +if (slot_nr <UINSNS_PER_PAGE ) { +if (!test_and_set_bit (slot_nr, area->bitmap)) +return slot_nr; + } + +return UINSNS_PER_PAGE ; +} +/* * - search for a free slot. */ static unsigned long xol_take_insn_slot (struct xol_area *area) { -unsigned int slot_nr; +unsigned long slot_nr; -for (;;) { - slot_nr =find_first_zero_bit (area->bitmap,UINSNS_PER_PAGE ); -if (slot_nr <UINSNS_PER_PAGE ) { -if (!test_and_set_bit (slot_nr, area->bitmap)) -break ; -continue ; - } -wait_event (area->wq, (atomic_read (&area->slot_count) <UINSNS_PER_PAGE )); - } +wait_event (area->wq, (slot_nr =xol_get_slot_nr (area)) <UINSNS_PER_PAGE ); -atomic_inc (&area->slot_count);return area->vaddr + slot_nr *UPROBE_XOL_SLOT_BYTES ; }
@@ -1682,7 +1684,6 @@ static void xol_free_insn_slot(struct uprobe_task *utask) slot_nr = offset /UPROBE_XOL_SLOT_BYTES ;clear_bit (slot_nr, area->bitmap); -atomic_dec (&area->slot_count);smp_mb__after_atomic ();/* pairs with prepare_to_wait() */ if (waitqueue_active (&area->wq))wake_up (&area->wq);
Oleg Nesterov
maintainerPATCH 2/2 uprobes: fold xol_take_insn_slot() into xol_get_insn_slot()
In reply to Oleg NesterovAfter the previous change xol_take_insn_slot() becomes trivial, kill it.
--- kernel/events/uprobes.c | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-)
kernel/events/uprobes.c +4 -13
@@ -1643,29 +1643,20 @@ static unsigned long xol_get_slot_nr(struct xol_area *area)return UINSNS_PER_PAGE ; } -/* -* - search for a free slot. -*/ -static unsigned long xol_take_insn_slot (struct xol_area *area) -{ -unsigned long slot_nr; - -wait_event (area->wq, (slot_nr =xol_get_slot_nr (area)) <UINSNS_PER_PAGE ); - -return area->vaddr + slot_nr *UPROBE_XOL_SLOT_BYTES ; -} -/* * xol_get_insn_slot - allocate a slot for xol. */ static bool xol_get_insn_slot (struct uprobe *uprobe,struct uprobe_task *utask) {struct xol_area *area =get_xol_area (); +unsigned long slot_nr;if (!area)return false; - utask->xol_vaddr =xol_take_insn_slot (area); +wait_event (area->wq, (slot_nr =xol_get_slot_nr (area)) <UINSNS_PER_PAGE ); + + utask->xol_vaddr = area->vaddr + slot_nr *UPROBE_XOL_SLOT_BYTES ;arch_uprobe_copy_ixol (area->page, utask->xol_vaddr, &uprobe->arch.ixol,sizeof (uprobe->arch.ixol));return true;
Signature · 1 line
2.25.1.362.g51ebf55
[tip: perf/core] uprobes: fold xol_take_insn_slot() into xol_get_insn_slot()
The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 6c74ca7aa81a23c613b8ca52bfe0a4b3734dd287 Gitweb: https://git.kernel.org/tip/6c74ca7aa81a23c613b8ca52bfe0a4b3734dd287 Author: Oleg Nesterov <[email protected]> AuthorDate: Tue, 01 Oct 2024 16:25:03 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 07 Oct 2024 09:28:45 +02:00
uprobes: fold xol_take_insn_slot() into xol_get_insn_slot()
After the previous change xol_take_insn_slot() becomes trivial, kill it.
--- kernel/events/uprobes.c | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-)
kernel/events/uprobes.c +4 -13
@@ -1641,28 +1641,19 @@ static unsigned long xol_get_slot_nr(struct xol_area *area) }/* -* - search for a free slot. -*/ -static unsigned long xol_take_insn_slot (struct xol_area *area) -{ -unsigned long slot_nr; - -wait_event (area->wq, (slot_nr =xol_get_slot_nr (area)) <UINSNS_PER_PAGE ); - -return area->vaddr + slot_nr *UPROBE_XOL_SLOT_BYTES ; -} - -/* * xol_get_insn_slot - allocate a slot for xol. */ static bool xol_get_insn_slot (struct uprobe *uprobe,struct uprobe_task *utask) {struct xol_area *area =get_xol_area (); +unsigned long slot_nr;if (!area)return false; - utask->xol_vaddr =xol_take_insn_slot (area); +wait_event (area->wq, (slot_nr =xol_get_slot_nr (area)) <UINSNS_PER_PAGE ); + + utask->xol_vaddr = area->vaddr + slot_nr *UPROBE_XOL_SLOT_BYTES ;arch_uprobe_copy_ixol (area->page, utask->xol_vaddr, &uprobe->arch.ixol,sizeof (uprobe->arch.ixol));return true;
Quoted 15 lines · Oleg Nesterov
Hello,
On top of
[PATCH 0/7] uprobes: deuglify xol_get_insn_slot/xol_free_insn_slot paths
https://lore.kernel.org/all/[email protected]/To me this change looks like a simplification, but perhaps it can also be considered as optimization.
Yes, in the contended case
xol_get_slot_nr()will be called twice, the second time afterprepare_to_wait_event(), but I don't think it can hurt in the slow path when the caller is likely going to sleep.
Quoted 6 lines
Oleg.
---kernel/events/uprobes.c | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-)
LGTM, for the series:
But I've also been wondering if the kernel already has some sort of slot allocator that we can utilize here, instead of reimplementing all this? I'll cc Vlastimil in case he has some good pointers (or can point to someone else who might know).