uprobes: make trace_uprobe->nhit counter a per-CPU one
Andrii Nakryiko
maintainerPATCH v2 1/1 uprobes: make trace_uprobe->nhit counter a per-CPU one
trace_uprobe->nhit counter is not incremented atomically, so its value is questionable in when uprobe is hit on multiple CPUs simultaneously.
Also, doing this shared counter increment across many CPUs causes heavy cache line bouncing, limiting uprobe/uretprobe performance scaling with number of CPUs.
Solve both problems by making this a per-CPU counter.
--- kernel/trace/trace_uprobe.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-)
kernel/trace/trace_uprobe.c +21 -3
@@ -17,6 +17,7 @@#include <linux/string.h> #include <linux/rculist.h> #include <linux/filter.h> +#include <linux/percpu.h> #include "trace_dynevent.h" #include "trace_probe.h"
@@ -62,7 +63,7 @@ struct trace_uprobe {struct uprobe *uprobe;unsigned long offset;unsigned long ref_ctr_offset; -unsigned long nhit; +unsigned long __percpu *nhits;struct trace_probe tp; };
@@ -337,6 +338,12 @@ alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)if (!tu)return ERR_PTR (-ENOMEM ); + tu->nhits = alloc_percpu(unsigned long); +if (!tu->nhits) { + ret = -ENOMEM ; + goto error; + } + ret =trace_probe_init (&tu->tp, event, group, true, nargs);if (ret <0 ) goto error;
@@ -349,6 +356,7 @@ alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)return tu; error: +free_percpu (tu->nhits);kfree (tu);return ERR_PTR (ret);
@@ -362,6 +370,7 @@ static void free_trace_uprobe(struct trace_uprobe *tu)path_put (&tu->path);trace_probe_cleanup (&tu->tp);kfree (tu->filename); +free_percpu (tu->nhits);kfree (tu); }
@@ -815,13 +824,21 @@ static int probes_profile_seq_show(struct seq_file *m, void *v) {struct dyn_event *ev = v;struct trace_uprobe *tu; +unsigned long nhits; +int cpu;if (!is_trace_uprobe (ev))return 0 ; tu =to_trace_uprobe (ev); + + nhits =0 ; +for_each_possible_cpu (cpu) { + nhits +=READ_ONCE (*per_cpu_ptr (tu->nhits, cpu)); + } +seq_printf (m," %s %-44s %15lu\n" , tu->filename, -trace_probe_name (&tu->tp), tu->nhit); +trace_probe_name (&tu->tp), nhits);return 0 ; }
@@ -1507,7 +1524,8 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)int ret =0 ; tu = container_of(con, struct trace_uprobe, consumer); - tu->nhit++; + +this_cpu_inc (*tu->nhits); udd.tu = tu; udd.bp_addr =instruction_pointer (regs);
Signature · 1 line
2.43.5
Masami Hiramatsu (Google)
maintainerQuoted 9 lines · On Fri, 9 Aug 2024 12:23:57 -0700 Andrii Nakryiko <[email protected]> wrote:
trace_uprobe->nhitcounter is not incremented atomically, so its value is questionable in when uprobe is hit on multiple CPUs simultaneously.Also, doing this shared counter increment across many CPUs causes heavy cache line bouncing, limiting uprobe/uretprobe performance scaling with number of CPUs.
Solve both problems by making this a per-CPU counter.
This looks good to me. I would like to pick this to linux-trace/probes/for-next.
kernel/trace/trace_uprobe.c struct trace_uprobe { · line 63
Quoted 2 lines
kernel/trace/trace_uprobe.c +0
-0@@ -62,7 +63,7 @@ struct trace_uprobe {struct uprobe *uprobe;
BTW, what is this change? I couldn't cleanly apply this to the v6.11-rc3. Which tree would you working on? (I missed something?)
Thanks,
Quoted 72 lines
kernel/trace/trace_uprobe.c +20
-3unsigned long offset;unsigned long ref_ctr_offset; -unsigned long nhit; +unsigned long __percpu *nhits;struct trace_probe tp; };@@ -337,6 +338,12 @@ alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)if (!tu)return ERR_PTR (-ENOMEM ); + tu->nhits = alloc_percpu(unsigned long); +if (!tu->nhits) { + ret = -ENOMEM ; + goto error; + } + ret =trace_probe_init (&tu->tp, event, group, true, nargs);if (ret <0 ) goto error;@@ -349,6 +356,7 @@ alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)return tu; error: +free_percpu (tu->nhits);kfree (tu);return ERR_PTR (ret);@@ -362,6 +370,7 @@ static void free_trace_uprobe(struct trace_uprobe *tu)path_put (&tu->path);trace_probe_cleanup (&tu->tp);kfree (tu->filename); +free_percpu (tu->nhits);kfree (tu); }@@ -815,13 +824,21 @@ static int probes_profile_seq_show(struct seq_file *m, void *v) {struct dyn_event *ev = v;struct trace_uprobe *tu; +unsigned long nhits; +int cpu;if (!is_trace_uprobe (ev))return 0 ; tu =to_trace_uprobe (ev); + + nhits =0 ; +for_each_possible_cpu (cpu) { + nhits +=READ_ONCE (*per_cpu_ptr (tu->nhits, cpu)); + } +seq_printf (m," %s %-44s %15lu\n" , tu->filename, -trace_probe_name (&tu->tp), tu->nhit); +trace_probe_name (&tu->tp), nhits);return 0 ; }@@ -1507,7 +1524,8 @@ static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs)int ret =0 ; tu = container_of(con, struct trace_uprobe, consumer); - tu->nhit++; + +this_cpu_inc (*tu->nhits); udd.tu = tu; udd.bp_addr =instruction_pointer (regs);Signature · 1 line
2.43.5
Signature · 1 line
Masami Hiramatsu (Google) <[email protected]>
Quoted 6 lines · Masami Hiramatsu
kernel/trace/trace_uprobe.c struct trace_uprobe { · line 63
Quoted 3 lines
kernel/trace/trace_uprobe.c +0
-0@@ -62,7 +63,7 @@ struct trace_uprobe {struct uprobe *uprobe;BTW, what is this change? I couldn't cleanly apply this to the v6.11-rc3. Which tree would you working on? (I missed something?)
tip/perf/core
Oleg.
Masami Hiramatsu (Google)
maintainerQuoted 11 lines · On Tue, 13 Aug 2024 17:41:04 +0200 Oleg Nesterov <[email protected]> wrote:
Quoted 6 lines · Masami Hiramatsu
kernel/trace/trace_uprobe.c struct trace_uprobe { · line 63
Quoted 3 lines
kernel/trace/trace_uprobe.c +0
-0@@ -62,7 +63,7 @@ struct trace_uprobe {struct uprobe *uprobe;BTW, what is this change? I couldn't cleanly apply this to the v6.11-rc3. Which tree would you working on? (I missed something?)
tip/perf/core
OK, let me consider to rebase on tip/perf/core.
Thank you,
Quoted 3 lines
Oleg.
Signature · 1 line
Masami Hiramatsu (Google) <[email protected]>
Quoted 12 lines · Masami Hiramatsu
Quoted 8 lines · On Tue, 13 Aug 2024 17:41:04 +0200 Oleg Nesterov <[email protected]> wrote:
Quoted 6 lines · Masami Hiramatsu
kernel/trace/trace_uprobe.c struct trace_uprobe { · line 63
Quoted 3 lines
kernel/trace/trace_uprobe.c +0
-0@@ -62,7 +63,7 @@ struct trace_uprobe {struct uprobe *uprobe;BTW, what is this change? I couldn't cleanly apply this to the v6.11-rc3. Which tree would you working on? (I missed something?)
Quoted 6 lines
Quoted 3 lines
tip/perf/core
OK, let me consider to rebase on tip/perf/core.
Hey Masami,
I've posted v3 rebased onto linux-trace/probes/for-next, so you shouldn't need to rebase anything just for this. See [0] for the latest revision.
[0] https://lore.kernel.org/linux-trace-kernel/[email protected]/
Quoted 9 lines
Thank you,
Quoted 3 lines
Oleg.
--
Masami Hiramatsu (Google) <[email protected]>
Quoted 6 lines · Andrii Nakryiko
kernel/trace/trace_uprobe.c +1
-0@@ -815,13 +824,21 @@ static int probes_profile_seq_show(struct seq_file *m, void *v) {struct dyn_event *ev = v;struct trace_uprobe *tu; +unsigned long nhits;
kernel/trace/trace_uprobe.c static int probes_profile_seq_show(struct seq_file *m, void *v) · line 837
Quoted 10 lines
kernel/trace/trace_uprobe.c +5
-0+int cpu;if (!is_trace_uprobe (ev))return 0 ; tu =to_trace_uprobe (ev); + + nhits =0 ; +for_each_possible_cpu (cpu) { + nhits +=READ_ONCE (*per_cpu_ptr (tu->nhits, cpu));
why not
nhits += per_cpu(*tu->nhits, cpu);
?
See for example per_cpu_sum() or nr_processes(), per_cpu() should work just fine...
Other than that
Quoted 21 lines · Oleg Nesterov
Quoted 6 lines · Andrii Nakryiko
kernel/trace/trace_uprobe.c +1
-0@@ -815,13 +824,21 @@ static int probes_profile_seq_show(struct seq_file *m, void *v) {struct dyn_event *ev = v;struct trace_uprobe *tu; +unsigned long nhits;kernel/trace/trace_uprobe.c static int probes_profile_seq_show(struct seq_file *m, void *v) · line 837
Quoted 10 lines
kernel/trace/trace_uprobe.c +5
-0+int cpu;if (!is_trace_uprobe (ev))return 0 ; tu =to_trace_uprobe (ev); + + nhits =0 ; +for_each_possible_cpu (cpu) { + nhits +=READ_ONCE (*per_cpu_ptr (tu->nhits, cpu));why not
Quoted 6 lines
nhits += per_cpu(*tu->nhits, cpu);?
See for example
per_cpu_sum()ornr_processes(),per_cpu()should work just fine...
I just monkeyed it from some existing code somewhere in the BPF code base. I like per_cpu, will send a v3 and rebase it onto a linux-trace tree.
Quoted 4 lines
Other than that
Reviewed-by Oleg Nesterov <[email protected]>