perf sharded_mutex: Introduce sharded_mutex
Ian Rogers
reviewerPATCH 1/2 perf sharded_mutex: Introduce sharded_mutex
Per object mutexes may come with significant memory cost while a global mutex can suffer from unnecessary contention. A sharded mutex is a compromise where objects are hashed and then a particular mutex for the hash of the object used. Contention can be controlled by the number of shards.
--- tools/perf/util/Build | 1 + tools/perf/util/sharded_mutex.c | 28 ++++++++++++++++++++++++++++ tools/perf/util/sharded_mutex.h | 27 +++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 tools/perf/util/sharded_mutex.c create mode 100644 tools/perf/util/sharded_mutex.h
tools/perf/util/Build +1 -0
@@ -145,6 +145,7 @@ perf-y += mem2node.o perf-y += clockid.o perf-y += list_sort.o perf-y += mutex.o +perf-y += sharded_mutex.o perf-$(CONFIG_LIBBPF) += bpf-loader.o perf-$(CONFIG_LIBBPF) += bpf_map.o
tools/perf/util/sharded_mutex.c new file +28 -0
@@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "sharded_mutex.h" + +#include <stdlib.h> + +struct sharded_mutex *sharded_mutex__new (size_t num_shards) +{ +struct sharded_mutex *result; +size_t size =sizeof (*result) +sizeof (struct mutex ) * num_shards; + + result =malloc (size); +if (!result) +return NULL ; + + result->num_shards = num_shards; +for (size_t i =0 ; i < num_shards; i++) +mutex_init (&result->mutexes[i]); + +return result; +} + +void sharded_mutex__delete (struct sharded_mutex *sm) +{ +for (size_t i =0 ; i < sm->num_shards; i++) +mutex_destroy (&sm->mutexes[i]); + +free (sm); +}
tools/perf/util/sharded_mutex.h new file +27 -0
@@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef PERF_SHARDED_MUTEX_H +#define PERF_SHARDED_MUTEX_H + +#include "mutex.h" + +/* +* In a situation where a lock is needed per object, having a mutex can be +* relatively memory expensive (40 bytes on x86-64). If the object can be +* constantly hashed, a sharded mutex is an alternative global pool of mutexes +* where the mutex is looked up from a hash value. This can lead to collisions +* if the number of shards isn't large enough. +*/ +struct sharded_mutex { +size_t num_shards; +struct mutex mutexes[]; +}; + +struct sharded_mutex *sharded_mutex__new (size_t num_shards); +void sharded_mutex__delete (struct sharded_mutex *sm); + +static inline struct mutex *sharded_mutex__get_mutex (struct sharded_mutex *sm,size_t hash) +{ +return &sm->mutexes[hash % sm->num_shards]; +} + +#endif /* PERF_SHARDED_MUTEX_H */
Signature · 1 line
2.41.0.162.gfafddb0af9-goog
Ian Rogers
reviewerPATCH 2/2 perf annotation: Switch lock from a mutex to a sharded_mutex
Remove the "struct mutex lock" variable from annotation that is allocated per symbol. This removes in the region of 40 bytes per symbol allocation. Use a sharded mutex where the number of shards is set to the number of CPUs. Assuming good hashing of the annotation (done based on the pointer), this means in order to contend there needs to be more threads than CPUs, which is not currently true in any perf command. Were contention an issue it is straightforward to increase the number of shards in the mutex.
On my Debian/glibc based machine, this reduces the size of struct annotation from 136 bytes to 96 bytes, or nearly 30%.
--- tools/perf/builtin-top.c | 14 +++---- tools/perf/ui/browsers/annotate.c | 10 ++--- tools/perf/util/annotate.c | 66 ++++++++++++++++++++++++++----- tools/perf/util/annotate.h | 11 ++++-- 4 files changed, 77 insertions(+), 24 deletions(-)
tools/perf/builtin-top.c +7 -7
@@ -137,10 +137,10 @@ static int perf_top__parse_source(struct perf_top *top, struct hist_entry *he) } notes =symbol__annotation (sym); -mutex_lock (¬es->lock); +annotation__lock (notes);if (!symbol__hists (sym, top->evlist->core.nr_entries)) { -mutex_unlock (¬es->lock); +annotation__unlock (notes);pr_err ("Not enough memory for annotating '%s' symbol!\n" , sym->name);sleep (1 );
@@ -156,7 +156,7 @@ static int perf_top__parse_source(struct perf_top *top, struct hist_entry *he)pr_err ("Couldn't annotate %s: %s\n" , sym->name, msg); } -mutex_unlock (¬es->lock); +annotation__unlock (notes);return err; }
@@ -211,12 +211,12 @@ static void perf_top__record_precise_ip(struct perf_top *top, notes =symbol__annotation (sym); -if (!mutex_trylock (¬es->lock)) +if (!annotation__trylock (notes))return ; err =hist_entry__inc_addr_samples (he, sample, evsel, ip); -mutex_unlock (¬es->lock); +annotation__unlock (notes);if (unlikely (err)) {/*
@@ -253,7 +253,7 @@ static void perf_top__show_details(struct perf_top *top) symbol = he->ms.sym; notes =symbol__annotation (symbol); -mutex_lock (¬es->lock); +annotation__lock (notes);symbol__calc_percent (symbol, evsel);
@@ -274,7 +274,7 @@ static void perf_top__show_details(struct perf_top *top)if (more !=0 )printf ("%d lines not displayed, maybe increase display entries [e]\n" , more); out_unlock: -mutex_unlock (¬es->lock); +annotation__unlock (notes); }static void perf_top__resort_hists (struct perf_top *t)
tools/perf/ui/browsers/annotate.c +5 -5
@@ -314,7 +314,7 @@ static void annotate_browser__calc_percent(struct annotate_browser *browser, browser->entries =RB_ROOT ; -mutex_lock (¬es->lock); +annotation__lock (notes);symbol__calc_percent (sym, evsel);
@@ -343,7 +343,7 @@ static void annotate_browser__calc_percent(struct annotate_browser *browser, }disasm_rb_tree__insert (browser, &pos->al); } -mutex_unlock (¬es->lock); +annotation__unlock (notes); browser->curr_hot =rb_last (&browser->entries); }
@@ -470,10 +470,10 @@ static bool annotate_browser__callq(struct annotate_browser *browser, } notes =symbol__annotation (dl->ops.target.sym); -mutex_lock (¬es->lock); +annotation__lock (notes);if (!symbol__hists (dl->ops.target.sym, evsel->evlist->core.nr_entries)) { -mutex_unlock (¬es->lock); +annotation__unlock (notes);ui__warning ("Not enough memory for annotating '%s' symbol!\n" , dl->ops.target.sym->name);return true;
@@ -482,7 +482,7 @@ static bool annotate_browser__callq(struct annotate_browser *browser, target_ms.maps = ms->maps; target_ms.map = ms->map; target_ms.sym = dl->ops.target.sym; -mutex_unlock (¬es->lock); +annotation__unlock (notes);symbol__tui_annotate (&target_ms, evsel, hbt, browser->opts);sym_title (ms->sym, ms->map, title,sizeof (title), browser->opts->percent_type);ui_browser__show_title (&browser->b, title);
tools/perf/util/annotate.c +57 -9
108 lines not on this page · raw message
tools/perf/util/annotate.h +8 -3
@@ -271,8 +271,7 @@ struct annotated_source {struct sym_hist *histograms; }; -struct annotation { -struct mutex lock; +struct LOCKABLE annotation {u64 max_coverage;u64 start;u64 hit_cycles;
@@ -298,9 +297,15 @@ struct annotation {struct annotated_source *src; }; -void annotation__init (struct annotation *notes); +static inline void annotation__init (struct annotation *notes__maybe_unused ) +{ +}void annotation__exit (struct annotation *notes); +void annotation__lock(struct annotation *notes) EXCLUSIVE_LOCK_FUNCTION(*notes); +void annotation__unlock(struct annotation *notes) UNLOCK_FUNCTION(*notes); +bool annotation__trylock(struct annotation *notes) EXCLUSIVE_TRYLOCK_FUNCTION(true, *notes); +static inline int annotation__cycles_width (struct annotation *notes) {if (notes->have_cycles && notes->options->show_minmax_cycle)
Signature · 1 line
2.41.0.162.gfafddb0af9-goog
Namhyung Kim
maintainerRe: [PATCH v1 2/2] perf annotation: Switch lock from a mutex to a sharded_mutex
Hi Ian,
Quoted 10 lines · Ian Rogers
Remove the "struct mutex lock" variable from annotation that is allocated per symbol. This removes in the region of 40 bytes per symbol allocation. Use a sharded mutex where the number of shards is set to the number of CPUs. Assuming good hashing of the annotation (done based on the pointer), this means in order to contend there needs to be more threads than CPUs, which is not currently true in any perf command. Were contention an issue it is straightforward to increase the number of shards in the mutex.
Quoted 2 lines
On my Debian/glibc based machine, this reduces the size of struct annotation from 136 bytes to 96 bytes, or nearly 30%.
That's quite a good improvement given the number of symbols we can have in a report session!
Quoted 3 lines
Signed-off-by Ian Rogers <[email protected]>---
[SNIP]
Quoted 15 lines
tools/perf/util/annotate.c +5
-3@@ -1291,17 +1292,64 @@ int disasm_line__scnprintf(struct disasm_line *dl, char *bf, size_t size, bool rreturn ins__scnprintf (&dl->ins, bf, size, &dl->ops, max_ins_name); } -void annotation__init (struct annotation *notes) +void annotation__exit (struct annotation *notes) { -mutex_init (¬es->lock); +annotated_source__delete (notes->src); } -void annotation__exit (struct annotation *notes) +static struct sharded_mutex *sharded_mutex; + +static void annotation__init_sharded_mutex (void )
tools/perf/util/annotate.c int disasm_line__scnprintf(struct disasm_line *dl, char *bf, size_t size, bool r · line 1310
Quoted 10 lines
tools/perf/util/annotate.c +7
-2{ -annotated_source__delete (notes->src); -mutex_destroy (¬es->lock); +/* As many mutexes as there are CPUs. */ + sharded_mutex =sharded_mutex__new (cpu__max_present_cpu ().cpu); +} + +static size_t annotation__hash (const struct annotation *notes) +{ +return ((size_t )notes) >>4 ;
But I'm afraid it might create more contention depending on the malloc implementation. If it always returns 128-byte (or 256) aligned memory for this struct then it could always collide in the slot 0 if the number of CPU is 8 or less, right?
Thanks,
Namhyung
Quoted 45 lines
tools/perf/util/annotate.c +40
-0} +static struct mutex *annotation__get_mutex (const struct annotation *notes) +{ +static pthread_once_t once =PTHREAD_ONCE_INIT ; + +pthread_once (&once, annotation__init_sharded_mutex); +if (!sharded_mutex) +return NULL ; + +return sharded_mutex__get_mutex (sharded_mutex,annotation__hash (notes)); +} + +void annotation__lock (struct annotation *notes) +NO_THREAD_SAFETY_ANALYSIS +{ +struct mutex *mutex =annotation__get_mutex (notes); + +if (mutex) +mutex_lock (mutex); +} + +void annotation__unlock (struct annotation *notes) +NO_THREAD_SAFETY_ANALYSIS +{ +struct mutex *mutex =annotation__get_mutex (notes); + +if (mutex) +mutex_unlock (mutex); +} + +bool annotation__trylock (struct annotation *notes) +{ +struct mutex *mutex =annotation__get_mutex (notes); + +if (!mutex) +return false; + +return mutex_trylock (mutex); +} + +static void annotation_line__add (struct annotation_line *al,struct list_head *head) {list_add_tail (&al->node, head);
Ian Rogers
reviewerRe: [PATCH v1 2/2] perf annotation: Switch lock from a mutex to a sharded_mutex
Quoted 47 lines · Namhyung Kim
Hi Ian,
Quoted 10 lines · Ian Rogers
Remove the "struct mutex lock" variable from annotation that is allocated per symbol. This removes in the region of 40 bytes per symbol allocation. Use a sharded mutex where the number of shards is set to the number of CPUs. Assuming good hashing of the annotation (done based on the pointer), this means in order to contend there needs to be more threads than CPUs, which is not currently true in any perf command. Were contention an issue it is straightforward to increase the number of shards in the mutex.
Quoted 2 lines
On my Debian/glibc based machine, this reduces the size of struct annotation from 136 bytes to 96 bytes, or nearly 30%.
That's quite a good improvement given the number of symbols we can have in a report session!
Quoted 3 lines
Signed-off-by Ian Rogers <[email protected]>---
[SNIP]
Quoted 22 lines
tools/perf/util/annotate.c +9
-5@@ -1291,17 +1292,64 @@ int disasm_line__scnprintf(struct disasm_line *dl, char *bf, size_t size, bool rreturn ins__scnprintf (&dl->ins, bf, size, &dl->ops, max_ins_name); } -void annotation__init (struct annotation *notes) +void annotation__exit (struct annotation *notes) { -mutex_init (¬es->lock); +annotated_source__delete (notes->src); } -void annotation__exit (struct annotation *notes) +static struct sharded_mutex *sharded_mutex; + +static void annotation__init_sharded_mutex (void ) { -annotated_source__delete (notes->src); -mutex_destroy (¬es->lock); +/* As many mutexes as there are CPUs. */ + sharded_mutex =sharded_mutex__new (cpu__max_present_cpu ().cpu); +} +
Quoted 8 lines
tools/perf/util/annotate.c int disasm_line__scnprintf(struct disasm_line *dl, char *bf, size_t size, bool r · line 1310
Quoted 3 lines
tools/perf/util/annotate.c +3
-0+static size_t annotation__hash (const struct annotation *notes) +{ +return ((size_t )notes) >>4 ;But I'm afraid it might create more contention depending on the malloc implementation. If it always returns 128-byte (or 256) aligned memory for this struct then it could always collide in the slot 0 if the number of CPU is 8 or less, right?
Right, I think we can use a secondary hash and hashmap.h has one lying around for us:
https://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git/tree/tools/perf/util/hashmap.h?h=tmp.perf-tools-next#n15
it will mean that the sharded locks will need to be a power of 2 capacity. I'll work on a v2. Fwiw, the hash of a pointer for collection like those in abseil is just the pointer value, so I'll drop the shift to remove the low-bits once I'm using a more expensive hash function.
Thanks,
Ian
Quoted 49 lines
Thanks,
NamhyungQuoted 45 lines
tools/perf/util/annotate.c +40
-0} +static struct mutex *annotation__get_mutex (const struct annotation *notes) +{ +static pthread_once_t once =PTHREAD_ONCE_INIT ; + +pthread_once (&once, annotation__init_sharded_mutex); +if (!sharded_mutex) +return NULL ; + +return sharded_mutex__get_mutex (sharded_mutex,annotation__hash (notes)); +} + +void annotation__lock (struct annotation *notes) +NO_THREAD_SAFETY_ANALYSIS +{ +struct mutex *mutex =annotation__get_mutex (notes); + +if (mutex) +mutex_lock (mutex); +} + +void annotation__unlock (struct annotation *notes) +NO_THREAD_SAFETY_ANALYSIS +{ +struct mutex *mutex =annotation__get_mutex (notes); + +if (mutex) +mutex_unlock (mutex); +} + +bool annotation__trylock (struct annotation *notes) +{ +struct mutex *mutex =annotation__get_mutex (notes); + +if (!mutex) +return false; + +return mutex_trylock (mutex); +} + +static void annotation_line__add (struct annotation_line *al,struct list_head *head) {list_add_tail (&al->node, head);