perf: Provide status of known PMUs
Subsystems (2)
Adrian Hunter
reviewerRFC PATCH 1/1 perf: Provide status of known PMUs
Known PMUs may not be present for various reasons. Provide a way for the user to know what the reason is.
A bus attribute is created for each known PMU beneath a group "known_pmus". The attribute name is the same as the PMU name. The value is a string consisting of one or, optionally, two parts: a canonical part, and a driver specific part. If there are two parts, they are separated by " - ". The canonical part is one of:
Supported
Driver error
Driver not loaded
Driver not in kernel config
Not supported by kernel
Not supported by hardware
Wrong vendor
Wrong architecture
Unknown status
Example:
$ cat /sys/bus/event_source/known_pmus/intel_pt Supported
--- arch/x86/kernel/cpu/perf_event.c | 3 + arch/x86/kernel/cpu/perf_event_intel_bts.c | 20 +++- arch/x86/kernel/cpu/perf_event_intel_pt.c | 18 ++- include/linux/perf_event.h | 23 ++++ kernel/events/core.c | 171 +++++++++++++++++++++++++++++ 5 files changed, 226 insertions(+), 9 deletions(-)
arch/x86/kernel/cpu/perf_event.c +3 -0
@@ -1670,12 +1670,15 @@ static int __init init_hw_perf_events(void)switch (boot_cpu_data.x86_vendor) {case X86_VENDOR_INTEL : +perf_pmu_vendor ("Intel" ); err =intel_pmu_init ();break ;case X86_VENDOR_AMD : +perf_pmu_vendor ("AMD" ); err =amd_pmu_init ();break ;default : +perf_pmu_vendor ("Unknown" ); err = -ENOTSUPP ; }if (err !=0 ) {
arch/x86/kernel/cpu/perf_event_intel_bts.c +17 -3
@@ -514,8 +514,19 @@ static void bts_event_read(struct perf_event *event)static __init int bts_init (void ) { -if (!boot_cpu_has (X86_FEATURE_DTES64 ) || !x86_pmu.bts) -return -ENODEV ; +const char *status_msg =NULL ; +int ret; + +if (!x86_pmu.bts) { + ret = -ENODEV ; + goto out; + } + +if (!boot_cpu_has (X86_FEATURE_DTES64 )) { + status_msg ="requires 64-bit kernel" ; + ret = -ENOTSUPP ; + goto out; + } bts_pmu.capabilities =PERF_PMU_CAP_AUX_NO_SG |PERF_PMU_CAP_ITRACE ; bts_pmu.task_ctx_nr = perf_sw_context;
@@ -528,6 +539,9 @@ static __init int bts_init(void) bts_pmu.setup_aux = bts_buffer_setup_aux; bts_pmu.free_aux = bts_buffer_free_aux; -return perf_pmu_register (&bts_pmu,"intel_bts" ,-1 ); + ret =perf_pmu_register (&bts_pmu,"intel_bts" ,-1 ); +out: +perf_pmu_error_status ("intel_bts" , ret, status_msg); +return ret; }arch_initcall (bts_init);
arch/x86/kernel/cpu/perf_event_intel_pt.c +12 -6
@@ -1059,6 +1059,7 @@ static int pt_event_init(struct perf_event *event)static __init int pt_init (void ) {int ret, cpu, prior_warn =0 ; +const char *status_msg =NULL ;BUILD_BUG_ON (sizeof (struct topa ) >PAGE_SIZE );get_online_cpus ();
@@ -1073,18 +1074,19 @@ static __init int pt_init(void)if (prior_warn) {x86_add_exclusive (x86_lbr_exclusive_pt); -pr_warn ("PT is enabled at boot time, doing nothing\n" ); - -return -EBUSY ; + status_msg ="PT is enabled at boot time, doing nothing" ; + ret = -EBUSY ; + goto out; } ret =pt_pmu_hw_init ();if (ret) -return ret; + goto out;if (!pt_cap_get (PT_CAP_topa_output)) { -pr_warn ("ToPA output is not supported on this CPU\n" ); -return -ENODEV ; + status_msg ="ToPA output is not supported on this CPU" ; + ret = -ENODEV ; + goto out; }if (!pt_cap_get (PT_CAP_topa_multiple_entries))
@@ -1103,6 +1105,10 @@ static __init int pt_init(void) pt_pmu.pmu.setup_aux = pt_buffer_setup_aux; pt_pmu.pmu.free_aux = pt_buffer_free_aux; ret =perf_pmu_register (&pt_pmu.pmu,"intel_pt" ,-1 ); +out: +if (status_msg) +pr_warn ("%s\n" , status_msg); +perf_pmu_error_status ("intel_pt" , ret, status_msg);return ret; }
include/linux/perf_event.h +23 -0
@@ -179,6 +179,22 @@ struct perf_event;#define PERF_PMU_CAP_ITRACE 0x20/** +* enum perf_pmu_status - the status of known PMUs +*/ +enum perf_pmu_status { +PERF_PMU_STATUS_SUPPORTED , +PERF_PMU_STATUS_ERROR , +PERF_PMU_STATUS_NOT_LOADED , +PERF_PMU_STATUS_NOT_CONFIG , +PERF_PMU_STATUS_NOT_SUPPORTED , +PERF_PMU_STATUS_WRONG_HW , +PERF_PMU_STATUS_WRONG_VENDOR , +PERF_PMU_STATUS_WRONG_ARCH , +PERF_PMU_STATUS_UNKNOWN , +PERF_PMU_STATUS_MAX , +}; + +/** * struct pmu - generic performance monitoring unit */ struct pmu {
@@ -631,6 +647,13 @@ extern void *perf_get_aux(struct perf_output_handle *handle);extern int perf_pmu_register (struct pmu *pmu,const char *name,int type);extern void perf_pmu_unregister (struct pmu *pmu); +extern void perf_pmu_update_status (const char *name, +enum perf_pmu_status status, +const char *status_msg); +extern void perf_pmu_error_status (const char *name,int err, +const char *status_msg); +extern void perf_pmu_vendor (const char *vendor); +extern int perf_num_counters (void );extern const char *perf_pmu_name (void ); extern void __perf_event_task_sched_in(struct task_struct *prev,
kernel/events/core.c +171 -0
183 lines not on this page · raw message
Signature · 1 line
1.9.1
Quoted 11 lines · Adrian Hunter
Known PMUs may not be present for various reasons. Provide a way for the user to know what the reason is.
A bus attribute is created for each known PMU beneath a group "
known_pmus". The attribute name is the same as the PMU name. The value is a string consisting of one or, optionally, two parts: a canonical part, and a driver specific part. If there are two parts, they are separated by " - ". The canonical part is one of:
Quoted 9 lines
Supported
Driver error
Driver not loaded
Driver not in kernel config
Not supported by kernel
Not supported by hardware
Wrong vendor
Wrong architecture
Unknown status
Very nice!
Quoted 4 lines
Example:
$ cat /sys/bus/event_source/known_pmus/intel_pt Supported
So I only have naming nits. 'Supported' is a bit ambiguous, because it could mean that the PMU is supported but the driver is not active. How about 'Enabled'?
I'd also make the strings more unambiguously structured, something like:
Enabled
Disabled: Driver error
Disabled: Driver not loaded
Disabled: Driver not in kernel config
Disabled: Not supported by the kernel
Disabled: Not supported by the hardware
Disabled: Not supported by the hardware vendor
Disabled: Not supported by the the architecture
Disabled: Unknown status
(Note the small changes I did to the text in some places.)
Also note that I'd suggest not enumerating all the error reasons rigidly - just have a single error code, but a free flowing error string that is provided by the low level driver (and maybe strdup()-ed by the core). That way you can provide very specific error descriptions, without having to change the core every time you need a new category. Agreed?
Thanks,
Ingo
Adrian Hunter
reviewerQuoted 46 lines · Ingo Molnar
Quoted 11 lines · Adrian Hunter
Known PMUs may not be present for various reasons. Provide a way for the user to know what the reason is.
A bus attribute is created for each known PMU beneath a group "
known_pmus". The attribute name is the same as the PMU name. The value is a string consisting of one or, optionally, two parts: a canonical part, and a driver specific part. If there are two parts, they are separated by " - ". The canonical part is one of:Quoted 9 lines
Supported
Driver error
Driver not loaded
Driver not in kernel config
Not supported by kernel
Not supported by hardware
Wrong vendor
Wrong architecture
Unknown statusVery nice!
Quoted 4 lines
Example:
$ cat /sys/bus/event_source/known_pmus/intel_pt SupportedSo I only have naming nits. 'Supported' is a bit ambiguous, because it could mean that the PMU is supported but the driver is not active. How about 'Enabled'?
I'd also make the strings more unambiguously structured, something like:
Enabled
Disabled: Driver error
Disabled: Driver not loaded
Disabled: Driver not in kernel config
Disabled: Not supported by the kernel
Disabled: Not supported by the hardware
Disabled: Not supported by the hardware vendor
Disabled: Not supported by the the architecture
Disabled: Unknown status
Quoted 1 line
(Note the small changes I did to the text in some places.)
OK
Quoted 6 lines
Also note that I'd suggest not enumerating all the error reasons rigidly - just have a single error code, but a free flowing error string that is provided by the low level driver (and maybe strdup()-ed by the core). That way you can provide very specific error descriptions, without having to change the core every time you need a new category. Agreed?
Drivers can optionally provide a string - the optional second part described above. For example:
$ cat /sys/bus/event_source/known_pmus/intel_pt Disabled: Not supported by the hardware - ToPA output is not supported on this CPU
Having a finite set of categories allows software to interpret the string for purposes other than displaying it. Having an optional second part allows drivers to detail anything else.
Peter Zijlstra
maintainerQuoted 3 lines · Adrian Hunter
Known PMUs may not be present for various reasons. Provide a way for the user to know what the reason is.
Not a bad idea, but I do wonder where we should draw the line on what is 'known'. The patch as proposed will have bts/pt listed as 'known' for every arch out there.
By that logic, x86 should list the ppc/sparc/mips/arm/etc.. PMUs as known and wrong_arch too, which might be a tad excessive.
Can we limit it to PMUs for which we've (attempted to) load the drivers? That would obviously make a few of your status bits redundant, but then you've not explained why we're interested in it.
Quoted 6 lines
Supported
Driver error
Driver not loaded
Not supported by hardware
Wrong vendor
Unknown status
There would work.
Quoted 3 lines
Driver not in kernel config
Not supported by kernel
Wrong architecture
These will be hard, for if we don't load the driver we don't 'know' of them.
Quoted 11 lines · Peter Zijlstra
Quoted 2 lines · Adrian Hunter
Known PMUs may not be present for various reasons. Provide a way for the user to know what the reason is.
Not a bad idea, but I do wonder where we should draw the line on what is 'known'. The patch as proposed will have bts/pt listed as 'known' for every arch out there.
By that logic, x86 should list the ppc/sparc/mips/arm/etc.. PMUs as known and
wrong_archtoo, which might be a tad excessive.
Absolutely x86 should list them as well - from a user POV arch dependent tooling sucks in general. There's nothing more annoying than trying to figure out why a particular tool does not work.
If memory usage becomes any concern in the future we can list them in a bit more compressed form.
Thanks,
Ingo
Peter Zijlstra
maintainerQuoted 12 lines · Ingo Molnar
Quoted 9 lines · Peter Zijlstra
Quoted 2 lines · Adrian Hunter
Known PMUs may not be present for various reasons. Provide a way for the user to know what the reason is.
Not a bad idea, but I do wonder where we should draw the line on what is 'known'. The patch as proposed will have bts/pt listed as 'known' for every arch out there.
Quoted 6 lines
Quoted 2 lines
By that logic, x86 should list the ppc/sparc/mips/arm/etc.. PMUs as known and
wrong_archtoo, which might be a tad excessive.Absolutely x86 should list them as well - from a user POV arch dependent tooling sucks in general. There's nothing more annoying than trying to figure out why a particular tool does not work.
But why would the tool care?
Quoted 13 lines · Peter Zijlstra
Quoted 12 lines · Ingo Molnar
Quoted 9 lines · Peter Zijlstra
Quoted 2 lines · Adrian Hunter
Known PMUs may not be present for various reasons. Provide a way for the user to know what the reason is.
Not a bad idea, but I do wonder where we should draw the line on what is 'known'. The patch as proposed will have bts/pt listed as 'known' for every arch out there.
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
By that logic, x86 should list the ppc/sparc/mips/arm/etc.. PMUs as known and
wrong_archtoo, which might be a tad excessive.Absolutely x86 should list them as well - from a user POV arch dependent tooling sucks in general. There's nothing more annoying than trying to figure out why a particular tool does not work.
But why would the tool care?
Yeah, but the user cares. If I type something like:
perf record -e bts// --per-thread sleep 1
... and it does not work, I expect the tool to print something informative and productive, for example one of these:
perf record error: The 'bts' PMU is not available, because this architecture does not support it perf record error: The 'bts' PMU is not available, because the CPU does not support it perf record error: The 'bts' PMU is not available, because its driver is not built into the kernel
Because if it's the wrong architecture or CPU, I look for a box with the right one, if it's simply the kernel not having the necessary PMU driver then I'll boot a kernel with it enabled.
Thanks,
Ingo
Peter Zijlstra
maintainerQuoted 2 lines · Ingo Molnar
perf record error: The 'bts' PMU is not available, because the CPU does not support it
This one makes sense.
Quoted 6 lines
perf record error: The 'bts' PMU is not available, because this architecture does not support it perf record error: The 'bts' PMU is not available, because its driver is not built into the kernel
Because if it's the wrong architecture or CPU, I look for a box with the right one, if it's simply the kernel not having the necessary PMU driver then I'll boot a kernel with it enabled.
These not so much; why won't a generic: "Unknown PMU, check arch/kernel" do?
The thing is, I hate that hard-coded list, its pain I don't need.
Arnaldo Carvalho de Melo
maintainerEm Thu, Jul 09, 2015 at 02:42:57PM +0200, Peter Zijlstra escreveu:
Quoted 9 lines · Ingo Molnar
Quoted 2 lines
perf record error: The 'bts' PMU is not available, because the CPU does not support it
This one makes sense.
Quoted 2 lines
perf record error: The 'bts' PMU is not available, because this architecture does not support it perf record error: The 'bts' PMU is not available, because its driver is not built into the kernel
Quoted 8 lines
Quoted 3 lines
Because if it's the wrong architecture or CPU, I look for a box with the right one, if it's simply the kernel not having the necessary PMU driver then I'll boot a kernel with it enabled.
These not so much; why won't a generic: "Unknown PMU, check
arch/kernel" do?The thing is, I hate that hard-coded list, its pain I don't need.
Well, then we need to have tooling looking around and having tables to provide those more precise error messages, which we want to have.
I guess we can do parts of that by looking at info already exported by the kernel, /proc/cpuinfo stuff, etc. But "Unknown PMU, check arch/kernel" is way too vague :-\
- Arnaldo
Quoted 6 lines · Peter Zijlstra
Quoted 2 lines · Ingo Molnar
perf record error: The 'bts' PMU is not available, because the CPU does not support it
This one makes sense.
Quoted 8 lines
Quoted 6 lines
perf record error: The 'bts' PMU is not available, because this architecture does not support it perf record error: The 'bts' PMU is not available, because its driver is not built into the kernel
Because if it's the wrong architecture or CPU, I look for a box with the right one, if it's simply the kernel not having the necessary PMU driver then I'll boot a kernel with it enabled.
These not so much; why won't a generic: "Unknown PMU, check
arch/kernel" do?
Yeah, I mean why not make the user's job harder if we can? We really don't want to solve this problem technically and we _really_ want tooling to be fundamentally unhelpful, right? ;-)
I realize that the 'Error: there was a bug, aborting' style of sado-masochistic error messages are the current Linux tooling status quo, which opaque error feedback comes from an early technological mistake of Unix system calls screwing up error handling, and I also see that after decades of abuse people are showing signs of the Stockholm Syndrome related to this problem, but it _really_ does not have to be so ...
Whenever we can we should change such bad patterns.
Quoted 1 line
The thing is, I hate that hard-coded list, its pain I don't need.
Absolutely! I pointed this out during review as well.
It does not impact the core concept though: we should have a single numeric error, and free form error strings provided by the place that first triggers some problem. That should be both programmatically easy to handle and maximally informative to the users.
At least half of a tool's usability comes not from how it behaves when it works, but how it behaves when it does not. (SystemD, I'm looking at you.)
Thanks,
Ingo
Hi,
Quoted 35 lines · Ingo Molnar
Quoted 6 lines · Peter Zijlstra
Quoted 2 lines · Ingo Molnar
perf record error: The 'bts' PMU is not available, because the CPU does not support it
This one makes sense.
Quoted 8 lines
Quoted 6 lines
perf record error: The 'bts' PMU is not available, because this architecture does not support it perf record error: The 'bts' PMU is not available, because its driver is not built into the kernel
Because if it's the wrong architecture or CPU, I look for a box with the right one, if it's simply the kernel not having the necessary PMU driver then I'll boot a kernel with it enabled.
These not so much; why won't a generic: "Unknown PMU, check
arch/kernel" do?Yeah, I mean why not make the user's job harder if we can? We really don't want to solve this problem technically and we _really_ want tooling to be fundamentally unhelpful, right? ;-)
I realize that the 'Error: there was a bug, aborting' style of sado-masochistic error messages are the current Linux tooling status quo, which opaque error feedback comes from an early technological mistake of Unix system calls screwing up error handling, and I also see that after decades of abuse people are showing signs of the Stockholm Syndrome related to this problem, but it _really_ does not have to be so ...
Whenever we can we should change such bad patterns.
Quoted 1 line
The thing is, I hate that hard-coded list, its pain I don't need.
Absolutely! I pointed this out during review as well.
Quoted 8 lines
It does not impact the core concept though: we should have a single numeric error, and free form error strings provided by the place that first triggers some problem. That should be both programmatically easy to handle and maximally informative to the users.
At least half of a tool's usability comes not from how it behaves when it works, but how it behaves when it does not. (SystemD, I'm looking at you.)
This patch looks useful but it does not address a related issue. Here you are reporting
on the status of specific PMU support, i.e., PMU is not supported by hardware. But there is another problem which I ran into on ARM very often (like on Tegra) and it
really annoys me. The PMU hardware is present, but the instance of the PMU on a CPU
is not present, simply because the CPU is hotpluggable and its offline at the time the
tool (perf) starts. I am not talking about explicit hotplugging by the user but instead be
the kernel. Then during the run, the CPU is plugged back in by the kernel to handle the
load. Perf misses monitoring that CPU completely, thus it does not measure what's going
on in reality.
I understand that reporting that a PMU instance is supported but offline does not
solve the entire problem. There needs to be some other kernel support. But I think
it would be good to have the tool at least issue a warning saying:
"some CPUs are
offline, not monitoring all CPUs, results may be partial".
Adrian Hunter
reviewerQuoted 23 lines · Peter Zijlstra
Quoted 3 lines · Adrian Hunter
Known PMUs may not be present for various reasons. Provide a way for the user to know what the reason is.
Not a bad idea, but I do wonder where we should draw the line on what is 'known'. The patch as proposed will have bts/pt listed as 'known' for every arch out there.
By that logic, x86 should list the ppc/sparc/mips/arm/etc.. PMUs as known and
wrong_archtoo, which might be a tad excessive.Can we limit it to PMUs for which we've (attempted to) load the drivers? That would obviously make a few of your status bits redundant, but then you've not explained why we're interested in it.
Quoted 6 lines
Supported
Driver error
Driver not loaded
Not supported by hardware
Wrong vendor
Unknown status
Quoted 8 lines
There would work.
Quoted 3 lines
Driver not in kernel config
Not supported by kernel
Wrong architectureThese will be hard, for if we don't load the driver we don't 'know' of them.
Are they that hard?
The architecture one is done by perf core at the moment when the initial status is defined e.g.
#if defined(CONFIG_X86)
#define PERF_PMU_STATUS_ARCH_X86 PERF_PMU_STATUS_UNKNOWN
#else
#define PERF_PMU_STATUS_ARCH_X86 PERF_PMU_STATUS_WRONG_ARCH
#endif
static struct known_pmu known_pmus[] = {
KNOWN_PMU("intel_pt", "Intel", PERF_PMU_STATUS_ARCH_X86),
KNOWN_PMU("intel_bts", "Intel", PERF_PMU_STATUS_ARCH_X86),
KNOWN_PMU(NULL, NULL, 0),
};For config and module the arch perf init code can check e.g.
#if !defined(CONFIG_MY_PMU) && !defined(CONFIG_MY_PMU_MODULE)
perf_pmu_update_status("my_pmu", PERF_PMU_STATUS_NOT_CONFIG, NULL);
#elif defined(CONFIG_MY_PMU_MODULE)
perf_pmu_update_status("my_pmu", PERF_PMU_STATUS_NOT_LOADED, NULL);#endif
Peter Zijlstra
maintainerQuoted 6 lines · Adrian Hunter
Quoted 3 lines · Peter Zijlstra
Can we limit it to PMUs for which we've (attempted to) load the drivers? That would obviously make a few of your status bits redundant, but then you've not explained why we're interested in it.
Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
Driver not in kernel config
Not supported by kernel
Wrong architectureThese will be hard, for if we don't load the driver we don't 'know' of them.
Are they that hard?
Because if we limit known to be what we're tried to probe, you simply do not know about PMUs for the wrong arch or not build by the kernel etc.
Quoted 5 lines
static struct known_pmu known_pmus[] = {
KNOWN_PMU("intel_pt", "Intel", PERF_PMU_STATUS_ARCH_X86),
KNOWN_PMU("intel_bts", "Intel", PERF_PMU_STATUS_ARCH_X86),
KNOWN_PMU(NULL, NULL, 0),
};So I really don't like this hard-coded table much, that's just going to be a pain to maintain.