perf/core: Change the layout of perf_sample_data
Subsystems (2)
Namhyung Kim
maintainerPATCH 1/3 perf/core: Change the layout of perf_sample_data
The layout of perf_sample_data is designed to minimize cache-line access. The perf_sample_data_init() used to initialize a couple of fields unconditionally so they were placed together at the head.
But it's changed now to set the fields according to the actual sample_type flags. The main user (the perf tools) sets the IP, TID, TIME, PERIOD always. Also group relevant fields like addr, phys_addr and data_page_size.
--- include/linux/perf_event.h | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-)
include/linux/perf_event.h +19 -15
@@ -1098,47 +1098,51 @@ extern u64 perf_event_read_value(struct perf_event *event,struct perf_sample_data {/* -* Fields set by perf_sample_data_init(), group so as to -* minimize the cachelines touched. +* Fields set by perf_sample_data_init() unconditionally, +* group so as to minimize the cachelines touched. */ u64 sample_flags;u64 period;/* -* The other fields, optionally {set,used} by -* perf_{prepare,output}_sample(). +* Fields commonly set by __perf_event_header__init_id(), +* group so as to minimize the cachelines touched. */ -struct perf_branch_stack *br_stack; -union perf_sample_weight weight; -union perf_mem_data_src data_src; -u64 txn; -u64 addr; -struct perf_raw_record *raw; -u64 type; -u64 ip;struct {u32 pid;u32 tid; } tid_entry;u64 time;u64 id; -u64 stream_id;struct {u32 cpu;u32 reserved; } cpu_entry; + +/* +* The other fields, optionally {set,used} by +* perf_{prepare,output}_sample(). +*/ +u64 ip;struct perf_callchain_entry *callchain; -u64 aux_size; +struct perf_raw_record *raw; +struct perf_branch_stack *br_stack; +union perf_sample_weight weight; +union perf_mem_data_src data_src; +u64 txn;struct perf_regs regs_user;struct perf_regs regs_intr;u64 stack_user_size; -u64 phys_addr; +u64 stream_id;u64 cgroup; +u64 addr; +u64 phys_addr;u64 data_page_size;u64 code_page_size; +u64 aux_size; }____cacheline_aligned ;/* default value for data source */
Signature · 1 line
2.39.0.314.g84b9a713c41-goog
Namhyung Kim
maintainerPATCH 2/3 perf/core: Set data->sample_flags in perf_prepare_sample()
The perf_prepare_sample() sets the perf_sample_data according to the attr->sample_type before copying it to the ring buffer. But BPF also wants to access the sample data so it needs to prepare the sample even before the regular path.
That means the perf_prepare_sample() can be called more than once. Set the data->sample_flags consistently so that it can indicate which fields are set already and skip them if sets.
Mostly it's just a matter of checking filtered_sample_type which is a bitmask for unset bits in the attr->sample_type. But some of sample data is implied by others even if it's not in the attr->sample_type (like PERF_SAMPLE_ADDR for PERF_SAMPLE_PHYS_ADDR). So they need to check data->sample_flags separately.
Also some of them like callchain, user regs/stack and aux data require more calculations. Protect them using the data->sample_flags to avoid the duplicate work.
---
Maybe we don't need this change to prevent duplication in favor of the next patch using the data->saved_size. But I think it's still useful to set data->sample_flags consistently. Anyway it's up to you.
kernel/events/core.c | 86 ++++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 23 deletions(-)
kernel/events/core.c +63 -23
@@ -7582,14 +7582,21 @@ void perf_prepare_sample(struct perf_event_header *header, filtered_sample_type = sample_type & ~data->sample_flags;__perf_event_header__init_id (header, data, event, filtered_sample_type); -if (sample_type & (PERF_SAMPLE_IP |PERF_SAMPLE_CODE_PAGE_SIZE )) - data->ip =perf_instruction_pointer (regs); +if (sample_type & (PERF_SAMPLE_IP |PERF_SAMPLE_CODE_PAGE_SIZE )) { +/* attr.sample_type may not have PERF_SAMPLE_IP */ +if (!(data->sample_flags &PERF_SAMPLE_IP )) { + data->ip =perf_instruction_pointer (regs); + data->sample_flags |=PERF_SAMPLE_IP ; + } + }if (sample_type &PERF_SAMPLE_CALLCHAIN ) {int size =1 ; -if (filtered_sample_type &PERF_SAMPLE_CALLCHAIN ) +if (filtered_sample_type &PERF_SAMPLE_CALLCHAIN ) { data->callchain =perf_callchain (event, regs); + data->sample_flags |=PERF_SAMPLE_CALLCHAIN ; + } size += data->callchain->nr;
@@ -7634,8 +7641,13 @@ void perf_prepare_sample(struct perf_event_header *header, header->size += size; } -if (sample_type & (PERF_SAMPLE_REGS_USER |PERF_SAMPLE_STACK_USER )) -perf_sample_regs_user (&data->regs_user, regs); +if (sample_type & (PERF_SAMPLE_REGS_USER |PERF_SAMPLE_STACK_USER )) { +/* attr.sample_type may not have PERF_SAMPLE_REGS_USER */ +if (!(data->sample_flags &PERF_SAMPLE_REGS_USER )) { +perf_sample_regs_user (&data->regs_user, regs); + data->sample_flags |=PERF_SAMPLE_REGS_USER ; + } + }if (sample_type &PERF_SAMPLE_REGS_USER ) {/* regs dump ABI info */
@@ -7656,11 +7668,18 @@ void perf_prepare_sample(struct perf_event_header *header,* in case new sample type is added, because we could eat * up the rest of the sample size. */ -u16 stack_size = event->attr.sample_stack_user;u16 size =sizeof (u64); +u16 stack_size; + +if (filtered_sample_type &PERF_SAMPLE_STACK_USER ) { + stack_size = event->attr.sample_stack_user; + stack_size =perf_sample_ustack_size (stack_size, header->size, + data->regs_user.regs); - stack_size =perf_sample_ustack_size (stack_size, header->size, - data->regs_user.regs); + data->stack_user_size = stack_size; + data->sample_flags |=PERF_SAMPLE_STACK_USER ; + } + stack_size = data->stack_user_size;/* * If there is something to dump, add space for the dump
@@ -7670,29 +7689,40 @@ void perf_prepare_sample(struct perf_event_header *header,if (stack_size) size +=sizeof (u64) + stack_size; - data->stack_user_size = stack_size; header->size += size; } -if (filtered_sample_type &PERF_SAMPLE_WEIGHT_TYPE ) +if (filtered_sample_type &PERF_SAMPLE_WEIGHT_TYPE ) { data->weight.full =0 ; + data->sample_flags |=PERF_SAMPLE_WEIGHT_TYPE ; + } -if (filtered_sample_type &PERF_SAMPLE_DATA_SRC ) +if (filtered_sample_type &PERF_SAMPLE_DATA_SRC ) { data->data_src.val =PERF_MEM_NA ; + data->sample_flags |=PERF_SAMPLE_DATA_SRC ; + } -if (filtered_sample_type &PERF_SAMPLE_TRANSACTION ) +if (filtered_sample_type &PERF_SAMPLE_TRANSACTION ) { data->txn =0 ; + data->sample_flags |=PERF_SAMPLE_TRANSACTION ; + }if (sample_type & (PERF_SAMPLE_ADDR |PERF_SAMPLE_PHYS_ADDR |PERF_SAMPLE_DATA_PAGE_SIZE )) { -if (filtered_sample_type &PERF_SAMPLE_ADDR ) +/* attr.sample_type may not have PERF_SAMPLE_ADDR */ +if (!(data->sample_flags &PERF_SAMPLE_ADDR )) { data->addr =0 ; + data->sample_flags |=PERF_SAMPLE_ADDR ; + } }if (sample_type &PERF_SAMPLE_REGS_INTR ) {/* regs dump ABI info */ int size =sizeof (u64); -perf_sample_regs_intr (&data->regs_intr, regs); +if (filtered_sample_type &PERF_SAMPLE_REGS_INTR ) { +perf_sample_regs_intr (&data->regs_intr, regs); + data->sample_flags |=PERF_SAMPLE_REGS_INTR ; + }if (data->regs_intr.regs) {u64 mask = event->attr.sample_regs_intr;
@@ -7703,17 +7733,19 @@ void perf_prepare_sample(struct perf_event_header *header, header->size += size; } -if (sample_type &PERF_SAMPLE_PHYS_ADDR && - filtered_sample_type &PERF_SAMPLE_PHYS_ADDR ) +if (filtered_sample_type &PERF_SAMPLE_PHYS_ADDR ) { data->phys_addr =perf_virt_to_phys (data->addr); + data->sample_flags |=PERF_SAMPLE_PHYS_ADDR ; + }#ifdef CONFIG_CGROUP_PERF -if (sample_type &PERF_SAMPLE_CGROUP ) { +if (filtered_sample_type &PERF_SAMPLE_CGROUP ) {struct cgroup *cgrp;/* protected by RCU */ cgrp =task_css_check (current, perf_event_cgrp_id,1 )->cgroup; data->cgroup =cgroup_id (cgrp); + data->sample_flags |=PERF_SAMPLE_CGROUP ; }#endif
@@ -7722,11 +7754,15 @@ void perf_prepare_sample(struct perf_event_header *header,* require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr, * but the value will not dump to the userspace. */ -if (sample_type &PERF_SAMPLE_DATA_PAGE_SIZE ) +if (filtered_sample_type &PERF_SAMPLE_DATA_PAGE_SIZE ) { data->data_page_size =perf_get_page_size (data->addr); + data->sample_flags |=PERF_SAMPLE_DATA_PAGE_SIZE ; + } -if (sample_type &PERF_SAMPLE_CODE_PAGE_SIZE ) +if (filtered_sample_type &PERF_SAMPLE_CODE_PAGE_SIZE ) { data->code_page_size =perf_get_page_size (data->ip); + data->sample_flags |=PERF_SAMPLE_CODE_PAGE_SIZE ; + }if (sample_type &PERF_SAMPLE_AUX ) {u64 size;
@@ -7739,10 +7775,14 @@ void perf_prepare_sample(struct perf_event_header *header,* Make sure this doesn't happen by using up to U16_MAX bytes * per sample in total (rounded down to 8 byte boundary). */ - size =min_t (size_t,U16_MAX - header->size, - event->attr.aux_sample_size); - size =rounddown (size,8 ); - size =perf_prepare_sample_aux (event, data, size); +if (filtered_sample_type &PERF_SAMPLE_AUX ) { + size =min_t (size_t,U16_MAX - header->size, + event->attr.aux_sample_size); + size =rounddown (size,8 ); +perf_prepare_sample_aux (event, data, size); + data->sample_flags |=PERF_SAMPLE_AUX ; + } + size = data->aux_size;WARN_ON_ONCE (size + header->size >U16_MAX ); header->size += size;
Signature · 1 line
2.39.0.314.g84b9a713c41-goog
Peter Zijlstra
maintainerRe: [PATCH 2/3] perf/core: Set data->sample_flags in perf_prepare_sample()
On Thu, Dec 29, 2022 at 12:41:00PM -0800, Namhyung Kim wrote:
So I like the general idea; I just think it's turned into a bit of a mess. That is code is already overly branchy which is known to hurt performance, we should really try and not make it worse than absolutely needed.
Quoted 8 lines
kernel/events/core.c | 86 ++++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 23 deletions(-)kernel/events/core.c +0
-0
kernel/events/core.c void perf_prepare_sample(struct perf_event_header *header, · line 7586
Quoted 7 lines
kernel/events/core.c +2
-2@@ -7582,14 +7582,21 @@ void perf_prepare_sample(struct perf_event_header *header, filtered_sample_type = sample_type & ~data->sample_flags;__perf_event_header__init_id (header, data, event, filtered_sample_type); -if (sample_type & (PERF_SAMPLE_IP |PERF_SAMPLE_CODE_PAGE_SIZE )) - data->ip =perf_instruction_pointer (regs); +if (sample_type & (PERF_SAMPLE_IP |PERF_SAMPLE_CODE_PAGE_SIZE )) { +/* attr.sample_type may not have PERF_SAMPLE_IP */
Right, but that shouldn't matter, IIRC its OK to have more bits set in data->sample_flags than we have set in attr.sample_type. It just means we have data available for sample types we're (possibly) not using.
That is, I think you can simply write this like:
kernel/events/core.c void perf_prepare_sample(struct perf_event_header *header, · line 7591
Quoted 5 lines
kernel/events/core.c +5
-0+if (!(data->sample_flags &PERF_SAMPLE_IP )) { + data->ip =perf_instruction_pointer (regs); + data->sample_flags |=PERF_SAMPLE_IP ; + } + }
if (filtered_sample_type & (PERF_SAMPLE_IP | PERF_SAMPLE_CODE_PAGE_SIZE)) {
data->ip = perf_instruction_pointer(regs);
data->sample_flags |= PERF_SAMPLE_IP);
}
...
if (filtered_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) {
data->code_page_size = perf_get_page_size(data->ip);
data->sample_flags |= PERF_SAMPLE_CODE_PAGE_SIZE;
}Then after a single perf_prepare_sample() run we have:
pre | post
----------------------------------------
0 | 0
IP | IP
CODE_PAGE_SIZE | IP|CODE_PAGE_SIZE
IP|CODE_PAGE_SIZE | IP|CODE_PAGE_SIZE
So while data->sample_flags will have an extra bit set in the 3rd case, that will not affect perf_sample_outout() which only looks at data->type (== attr.sample_type).
And since data->sample_flags will have both bits set, a second run will filter out both and avoid the extra work (except doing that will mess up the branch predictors).
kernel/events/core.c void perf_prepare_sample(struct perf_event_header *header, · line 7602
Quoted 11 lines
kernel/events/core.c +3
-1if (sample_type &PERF_SAMPLE_CALLCHAIN ) {int size =1 ; -if (filtered_sample_type &PERF_SAMPLE_CALLCHAIN ) +if (filtered_sample_type &PERF_SAMPLE_CALLCHAIN ) { data->callchain =perf_callchain (event, regs); + data->sample_flags |=PERF_SAMPLE_CALLCHAIN ; + } size += data->callchain->nr;
This, why can't this be:
if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) {
data->callchain = perf_callchain(event, regs);
data->sample_flags |= PERF_SAMPLE_CALLCHAIN;
header->size += (1 + data->callchain->nr) * sizeof(u64);
}I suppose this is because perf_event_header lives on the stack of the overflow handler and all that isn't available / relevant for the BPF thing.
And we can't pull that out into anther function without adding yet another branch fest.
However; inspired by your next patch; we can do something like so:
if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) {
data->callchain = perf_callchain(event, regs);
data->sample_flags |= PERF_SAMPLE_CALLCHAIN;
data->size += (1 + data->callchain->nr) * sizeof(u64);
}And then have __perf_event_output() (or something thereabout) do:
perf_prepare_sample(data, event, regs); perf_prepare_header(&header, data, event); err = output_begin(&handle, data, event, header.size); if (err) goto exit; perf_output_sample(&handle, &header, data, event); perf_output_end(&handle);
With perf_prepare_header() being something like:
header->type = PERF_RECORD_SAMPLE; header->size = sizeof(*header) + event->header_size + data->size; header->misc = perf_misc_flags(regs); ...
Hmm ?
(same for all the other sites)
Namhyung Kim
maintainerRe: [PATCH 2/3] perf/core: Set data->sample_flags in perf_prepare_sample()
Hi Peter,
Quoted 6 lines · Peter Zijlstra
On Thu, Dec 29, 2022 at 12:41:00PM -0800, Namhyung Kim wrote:
So I like the general idea; I just think it's turned into a bit of a mess. That is code is already overly branchy which is known to hurt performance, we should really try and not make it worse than absolutely needed.
Agreed.
Quoted 50 lines
Quoted 9 lines
kernel/events/core.c | 86 ++++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 23 deletions(-)kernel/events/core.c +0
-0kernel/events/core.c void perf_prepare_sample(struct perf_event_header *header, · line 7586
Quoted 7 lines
kernel/events/core.c +2
-2@@ -7582,14 +7582,21 @@ void perf_prepare_sample(struct perf_event_header *header, filtered_sample_type = sample_type & ~data->sample_flags;__perf_event_header__init_id (header, data, event, filtered_sample_type); -if (sample_type & (PERF_SAMPLE_IP |PERF_SAMPLE_CODE_PAGE_SIZE )) - data->ip =perf_instruction_pointer (regs); +if (sample_type & (PERF_SAMPLE_IP |PERF_SAMPLE_CODE_PAGE_SIZE )) { +/* attr.sample_type may not have PERF_SAMPLE_IP */ Right, but that shouldn't matter, IIRC its OK to have more bits set in
data->sample_flagsthan we have set in attr.sample_type. It just means we have data available for sample types we're (possibly) not using.That is, I think you can simply write this like:
kernel/events/core.c void perf_prepare_sample(struct perf_event_header *header, · line 7591
Quoted 5 lines
kernel/events/core.c +5
-0+if (!(data->sample_flags &PERF_SAMPLE_IP )) { + data->ip =perf_instruction_pointer (regs); + data->sample_flags |=PERF_SAMPLE_IP ; + } + }if (filtered_sample_type & (PERF_SAMPLE_IP | PERF_SAMPLE_CODE_PAGE_SIZE)) { data->ip = perf_instruction_pointer(regs); data->sample_flags |= PERF_SAMPLE_IP); } ... if (filtered_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) { data->code_page_size = perf_get_page_size(data->ip); data->sample_flags |= PERF_SAMPLE_CODE_PAGE_SIZE; }Then after a single
perf_prepare_sample()run we have:pre | post
----------------------------------------0 | 0IP | IP
CODE_PAGE_SIZE| IP|CODE_PAGE_SIZE
IP|CODE_PAGE_SIZE | IP|CODE_PAGE_SIZE
Quoted 7 lines
So while
data->sample_flagswill have an extra bit set in the 3rd case, that will not affectperf_sample_outout()which only looks atdata->type(== attr.sample_type).And since
data->sample_flagswill have both bits set, a second run will filter out both and avoid the extra work (except doing that will mess up the branch predictors).
Yeah, it'd be better to check filtered_sample_type in the first place.
Btw, I was thinking about a hypothetical scenario that IP set by a PMU driver not from the regs. In this case, having CODE_PAGE_SIZE will overwrite the IP. I don't think we need to worry about that for now since PMU drivers updates the regs (using set_linear_ip). But it seems like a possible scenario for something like PEBS or IBS.
Quoted 23 lines
kernel/events/core.c void perf_prepare_sample(struct perf_event_header *header, · line 7602
Quoted 13 lines
kernel/events/core.c +3
-1if (sample_type &PERF_SAMPLE_CALLCHAIN ) {int size =1 ; -if (filtered_sample_type &PERF_SAMPLE_CALLCHAIN ) +if (filtered_sample_type &PERF_SAMPLE_CALLCHAIN ) { data->callchain =perf_callchain (event, regs); + data->sample_flags |=PERF_SAMPLE_CALLCHAIN ; + } size += data->callchain->nr;This, why can't this be:
if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) { data->callchain = perf_callchain(event, regs); data->sample_flags |= PERF_SAMPLE_CALLCHAIN; header->size += (1 + data->callchain->nr) * sizeof(u64); }
Quoted 3 lines
I suppose this is because
perf_event_headerlives on the stack of the overflow handler and all that isn't available / relevant for the BPF thing.
Right, it needs to calculate the data size for each sample data.
Quoted 4 lines
And we can't pull that out into anther function without adding yet another branch fest.
Quoted 8 lines
However; inspired by your next patch; we can do something like so:
if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) { data->callchain = perf_callchain(event, regs); data->sample_flags |= PERF_SAMPLE_CALLCHAIN; data->size += (1 + data->callchain->nr) * sizeof(u64); }
This is fine as long as all other places (like in PMU drivers) set the callchain update the sample data size accordingly. If not, we can get the callchain but the data size will be wrong.
Quoted 13 lines
And then have
__perf_event_output()(or something thereabout) do:perf_prepare_sample(data, event, regs); perf_prepare_header(&header, data, event); err = output_begin(&handle, data, event, header.size); if (err) goto exit; perf_output_sample(&handle, &header, data, event); perf_output_end(&handle);With
perf_prepare_header()being something like:
Quoted 8 lines
header->type = PERF_RECORD_SAMPLE; header->size = sizeof(*header) + event->header_size + data->size; header->misc = perf_misc_flags(regs); ...Hmm ?
(same for all the other sites)
Looks good. But I'm confused by the tip-bot2 messages saying it's merged. Do you want me to work on it as a follow up?
Thanks,
Namhyung
Peter Zijlstra
maintainerRe: [PATCH 2/3] perf/core: Set data->sample_flags in perf_prepare_sample()
Quoted 2 lines · Namhyung Kim
Looks good. But I'm confused by the tip-bot2 messages saying it's merged. Do you want me to work on it as a follow up?
Ingo and me talked past one another, I agreed with 1/3 and he applied the whole series. Just talked to him again and he's just zapped these last two patches.
Sorry for the confusion.
Re: [PATCH 2/3] perf/core: Set data->sample_flags in perf_prepare_sample()
Quoted 8 lines · Peter Zijlstra
Quoted 2 lines · Namhyung Kim
Looks good. But I'm confused by the tip-bot2 messages saying it's merged. Do you want me to work on it as a follow up?
Ingo and me talked past one another, I agreed with 1/3 and he applied the whole series. Just talked to him again and he's just zapped these last two patches.
Yeah - perf/core is now:
7bdb1767bf01 ("perf/core: Change the layout of perf_sample_data")
which can be used for further work. Sorry about the confusion ...
Thanks,
Ingo
Namhyung Kim
maintainerRe: [PATCH 2/3] perf/core: Set data->sample_flags in perf_prepare_sample()
Hi Ingo,
Quoted 13 lines · Ingo Molnar
Quoted 8 lines · Peter Zijlstra
Quoted 2 lines · Namhyung Kim
Looks good. But I'm confused by the tip-bot2 messages saying it's merged. Do you want me to work on it as a follow up?
Ingo and me talked past one another, I agreed with 1/3 and he applied the whole series. Just talked to him again and he's just zapped these last two patches.
Quoted 5 lines
Yeah - perf/core is now:
7bdb1767bf01("perf/core: Change the layout ofperf_sample_data")which can be used for further work. Sorry about the confusion ...
No problem, thanks for your work!
Thanks,
Namhyung
Peter Zijlstra
maintainerRe: [PATCH 2/3] perf/core: Set data->sample_flags in perf_prepare_sample()
In reply to Namhyung KimQuoted 6 lines · Namhyung Kim
Quoted 6 lines
However; inspired by your next patch; we can do something like so:
if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) { data->callchain = perf_callchain(event, regs); data->sample_flags |= PERF_SAMPLE_CALLCHAIN;
Quoted 6 lines
Quoted 2 lines
data->size += (1 + data->callchain->nr) * sizeof(u64); }This is fine as long as all other places (like in PMU drivers) set the callchain update the sample data size accordingly. If not, we can get the callchain but the data size will be wrong.
Good point, maybe add a helper there to ensure that code doesn't duplicate/diverge?
Namhyung Kim
maintainerRe: [PATCH 2/3] perf/core: Set data->sample_flags in perf_prepare_sample()
Quoted 12 lines · Peter Zijlstra
Quoted 9 lines · Namhyung Kim
Quoted 8 lines
However; inspired by your next patch; we can do something like so:
if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) { data->callchain = perf_callchain(event, regs); data->sample_flags |= PERF_SAMPLE_CALLCHAIN; data->size += (1 + data->callchain->nr) * sizeof(u64); }
Quoted 6 lines
Quoted 3 lines
This is fine as long as all other places (like in PMU drivers) set the callchain update the sample data size accordingly. If not, we can get the callchain but the data size will be wrong.
Good point, maybe add a helper there to ensure that code doesn't duplicate/diverge?
Sure, will do.
Thanks,
Namhyung
Namhyung Kim
maintainerRe: [PATCH 2/3] perf/core: Set data->sample_flags in perf_prepare_sample()
In reply to Namhyung KimQuoted 71 lines · Namhyung Kim
Hi Peter,
Quoted 6 lines · Peter Zijlstra
On Thu, Dec 29, 2022 at 12:41:00PM -0800, Namhyung Kim wrote:
So I like the general idea; I just think it's turned into a bit of a mess. That is code is already overly branchy which is known to hurt performance, we should really try and not make it worse than absolutely needed.
Agreed.
Quoted 50 lines
Quoted 9 lines
kernel/events/core.c | 86 ++++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 23 deletions(-)kernel/events/core.c +0
-0kernel/events/core.c void perf_prepare_sample(struct perf_event_header *header, · line 7586
Quoted 7 lines
kernel/events/core.c +2
-2@@ -7582,14 +7582,21 @@ void perf_prepare_sample(struct perf_event_header *header, filtered_sample_type = sample_type & ~data->sample_flags;__perf_event_header__init_id (header, data, event, filtered_sample_type); -if (sample_type & (PERF_SAMPLE_IP |PERF_SAMPLE_CODE_PAGE_SIZE )) - data->ip =perf_instruction_pointer (regs); +if (sample_type & (PERF_SAMPLE_IP |PERF_SAMPLE_CODE_PAGE_SIZE )) { +/* attr.sample_type may not have PERF_SAMPLE_IP */ Right, but that shouldn't matter, IIRC its OK to have more bits set in
data->sample_flagsthan we have set in attr.sample_type. It just means we have data available for sample types we're (possibly) not using.That is, I think you can simply write this like:
kernel/events/core.c void perf_prepare_sample(struct perf_event_header *header, · line 7591
Quoted 5 lines
kernel/events/core.c +5
-0+if (!(data->sample_flags &PERF_SAMPLE_IP )) { + data->ip =perf_instruction_pointer (regs); + data->sample_flags |=PERF_SAMPLE_IP ; + } + }if (filtered_sample_type & (PERF_SAMPLE_IP | PERF_SAMPLE_CODE_PAGE_SIZE)) { data->ip = perf_instruction_pointer(regs); data->sample_flags |= PERF_SAMPLE_IP); } ... if (filtered_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) { data->code_page_size = perf_get_page_size(data->ip); data->sample_flags |= PERF_SAMPLE_CODE_PAGE_SIZE; }Then after a single
perf_prepare_sample()run we have:pre | post
----------------------------------------0 | 0 IP | IP CODE_PAGE_SIZE | IP|CODE_PAGE_SIZE IP|CODE_PAGE_SIZE | IP|CODE_PAGE_SIZEQuoted 7 lines
So while
data->sample_flagswill have an extra bit set in the 3rd case, that will not affectperf_sample_outout()which only looks atdata->type(== attr.sample_type).And since
data->sample_flagswill have both bits set, a second run will filter out both and avoid the extra work (except doing that will mess up the branch predictors).
Quoted 7 lines
Yeah, it'd be better to check
filtered_sample_typein the first place.Btw, I was thinking about a hypothetical scenario that IP set by a PMU driver not from the regs. In this case, having
CODE_PAGE_SIZEwill overwrite the IP. I don't think we need to worry about that for now since PMU drivers updates the regs (usingset_linear_ip). But it seems like a possible scenario for something like PEBS or IBS.
Another example, but in this case it's real, is ADDR. We cannot update the data->addr just because filtered_sample_type has PHYS_ADDR or DATA_PAGE_SIZE as it'd lose the original value.
Other than that, I'll update the other paths to minimized the branches.
Thanks,
Namhyung
Peter Zijlstra
maintainerRe: [PATCH 2/3] perf/core: Set data->sample_flags in perf_prepare_sample()
Quoted 3 lines · Namhyung Kim
Another example, but in this case it's real, is ADDR. We cannot update the
data->addrjust becausefiltered_sample_typehasPHYS_ADDRorDATA_PAGE_SIZEas it'd lose the original value.
Hmm, how about something like so?
/*
* if (flags & s) flags |= d; // without branches
*/
static __always_inline unsigned long __cond_set(unsigned long flags, unsigned long s, unsigned long d)
{
return flags | (d * !!(flags & s));
}Then:
fst = sample_type; fst = __cond_set(fst, PERF_SAMPLE_CODE_PAGE_SIZE, PERF_SAMPLE_IP); fst = __cond_set(fst, PERF_SAMPLE_DATA_PAGE_SIZE | PERF_SAMPLE_PHYS_ADDR, PERF_SAMPLE_ADDR); fst = __cond_set(fst, PERF_SAMPLE_STACK_USER, PERF_SAMPLE_REGS_USER); fst &= ~data->sample_flags;
This way we express the implicit conditions by setting the required sample data flags, then we mask those we already have set.
After the above something like:
if (fst & PERF_SAMPLE_ADDR) {
data->addr = 0;
data->sample_flags |= PERF_SAMPLE_ADDR;
}
if (fst & PERF_SAMPLE_PHYS_ADDR) {
data->phys_addr = perf_virt_to_phys(data->addr);
data->sample_flags |= PERF_SAMPLE_PHYS_ADDR;
}
if (fst & PERF_SAMPLE_DATA_PAGE_SIZE) {
data->data_page_size = perf_get_page_size(data->addr);
data->sample_flags |= PERF_SAMPLE_DATA_PAGE_SIZE;
}And maybe something like:
#define __IF_SAMPLE_DATA(f_) ({ \
bool __f = fst & PERF_SAMPLE_##f_; \
if (__f) data->sample_flags |= PERF_SAMPLE_##f_;\
__f; })#define IF_SAMPLE_DATA(f_) if (__IF_SAMPLE_DATA(f_))
Then we can write:
IF_SAMPLE_DATA(ADDR) data->addr = 0; IF_SAMPLE_DATA(PHYS_ADDR) data->phys_addr = perf_virt_to_phys(data->addr); IF_SAMPLE_DATA(DATA_PAGE_SIZE) data->data_page_size = perf_get_page_size(data->addr);
But I didn't check code-gen for this last suggestion.
Peter Zijlstra
maintainerRe: [PATCH 2/3] perf/core: Set data->sample_flags in perf_prepare_sample()
Quoted 17 lines · Peter Zijlstra
Quoted 3 lines · Namhyung Kim
Another example, but in this case it's real, is ADDR. We cannot update the
data->addrjust becausefiltered_sample_typehasPHYS_ADDRorDATA_PAGE_SIZEas it'd lose the original value.Hmm, how about something like so?
/*
* if (flags & s) flags |= d; // without branches
*/
static__always_inlineunsigned long__cond_set(unsigned long flags, unsigned long s, unsigned long d){ return flags | (d * !!(flags & s)); }
Quoted 9 lines
Then:
fst = sample_type; fst = __cond_set(fst, PERF_SAMPLE_CODE_PAGE_SIZE, PERF_SAMPLE_IP); fst = __cond_set(fst, PERF_SAMPLE_DATA_PAGE_SIZE | PERF_SAMPLE_PHYS_ADDR, PERF_SAMPLE_ADDR); fst = __cond_set(fst, PERF_SAMPLE_STACK_USER, PERF_SAMPLE_REGS_USER); fst &= ~data->sample_flags;
Hmm, I think it's better to write this like:
static __always_inline unsigned long __cond_set(unsigned long flags, unsigned long s, unsigned long d)
{
return d * !!(flags & s);
}fst = sample_type; fst |= __cond_set(sample_type, PERF_SAMPLE_CODE_PAGE_SIZE, PERF_SAMPLE_IP); fst |= __cond_set(sample_type, PERF_SAMPLE_DATA_PAGE_SIZE | PERF_SAMPLE_PHYS_ADDR, PERF_SAMPLE_ADDR); fst |= __cond_set(sample_type, PERF_SAMPLE_STACK_USER, PERF_SAMPLE_REGS_USER); fst &= ~data->sample_flags;
Which should be identical but has less data dependencies and thus gives an OoO CPU more leaway to paralleize things.
Namhyung Kim
maintainerRe: [PATCH 2/3] perf/core: Set data->sample_flags in perf_prepare_sample()
Quoted 44 lines · Peter Zijlstra
Quoted 17 lines · Peter Zijlstra
Quoted 3 lines · Namhyung Kim
Another example, but in this case it's real, is ADDR. We cannot update the
data->addrjust becausefiltered_sample_typehasPHYS_ADDRorDATA_PAGE_SIZEas it'd lose the original value.Hmm, how about something like so?
/*
* if (flags & s) flags |= d; // without branches
*/
static__always_inlineunsigned long__cond_set(unsigned long flags, unsigned long s, unsigned long d){ return flags | (d * !!(flags & s)); }Quoted 9 lines
Then:
fst = sample_type; fst = __cond_set(fst, PERF_SAMPLE_CODE_PAGE_SIZE, PERF_SAMPLE_IP); fst = __cond_set(fst, PERF_SAMPLE_DATA_PAGE_SIZE | PERF_SAMPLE_PHYS_ADDR, PERF_SAMPLE_ADDR); fst = __cond_set(fst, PERF_SAMPLE_STACK_USER, PERF_SAMPLE_REGS_USER); fst &= ~data->sample_flags;Hmm, I think it's better to write this like:
static
__always_inlineunsigned long__cond_set(unsigned long flags, unsigned long s, unsigned long d){ return d * !!(flags & s); }fst = sample_type; fst |= __cond_set(sample_type, PERF_SAMPLE_CODE_PAGE_SIZE, PERF_SAMPLE_IP); fst |= __cond_set(sample_type, PERF_SAMPLE_DATA_PAGE_SIZE | PERF_SAMPLE_PHYS_ADDR, PERF_SAMPLE_ADDR); fst |= __cond_set(sample_type, PERF_SAMPLE_STACK_USER, PERF_SAMPLE_REGS_USER); fst &= ~data->sample_flags;
Quoted 2 lines
Which should be identical but has less data dependencies and thus gives an OoO CPU more leaway to paralleize things.
Looks good. Let me try this.
Thanks,
Namhyung
[tip: perf/core] perf/core: Set data->sample_flags in perf_prepare_sample()
In reply to Namhyung KimThe following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 965547f0a2f3cd0786d4faee2db008bb281d4922 Gitweb: https://git.kernel.org/tip/965547f0a2f3cd0786d4faee2db008bb281d4922 Author: Namhyung Kim <[email protected]> AuthorDate: Thu, 29 Dec 2022 12:41:00 -08:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Mon, 09 Jan 2023 12:22:09 +01:00
perf/core: Set data->sample_flags in perf_prepare_sample()
The perf_prepare_sample() sets the perf_sample_data according to the attr->sample_type before copying it to the ring buffer. But BPF also wants to access the sample data so it needs to prepare the sample even before the regular path.
That means the perf_prepare_sample() can be called more than once. Set the data->sample_flags consistently so that it can indicate which fields are set already and skip them if sets.
Mostly it's just a matter of checking filtered_sample_type which is a bitmask for unset bits in the attr->sample_type. But some of sample data is implied by others even if it's not in the attr->sample_type (like PERF_SAMPLE_ADDR for PERF_SAMPLE_PHYS_ADDR). So they need to check data->sample_flags separately.
Also some of them like callchain, user regs/stack and aux data require more calculations. Protect them using the data->sample_flags to avoid the duplicate work.
--- kernel/events/core.c | 86 +++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 23 deletions(-)
kernel/events/core.c +63 -23
@@ -7582,14 +7582,21 @@ void perf_prepare_sample(struct perf_event_header *header, filtered_sample_type = sample_type & ~data->sample_flags;__perf_event_header__init_id (header, data, event, filtered_sample_type); -if (sample_type & (PERF_SAMPLE_IP |PERF_SAMPLE_CODE_PAGE_SIZE )) - data->ip =perf_instruction_pointer (regs); +if (sample_type & (PERF_SAMPLE_IP |PERF_SAMPLE_CODE_PAGE_SIZE )) { +/* attr.sample_type may not have PERF_SAMPLE_IP */ +if (!(data->sample_flags &PERF_SAMPLE_IP )) { + data->ip =perf_instruction_pointer (regs); + data->sample_flags |=PERF_SAMPLE_IP ; + } + }if (sample_type &PERF_SAMPLE_CALLCHAIN ) {int size =1 ; -if (filtered_sample_type &PERF_SAMPLE_CALLCHAIN ) +if (filtered_sample_type &PERF_SAMPLE_CALLCHAIN ) { data->callchain =perf_callchain (event, regs); + data->sample_flags |=PERF_SAMPLE_CALLCHAIN ; + } size += data->callchain->nr;
@@ -7634,8 +7641,13 @@ void perf_prepare_sample(struct perf_event_header *header, header->size += size; } -if (sample_type & (PERF_SAMPLE_REGS_USER |PERF_SAMPLE_STACK_USER )) -perf_sample_regs_user (&data->regs_user, regs); +if (sample_type & (PERF_SAMPLE_REGS_USER |PERF_SAMPLE_STACK_USER )) { +/* attr.sample_type may not have PERF_SAMPLE_REGS_USER */ +if (!(data->sample_flags &PERF_SAMPLE_REGS_USER )) { +perf_sample_regs_user (&data->regs_user, regs); + data->sample_flags |=PERF_SAMPLE_REGS_USER ; + } + }if (sample_type &PERF_SAMPLE_REGS_USER ) {/* regs dump ABI info */
@@ -7656,11 +7668,18 @@ void perf_prepare_sample(struct perf_event_header *header,* in case new sample type is added, because we could eat * up the rest of the sample size. */ -u16 stack_size = event->attr.sample_stack_user;u16 size =sizeof (u64); +u16 stack_size; + +if (filtered_sample_type &PERF_SAMPLE_STACK_USER ) { + stack_size = event->attr.sample_stack_user; + stack_size =perf_sample_ustack_size (stack_size, header->size, + data->regs_user.regs); - stack_size =perf_sample_ustack_size (stack_size, header->size, - data->regs_user.regs); + data->stack_user_size = stack_size; + data->sample_flags |=PERF_SAMPLE_STACK_USER ; + } + stack_size = data->stack_user_size;/* * If there is something to dump, add space for the dump
@@ -7670,29 +7689,40 @@ void perf_prepare_sample(struct perf_event_header *header,if (stack_size) size +=sizeof (u64) + stack_size; - data->stack_user_size = stack_size; header->size += size; } -if (filtered_sample_type &PERF_SAMPLE_WEIGHT_TYPE ) +if (filtered_sample_type &PERF_SAMPLE_WEIGHT_TYPE ) { data->weight.full =0 ; + data->sample_flags |=PERF_SAMPLE_WEIGHT_TYPE ; + } -if (filtered_sample_type &PERF_SAMPLE_DATA_SRC ) +if (filtered_sample_type &PERF_SAMPLE_DATA_SRC ) { data->data_src.val =PERF_MEM_NA ; + data->sample_flags |=PERF_SAMPLE_DATA_SRC ; + } -if (filtered_sample_type &PERF_SAMPLE_TRANSACTION ) +if (filtered_sample_type &PERF_SAMPLE_TRANSACTION ) { data->txn =0 ; + data->sample_flags |=PERF_SAMPLE_TRANSACTION ; + }if (sample_type & (PERF_SAMPLE_ADDR |PERF_SAMPLE_PHYS_ADDR |PERF_SAMPLE_DATA_PAGE_SIZE )) { -if (filtered_sample_type &PERF_SAMPLE_ADDR ) +/* attr.sample_type may not have PERF_SAMPLE_ADDR */ +if (!(data->sample_flags &PERF_SAMPLE_ADDR )) { data->addr =0 ; + data->sample_flags |=PERF_SAMPLE_ADDR ; + } }if (sample_type &PERF_SAMPLE_REGS_INTR ) {/* regs dump ABI info */ int size =sizeof (u64); -perf_sample_regs_intr (&data->regs_intr, regs); +if (filtered_sample_type &PERF_SAMPLE_REGS_INTR ) { +perf_sample_regs_intr (&data->regs_intr, regs); + data->sample_flags |=PERF_SAMPLE_REGS_INTR ; + }if (data->regs_intr.regs) {u64 mask = event->attr.sample_regs_intr;
@@ -7703,17 +7733,19 @@ void perf_prepare_sample(struct perf_event_header *header, header->size += size; } -if (sample_type &PERF_SAMPLE_PHYS_ADDR && - filtered_sample_type &PERF_SAMPLE_PHYS_ADDR ) +if (filtered_sample_type &PERF_SAMPLE_PHYS_ADDR ) { data->phys_addr =perf_virt_to_phys (data->addr); + data->sample_flags |=PERF_SAMPLE_PHYS_ADDR ; + }#ifdef CONFIG_CGROUP_PERF -if (sample_type &PERF_SAMPLE_CGROUP ) { +if (filtered_sample_type &PERF_SAMPLE_CGROUP ) {struct cgroup *cgrp;/* protected by RCU */ cgrp =task_css_check (current, perf_event_cgrp_id,1 )->cgroup; data->cgroup =cgroup_id (cgrp); + data->sample_flags |=PERF_SAMPLE_CGROUP ; }#endif
@@ -7722,11 +7754,15 @@ void perf_prepare_sample(struct perf_event_header *header,* require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr, * but the value will not dump to the userspace. */ -if (sample_type &PERF_SAMPLE_DATA_PAGE_SIZE ) +if (filtered_sample_type &PERF_SAMPLE_DATA_PAGE_SIZE ) { data->data_page_size =perf_get_page_size (data->addr); + data->sample_flags |=PERF_SAMPLE_DATA_PAGE_SIZE ; + } -if (sample_type &PERF_SAMPLE_CODE_PAGE_SIZE ) +if (filtered_sample_type &PERF_SAMPLE_CODE_PAGE_SIZE ) { data->code_page_size =perf_get_page_size (data->ip); + data->sample_flags |=PERF_SAMPLE_CODE_PAGE_SIZE ; + }if (sample_type &PERF_SAMPLE_AUX ) {u64 size;
@@ -7739,10 +7775,14 @@ void perf_prepare_sample(struct perf_event_header *header,* Make sure this doesn't happen by using up to U16_MAX bytes * per sample in total (rounded down to 8 byte boundary). */ - size =min_t (size_t,U16_MAX - header->size, - event->attr.aux_sample_size); - size =rounddown (size,8 ); - size =perf_prepare_sample_aux (event, data, size); +if (filtered_sample_type &PERF_SAMPLE_AUX ) { + size =min_t (size_t,U16_MAX - header->size, + event->attr.aux_sample_size); + size =rounddown (size,8 ); +perf_prepare_sample_aux (event, data, size); + data->sample_flags |=PERF_SAMPLE_AUX ; + } + size = data->aux_size;WARN_ON_ONCE (size + header->size >U16_MAX ); header->size += size;
Namhyung Kim
maintainerPATCH 3/3 perf/core: Save calculated sample data size
In reply to Namhyung KimTo avoid duplicate work in perf_prepare_sample(), save the final header size in data->saved_size. It's initialized to 0 and set to an actual value at the end of perf_prepare_sample().
If it sees a non-zero value that means it's the second time of the call and it knows the sample data is populated already. So update the header size with the data->saved_size and bail out.
--- include/linux/perf_event.h | 2 ++ kernel/events/core.c | 11 +++++++++++ 2 files changed, 13 insertions(+)
include/linux/perf_event.h +2 -0
@@ -1103,6 +1103,7 @@ struct perf_sample_data {*/ u64 sample_flags;u64 period; +u64 saved_size;/* * Fields commonly set by __perf_event_header__init_id(),
@@ -1158,6 +1159,7 @@ static inline void perf_sample_data_init(struct perf_sample_data *data,/* remaining struct members initialized in perf_prepare_sample() */ data->sample_flags =PERF_SAMPLE_PERIOD ; data->period = period; + data->saved_size =0 ;if (addr) { data->addr = addr;
kernel/events/core.c +11 -0
@@ -7575,6 +7575,15 @@ void perf_prepare_sample(struct perf_event_header *header, header->misc =0 ; header->misc |=perf_misc_flags (regs); +/* +* If it called perf_prepare_sample() already, it set the all data fields +* and recorded the final size to data->saved_size. +*/ +if (data->saved_size) { + header->size = data->saved_size; +return ; + } +/* * Clear the sample flags that have already been done by the * PMU driver.
@@ -7796,6 +7805,8 @@ void perf_prepare_sample(struct perf_event_header *header,* do here next. */ WARN_ON_ONCE (header->size &7 ); + + data->saved_size = header->size; }static __always_inline int
Signature · 1 line
2.39.0.314.g84b9a713c41-goog
[tip: perf/core] perf/core: Save calculated sample data size
The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 0ecc518e5c1a83fbfc6262d20d0df289eafc2207 Gitweb: https://git.kernel.org/tip/0ecc518e5c1a83fbfc6262d20d0df289eafc2207 Author: Namhyung Kim <[email protected]> AuthorDate: Thu, 29 Dec 2022 12:41:01 -08:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Mon, 09 Jan 2023 12:22:09 +01:00
perf/core: Save calculated sample data size
To avoid duplicate work in perf_prepare_sample(), save the final header size in data->saved_size. It's initialized to 0 and set to an actual value at the end of perf_prepare_sample().
If it sees a non-zero value that means it's the second time of the call and it knows the sample data is populated already. So update the header size with the data->saved_size and bail out.
--- include/linux/perf_event.h | 2 ++ kernel/events/core.c | 11 +++++++++++ 2 files changed, 13 insertions(+)
include/linux/perf_event.h +2 -0
@@ -1103,6 +1103,7 @@ struct perf_sample_data {*/ u64 sample_flags;u64 period; +u64 saved_size;/* * Fields commonly set by __perf_event_header__init_id(),
@@ -1158,6 +1159,7 @@ static inline void perf_sample_data_init(struct perf_sample_data *data,/* remaining struct members initialized in perf_prepare_sample() */ data->sample_flags =PERF_SAMPLE_PERIOD ; data->period = period; + data->saved_size =0 ;if (addr) { data->addr = addr;
kernel/events/core.c +11 -0
@@ -7576,6 +7576,15 @@ void perf_prepare_sample(struct perf_event_header *header, header->misc |=perf_misc_flags (regs);/* +* If it called perf_prepare_sample() already, it set the all data fields +* and recorded the final size to data->saved_size. +*/ +if (data->saved_size) { + header->size = data->saved_size; +return ; + } + +/* * Clear the sample flags that have already been done by the * PMU driver. */
@@ -7796,6 +7805,8 @@ void perf_prepare_sample(struct perf_event_header *header,* do here next. */ WARN_ON_ONCE (header->size &7 ); + + data->saved_size = header->size; }static __always_inline int
The following commit has been merged into the perf/core branch of tip:
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 7bdb1767bf011c7f6065ac483ad2f00e434c3979 Gitweb: https://git.kernel.org/tip/7bdb1767bf011c7f6065ac483ad2f00e434c3979 Author: Namhyung Kim <[email protected]> AuthorDate: Thu, 29 Dec 2022 12:40:59 -08:00 Committer: Ingo Molnar <[email protected]> CommitterDate: Mon, 09 Jan 2023 12:22:09 +01:00
perf/core: Change the layout of perf_sample_data
The layout of perf_sample_data is designed to minimize cache-line access. The perf_sample_data_init() used to initialize a couple of fields unconditionally so they were placed together at the head.
But it's changed now to set the fields according to the actual sample_type flags. The main user (the perf tools) sets the IP, TID, TIME, PERIOD always. Also group relevant fields like addr, phys_addr and data_page_size.
--- include/linux/perf_event.h | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-)
include/linux/perf_event.h +19 -15
@@ -1098,47 +1098,51 @@ extern u64 perf_event_read_value(struct perf_event *event,struct perf_sample_data {/* -* Fields set by perf_sample_data_init(), group so as to -* minimize the cachelines touched. +* Fields set by perf_sample_data_init() unconditionally, +* group so as to minimize the cachelines touched. */ u64 sample_flags;u64 period;/* -* The other fields, optionally {set,used} by -* perf_{prepare,output}_sample(). +* Fields commonly set by __perf_event_header__init_id(), +* group so as to minimize the cachelines touched. */ -struct perf_branch_stack *br_stack; -union perf_sample_weight weight; -union perf_mem_data_src data_src; -u64 txn; -u64 addr; -struct perf_raw_record *raw; -u64 type; -u64 ip;struct {u32 pid;u32 tid; } tid_entry;u64 time;u64 id; -u64 stream_id;struct {u32 cpu;u32 reserved; } cpu_entry; + +/* +* The other fields, optionally {set,used} by +* perf_{prepare,output}_sample(). +*/ +u64 ip;struct perf_callchain_entry *callchain; -u64 aux_size; +struct perf_raw_record *raw; +struct perf_branch_stack *br_stack; +union perf_sample_weight weight; +union perf_mem_data_src data_src; +u64 txn;struct perf_regs regs_user;struct perf_regs regs_intr;u64 stack_user_size; -u64 phys_addr; +u64 stream_id;u64 cgroup; +u64 addr; +u64 phys_addr;u64 data_page_size;u64 code_page_size; +u64 aux_size; }____cacheline_aligned ;/* default value for data source */