Various x86 pmu scheduling patches
Subsystems (2)
Peter Zijlstra
maintainerv2 cover Various x86 pmu scheduling patches
The first two would be for perf/urgent, the rest is cleanups and can wait.
These patches have some testing and the new HT constraints code seems to actually work now.
It only constraints the general purpose counters when there are exclusive events present on either HT sibling, otherwise all GP counters are available as per usual.
Peter Zijlstra
maintainerPATCH v2 1/11 perf,x86: Fix event/group validation
Commit 43b4578071c0 ("perf/x86: Reduce stack usage of x86_schedule_events()") violated the rule that 'fake' scheduling; as used for event/group validation; should not change the event state.
This went mostly un-noticed because repeated calls of x86_pmu::get_event_constraints() would give the same result. And x86_pmu::put_event_constraints() would mostly not do anything.
Commit e979121b1b15 ("perf/x86/intel: Implement cross-HT corruption bug workaround") made the situation much worse by actually setting the event->hw.constraint value to NULL, so when validation and actual scheduling interact we get NULL ptr derefs.
Fix it by removing the constraint pointer from the event and move it back to an array, this time in cpuc instead of on the stack.
validate_group()
x86_schedule_events()
event->hw.constraint = c; # store
<context switch>
perf_task_event_sched_in()
...
x86_schedule_events();
event->hw.constraint = c2; # store
...
put_event_constraints(event); # assume failure to schedule
intel_put_event_constraints()
event->hw.constraint = NULL;
<context switch end>
c = event->hw.constraint; # read -> NULL
if (!test_bit(hwc->idx, c->idxmsk)) # <- *BOOM* NULL derefThis in particular is possible when the event in question is a cpu-wide event and group-leader, where the validate_group() tries to add an event to the group.
--- arch/x86/kernel/cpu/perf_event.c | 33 ++++++++++++++------------ arch/x86/kernel/cpu/perf_event.h | 9 +++---- arch/x86/kernel/cpu/perf_event_intel.c | 15 +++-------- arch/x86/kernel/cpu/perf_event_intel_ds.c | 4 +-- arch/x86/kernel/cpu/perf_event_intel_uncore.c | 7 ++--- arch/x86/kernel/cpu/perf_event_intel_uncore.h | 1 6 files changed, 33 insertions(+), 36 deletions(-)
arch/x86/kernel/cpu/perf_event.c +18 -15
@@ -620,7 +620,7 @@ struct sched_state {struct perf_sched {int max_weight;int max_events; -struct perf_event **events; +struct event_constraint **constraints;struct sched_state state;int saved_states;struct sched_state saved[SCHED_STATES_MAX ];
@@ -629,7 +629,7 @@ struct perf_sched {/* * Initialize interator that runs through all events and counters. */ -static void perf_sched_init (struct perf_sched *sched,struct perf_event **events, +static void perf_sched_init (struct perf_sched *sched,struct event_constraint **constraints,int num,int wmin,int wmax) {int idx;
@@ -637,10 +637,10 @@ static void perf_sched_init(struct perf_memset (sched,0 ,sizeof (*sched)); sched->max_events = num; sched->max_weight = wmax; - sched->events = events; + sched->constraints = constraints;for (idx =0 ; idx < num; idx++) { -if (events[idx]->hw.constraint->weight == wmin) +if (constraints[idx]->weight == wmin)break ; }
@@ -687,7 +687,7 @@ static bool __perf_sched_find_counter(stif (sched->state.event >= sched->max_events)return false; - c = sched->events[sched->state.event]->hw.constraint; + c = sched->constraints[sched->state.event];/* Prefer fixed purpose counters */ if (c->idxmsk64 & (~0ULL <<INTEL_PMC_IDX_FIXED )) { idx =INTEL_PMC_IDX_FIXED ;
@@ -745,7 +745,7 @@ static bool perf_sched_next_event(structif (sched->state.weight > sched->max_weight)return false; } - c = sched->events[sched->state.event]->hw.constraint; + c = sched->constraints[sched->state.event]; }while (c->weight != sched->state.weight); sched->state.counter =0 ;/* start with first counter */
@@ -756,12 +756,12 @@ static bool perf_sched_next_event(struct/* * Assign a counter for each event. */ -int perf_assign_events (struct perf_event **events,int n, +int perf_assign_events (struct event_constraint **constraints,int n,int wmin,int wmax,int *assign) {struct perf_sched sched; -perf_sched_init (&sched, events, n, wmin, wmax); +perf_sched_init (&sched, constraints, n, wmin, wmax); do { if (!perf_sched_find_counter(&sched))
@@ -788,9 +788,9 @@ int x86_schedule_events(struct cpu_hw_ev x86_pmu.start_scheduling (cpuc);for (i =0 , wmin =X86_PMC_IDX_MAX , wmax =0 ; i < n; i++) { - hwc = &cpuc->event_list[i]->hw; + cpuc->event_constraint[i] =NULL ; c = x86_pmu.get_event_constraints (cpuc, i, cpuc->event_list[i]); - hwc->constraint = c; + cpuc->event_constraint[i] = c; wmin =min (wmin, c->weight); wmax =max (wmax, c->weight);
@@ -801,7 +801,7 @@ int x86_schedule_events(struct cpu_hw_ev*/ for (i =0 ; i < n; i++) { hwc = &cpuc->event_list[i]->hw; - c = hwc->constraint; + c = cpuc->event_constraint[i];/* never assigned */ if (hwc->idx ==-1 )
@@ -821,9 +821,10 @@ int x86_schedule_events(struct cpu_hw_ev }/* slow path */ -if (i != n) - unsched =perf_assign_events (cpuc->event_list, n, wmin, +if (i != n) { + unsched =perf_assign_events (cpuc->event_constraint, n, wmin, wmax, assign); + }/* * In case of success (unsched = 0), mark events as committed,
@@ -840,7 +841,7 @@ int x86_schedule_events(struct cpu_hw_ev e = cpuc->event_list[i]; e->hw.flags |=PERF_X86_EVENT_COMMITTED ;if (x86_pmu.commit_scheduling) - x86_pmu.commit_scheduling (cpuc, e, assign[i]); + x86_pmu.commit_scheduling (cpuc, i, assign[i]); } }
@@ -1292,8 +1293,10 @@ static void x86_pmu_del(struct perf_even x86_pmu.put_event_constraints (cpuc, event);/* Delete the array entry. */ -while (++i < cpuc->n_events) +while (++i < cpuc->n_events) { cpuc->event_list[i-1 ] = cpuc->event_list[i]; + cpuc->event_constraint[i-1 ] = cpuc->event_constraint[i]; + } --cpuc->n_events;perf_event_update_userpage (event);
arch/x86/kernel/cpu/perf_event.h +5 -4
@@ -172,7 +172,10 @@ struct cpu_hw_events {added in the current transaction */ int assign[X86_PMC_IDX_MAX ];/* event to counter assignment */ u64 tags[X86_PMC_IDX_MAX ]; +struct perf_event *event_list[X86_PMC_IDX_MAX ];/* in enabled order */ +struct event_constraint *event_constraint[X86_PMC_IDX_MAX ]; +unsigned int group_flag;int is_fake;
@@ -519,9 +522,7 @@ struct x86_pmu {void (*put_event_constraints)(struct cpu_hw_events *cpuc,struct perf_event *event); -void (*commit_scheduling)(struct cpu_hw_events *cpuc, -struct perf_event *event, -int cntr); +void (*commit_scheduling)(struct cpu_hw_events *cpuc,int idx,int cntr);void (*start_scheduling)(struct cpu_hw_events *cpuc);
@@ -717,7 +718,7 @@ static inline void __x86_pmu_enable_evenvoid x86_pmu_enable_all (int added); -int perf_assign_events (struct perf_event **events,int n, +int perf_assign_events (struct event_constraint **constraints,int n,int wmin,int wmax,int *assign);int x86_schedule_events (struct cpu_hw_events *cpuc,int n,int *assign);
arch/x86/kernel/cpu/perf_event_intel.c +4 -11
@@ -2106,7 +2106,7 @@ static struct event_constraint * intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx, struct perf_event *event) { - struct event_constraint *c1 = event->hw.constraint; + struct event_constraint *c1 = cpuc->event_constraint[idx];struct event_constraint *c2;/*
@@ -2188,8 +2188,6 @@ intel_put_shared_regs_event_constraints(static void intel_put_event_constraints (struct cpu_hw_events *cpuc,struct perf_event *event) { -struct event_constraint *c = event->hw.constraint; -intel_put_shared_regs_event_constraints (cpuc, event);/*
@@ -2197,19 +2195,14 @@ static void intel_put_event_constraints(* all events are subject to and must call the * put_excl_constraints() routine */ -if (c && cpuc->excl_cntrs) +if (cpuc->excl_cntrs)intel_put_excl_constraints (cpuc, event); - -/* cleanup dynamic constraint */ -if (c && (c->flags &PERF_X86_EVENT_DYNAMIC )) - event->hw.constraint =NULL ; } -static void intel_commit_scheduling (struct cpu_hw_events *cpuc, -struct perf_event *event,int cntr) +static void intel_commit_scheduling (struct cpu_hw_events *cpuc,int idx,int cntr) {struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs; -struct event_constraint *c = event->hw.constraint; +struct event_constraint *c = cpuc->event_constraint[idx];struct intel_excl_states *xlo, *xl;int tid = cpuc->excl_thread_id;int o_tid =1 - tid;
arch/x86/kernel/cpu/perf_event_intel_ds.c +2 -2
@@ -706,9 +706,9 @@ void intel_pmu_pebs_disable(struct perf_ cpuc->pebs_enabled &= ~(1ULL << hwc->idx); -if (event->hw.constraint->flags &PERF_X86_EVENT_PEBS_LDLAT ) +if (event->hw.flags &PERF_X86_EVENT_PEBS_LDLAT ) cpuc->pebs_enabled &= ~(1ULL << (hwc->idx +32 )); -else if (event->hw.constraint->flags &PERF_X86_EVENT_PEBS_ST ) +else if (event->hw.flags &PERF_X86_EVENT_PEBS_ST ) cpuc->pebs_enabled &= ~(1ULL <<63 );if (cpuc->enabled)
arch/x86/kernel/cpu/perf_event_intel_uncore.c +3 -4
@@ -364,9 +364,8 @@ static int uncore_assign_events(struct ibitmap_zero (used_mask,UNCORE_PMC_IDX_MAX );for (i =0 , wmin =UNCORE_PMC_IDX_MAX , wmax =0 ; i < n; i++) { - hwc = &box->event_list[i]->hw; c =uncore_get_event_constraint (box, box->event_list[i]); - hwc->constraint = c; + box->event_constraint[i] = c; wmin =min (wmin, c->weight); wmax =max (wmax, c->weight); }
@@ -374,7 +373,7 @@ static int uncore_assign_events(struct i/* fastpath, try to reuse previous register */ for (i =0 ; i < n; i++) { hwc = &box->event_list[i]->hw; - c = hwc->constraint; + c = box->event_constraint[i];/* never assigned */ if (hwc->idx ==-1 )
@@ -394,7 +393,7 @@ static int uncore_assign_events(struct i }/* slow path */ if (i != n) - ret =perf_assign_events (box->event_list, n, + ret =perf_assign_events (box->event_constraint, n, wmin, wmax, assign);if (!assign || ret) {
arch/x86/kernel/cpu/perf_event_intel_uncore.h +1 -0
@@ -97,6 +97,7 @@ struct intel_uncore_box {atomic_t refcnt;struct perf_event *events[UNCORE_PMC_IDX_MAX ];struct perf_event *event_list[UNCORE_PMC_IDX_MAX ]; +struct event_constraint *event_constraint[UNCORE_PMC_IDX_MAX ];unsigned long active_mask[BITS_TO_LONGS (UNCORE_PMC_IDX_MAX )];u64 tags[UNCORE_PMC_IDX_MAX ];struct pci_dev *pci_dev;
Peter Zijlstra
maintainerRe: [PATCH v2 01/11] perf,x86: Fix event/group validation
arch/x86/kernel/cpu/perf_event.c int x86_schedule_events(struct cpu_hw_ev · line 791
Quoted 5 lines · Peter Zijlstra
arch/x86/kernel/cpu/perf_event.c +1
-0@@ -788,9 +788,9 @@ int x86_schedule_events(struct cpu_hw_ev x86_pmu.start_scheduling (cpuc);for (i =0 , wmin =X86_PMC_IDX_MAX , wmax =0 ; i < n; i++) { + cpuc->event_constraint[i] =NULL ;
^^^ that is new, which is esp. important in light of the intel_get_event_constraints() hunk below, which would happily continue life with a garbage constraint.
Quoted 17 lines
arch/x86/kernel/cpu/perf_event.c +3
-1c = x86_pmu.get_event_constraints (cpuc, i, cpuc->event_list[i]); + cpuc->event_constraint[i] = c; wmin =min (wmin, c->weight); wmax =max (wmax, c->weight); +++ b/arch/x86/kernel/cpu/perf_event_intel.c@@ -2106,7 +2106,7 @@ static struct event_constraint * intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx, struct perf_event *event) { - struct event_constraint *c1 = event->hw.constraint; + struct event_constraint *c1 = cpuc->event_constraint[idx];struct event_constraint *c2;/*
Re: [PATCH v2 01/11] perf,x86: Fix event/group validation
Quoted 11 lines · Peter Zijlstra
arch/x86/kernel/cpu/perf_event.c int x86_schedule_events(struct cpu_hw_ev · line 791
Quoted 5 lines · Peter Zijlstra
arch/x86/kernel/cpu/perf_event.c +1
-0@@ -788,9 +788,9 @@ int x86_schedule_events(struct cpu_hw_ev x86_pmu.start_scheduling (cpuc);for (i =0 , wmin =X86_PMC_IDX_MAX , wmax =0 ; i < n; i++) { + cpuc->event_constraint[i] =NULL ;^^^ that is new, which is esp. important in light of the
intel_get_event_constraints()hunk below, which would happily continue life with a garbage constraint.
You've moved the constraint list from event to cpuc. Yet, it is still an array of pointers
to constraints. So here you are saying, that in the case validate_group() is preempted and there is a context switch, there is still a risk of overwriting the
constraint? I don't see how because validate_group() is using a fake_cpuc. So yes, the cpuc->event_constraint[] array is modified but it is not the same as the actual cpuc used by non-validate code. Or am I still missing something?
When using dynamic constraints, we already have constraint storage in cpuc (to avoid calling kmalloc() in ctxsw context). Thus, I am wondering if it would not be easier to always use cpuc for constraint storage (no more pointers).
Quoted 17 lines
Quoted 17 lines
arch/x86/kernel/cpu/perf_event.c +3
-1c = x86_pmu.get_event_constraints (cpuc, i, cpuc->event_list[i]); + cpuc->event_constraint[i] = c; wmin =min (wmin, c->weight); wmax =max (wmax, c->weight); +++ b/arch/x86/kernel/cpu/perf_event_intel.c@@ -2106,7 +2106,7 @@ static struct event_constraint * intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx, struct perf_event *event) { - struct event_constraint *c1 = event->hw.constraint; + struct event_constraint *c1 = cpuc->event_constraint[idx];struct event_constraint *c2;/*
Peter Zijlstra
maintainerRe: [PATCH v2 01/11] perf,x86: Fix event/group validation
Quoted 21 lines · Stephane Eranian
Quoted 11 lines · Peter Zijlstra
arch/x86/kernel/cpu/perf_event.c int x86_schedule_events(struct cpu_hw_ev · line 791
Quoted 5 lines · Peter Zijlstra
arch/x86/kernel/cpu/perf_event.c +1
-0@@ -788,9 +788,9 @@ int x86_schedule_events(struct cpu_hw_ev x86_pmu.start_scheduling (cpuc);for (i =0 , wmin =X86_PMC_IDX_MAX , wmax =0 ; i < n; i++) { + cpuc->event_constraint[i] =NULL ;^^^ that is new, which is esp. important in light of the
intel_get_event_constraints()hunk below, which would happily continue life with a garbage constraint.You've moved the constraint list from event to cpuc. Yet, it is still an array of pointers
to constraints. So here you are saying, that in the casevalidate_group()is preempted and there is a context switch, there is still a risk of overwriting the
constraint? I don't see how becausevalidate_group()is using afake_cpuc. So yes, the cpuc->event_constraint[] array is modified but it is not the same as the actual cpuc used by non-validate code. Or am I still missing something?
Quoted 3 lines
When using dynamic constraints, we already have constraint storage in cpuc (to avoid calling
kmalloc()in ctxsw context). Thus, I am wondering if it would not be easier to always use cpuc for constraint storage (no more pointers).
No; the problem here is repeated use of the cpuc (the real one). Say one scheduling run installs a constraint pointer for event i. Then event i gets removed and another installed in the same spot.
Then the next scheduling run will pick up the old pointer in intel_get_event_constraints() as a base for the new one.
Re: [PATCH v2 01/11] perf,x86: Fix event/group validation
Quoted 26 lines · Peter Zijlstra
Quoted 21 lines · Stephane Eranian
Quoted 11 lines · Peter Zijlstra
arch/x86/kernel/cpu/perf_event.c int x86_schedule_events(struct cpu_hw_ev · line 791
Quoted 5 lines · Peter Zijlstra
arch/x86/kernel/cpu/perf_event.c +1
-0@@ -788,9 +788,9 @@ int x86_schedule_events(struct cpu_hw_ev x86_pmu.start_scheduling (cpuc);for (i =0 , wmin =X86_PMC_IDX_MAX , wmax =0 ; i < n; i++) { + cpuc->event_constraint[i] =NULL ;^^^ that is new, which is esp. important in light of the
intel_get_event_constraints()hunk below, which would happily continue life with a garbage constraint.You've moved the constraint list from event to cpuc. Yet, it is still an array of pointers
to constraints. So here you are saying, that in the casevalidate_group()is preempted and there is a context switch, there is still a risk of overwriting the
constraint? I don't see how becausevalidate_group()is using afake_cpuc. So yes, the cpuc->event_constraint[] array is modified but it is not the same as the actual cpuc used by non-validate code. Or am I still missing something?Quoted 3 lines
When using dynamic constraints, we already have constraint storage in cpuc (to avoid calling
kmalloc()in ctxsw context). Thus, I am wondering if it would not be easier to always use cpuc for constraint storage (no more pointers).
Quoted 7 lines
No; the problem here is repeated use of the cpuc (the real one). Say one scheduling run installs a constraint pointer for event i. Then event i gets removed and another installed in the same spot.
Then the next scheduling run will pick up the old pointer in
intel_get_event_constraints()as a base for the new one.
But where is the code that says: skip reinstalling the constraint in intel_get_event_constraints() because there is already a (stale) one? I don't see where that is.
Peter Zijlstra
maintainerRe: [PATCH v2 01/11] perf,x86: Fix event/group validation
Quoted 7 lines · Stephane Eranian
Quoted 6 lines · Peter Zijlstra
Quoted 5 lines · Stephane Eranian
Quoted 4 lines · Peter Zijlstra
Quoted 3 lines · Peter Zijlstra
arch/x86/kernel/cpu/perf_event.c +0
-0@@ -788,9 +788,9 @@ int x86_schedule_events(struct cpu_hw_ev x86_pmu.start_scheduling (cpuc);
Quoted 7 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
arch/x86/kernel/cpu/perf_event.c int x86_schedule_events(struct cpu_hw_ev · line 791
Quoted 2 lines
arch/x86/kernel/cpu/perf_event.c +1
-0for (i =0 , wmin =X86_PMC_IDX_MAX , wmax =0 ; i < n; i++) { + cpuc->event_constraint[i] =NULL ;But where is the code that says: skip reinstalling the constraint in
intel_get_event_constraints()because there is already a (stale) one? I don't see where that is.
IIRC the problem was that the copy from c2 into c1:
if (c1 && (c1->flags & PERF_X86_EVENT_DYNAMIC)) {
bitmap_copy(c1->idxmsk, c2->idxmsk, X86_PMC_IDX_MAX);
c1->weight = c2->weight;
c2 = c1;
}is incomplete. For instance, flags is not copied, and some code down the line might check that and get wrong flags.
I'm not entirely sure I saw misbehaviour, but I figured I'd better close that hole and rule out this is contributing to fail.
Re: [PATCH v2 01/11] perf,x86: Fix event/group validation
Quoted 18 lines · Peter Zijlstra
Quoted 7 lines · Stephane Eranian
Quoted 6 lines · Peter Zijlstra
Quoted 5 lines · Stephane Eranian
Quoted 4 lines · Peter Zijlstra
Quoted 3 lines · Peter Zijlstra
arch/x86/kernel/cpu/perf_event.c +0
-0@@ -788,9 +788,9 @@ int x86_schedule_events(struct cpu_hw_ev x86_pmu.start_scheduling (cpuc);Quoted 7 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
arch/x86/kernel/cpu/perf_event.c int x86_schedule_events(struct cpu_hw_ev · line 791
Quoted 2 lines
arch/x86/kernel/cpu/perf_event.c +1
-0for (i =0 , wmin =X86_PMC_IDX_MAX , wmax =0 ; i < n; i++) { + cpuc->event_constraint[i] =NULL ;But where is the code that says: skip reinstalling the constraint in
intel_get_event_constraints()because there is already a (stale) one? I don't see where that is.IIRC the problem was that the copy from c2 into c1:
Quoted 9 lines
if (c1 && (c1->flags & PERF_X86_EVENT_DYNAMIC)) { bitmap_copy(c1->idxmsk, c2->idxmsk, X86_PMC_IDX_MAX); c1->weight = c2->weight; c2 = c1; }is incomplete. For instance, flags is not copied, and some code down the line might check that and get wrong flags.
Ok, now I remember this code. It has to do with incremental scheduling. Suppose E1, E2, E3 events where E1 is exclusive. The first call is for scheduling E1. It gets to get_event_constraint() which "allocates" a dynamic constraint. The second call tries to schedule E1, E2. But the second time for E1, you already have the dynamic constraint allocated, so this code is reusing the constraint storage and just updates the bitmask and weight.
Now, that the storage is not actually dynamic (kmalloc'd), but taken from a fixed size array in cpuc, I believe we can simplify this and "re-allocate" the constraint for each incremental call to intel_get_event_constraints(). Do you agree?
Quoted 3 lines
I'm not entirely sure I saw misbehaviour, but I figured I'd better close that hole and rule out this is contributing to fail.
Peter Zijlstra
maintainerRe: [PATCH v2 01/11] perf,x86: Fix event/group validation
Quoted 19 lines · Stephane Eranian
Quoted 11 lines
IIRC the problem was that the copy from c2 into c1:
if (c1 && (c1->flags & PERF_X86_EVENT_DYNAMIC)) { bitmap_copy(c1->idxmsk, c2->idxmsk, X86_PMC_IDX_MAX); c1->weight = c2->weight; c2 = c1; }is incomplete. For instance, flags is not copied, and some code down the line might check that and get wrong flags.
Ok, now I remember this code. It has to do with incremental scheduling. Suppose E1, E2, E3 events where E1 is exclusive. The first call is for scheduling E1. It gets to
get_event_constraint()which "allocates" a dynamic constraint. The second call tries to schedule E1, E2. But the second time for E1, you already have the dynamic constraint allocated, so this code is reusing the constraint storage and just updates the bitmask and weight.
Quoted 4 lines
Now, that the storage is not actually dynamic (kmalloc'd), but taken from a fixed size array in cpuc, I believe we can simplify this and "re-allocate" the constraint for each incremental call to
intel_get_event_constraints(). Do you agree?
That would probably work, the whole incremental thing seems superfluous to me.
Re: [PATCH v2 01/11] perf,x86: Fix event/group validation
Quoted 20 lines · Peter Zijlstra
Quoted 19 lines · Stephane Eranian
Quoted 11 lines
IIRC the problem was that the copy from c2 into c1:
if (c1 && (c1->flags & PERF_X86_EVENT_DYNAMIC)) { bitmap_copy(c1->idxmsk, c2->idxmsk, X86_PMC_IDX_MAX); c1->weight = c2->weight; c2 = c1; }is incomplete. For instance, flags is not copied, and some code down the line might check that and get wrong flags.
Ok, now I remember this code. It has to do with incremental scheduling. Suppose E1, E2, E3 events where E1 is exclusive. The first call is for scheduling E1. It gets to
get_event_constraint()which "allocates" a dynamic constraint. The second call tries to schedule E1, E2. But the second time for E1, you already have the dynamic constraint allocated, so this code is reusing the constraint storage and just updates the bitmask and weight.
Quoted 7 lines
Quoted 4 lines
Now, that the storage is not actually dynamic (kmalloc'd), but taken from a fixed size array in cpuc, I believe we can simplify this and "re-allocate" the constraint for each incremental call to
intel_get_event_constraints(). Do you agree?That would probably work, the whole incremental thing seems superfluous to me.
Well, you could pass the whole lists (4) of events in one call and let the arch level
sort it out. It could do one pass and stop at first error to get a fixed bound like
today. But it would need to deal with groups as well...so not so easy and why repeat the group processing for each arch?
Peter Zijlstra
maintainerPATCH v2 2/11 perf/x86: Improve HT workaround GP counter constraint
In reply to Peter ZijlstraThe (SNB/IVB/HSW) HT bug only affects events that can be programmed onto GP counters, therefore we should only limit the number of GP counters that can be used per cpu -- iow we should not constrain the FP counters.
Furthermore, we should only enfore such a limit when there are in fact exclusive events being scheduled on either sibling.
--- arch/x86/kernel/cpu/perf_event.c | 36 +++++++++++++++++++++----- arch/x86/kernel/cpu/perf_event.h | 11 +++++-- arch/x86/kernel/cpu/perf_event_intel.c | 30 +++++++-------------- arch/x86/kernel/cpu/perf_event_intel_uncore.c | 2 - 4 files changed, 49 insertions(+), 30 deletions(-)
arch/x86/kernel/cpu/perf_event.c +30 -6
@@ -611,6 +611,7 @@ struct sched_state {int event;/* event index */ int counter;/* counter index */ int unassigned;/* number of events to be assigned left */ +int nr_gp;/* number of GP counters used */ unsigned long used[BITS_TO_LONGS (X86_PMC_IDX_MAX )]; };
@@ -620,9 +621,10 @@ struct sched_state {struct perf_sched {int max_weight;int max_events; +int max_gp; +int saved_states;struct event_constraint **constraints;struct sched_state state; -int saved_states;struct sched_state saved[SCHED_STATES_MAX ]; };
@@ -630,13 +632,14 @@ struct perf_sched {* Initialize interator that runs through all events and counters. */ static void perf_sched_init (struct perf_sched *sched,struct event_constraint **constraints, -int num,int wmin,int wmax) +int num,int wmin,int wmax,int gpmax) {int idx;memset (sched,0 ,sizeof (*sched)); sched->max_events = num; sched->max_weight = wmax; + sched->max_gp = gpmax; sched->constraints = constraints;for (idx =0 ; idx < num; idx++) {
@@ -696,11 +699,16 @@ static bool __perf_sched_find_counter(st goto done; } } +/* Grab the first unused counter starting with idx */ idx = sched->state.counter;for_each_set_bit_from (idx, c->idxmsk,INTEL_PMC_IDX_FIXED ) { -if (!__test_and_set_bit (idx, sched->state.used)) +if (!__test_and_set_bit (idx, sched->state.used)) { +if (sched->state.nr_gp++ >= sched->max_gp) +return false; + goto done; + } }return false;
@@ -757,11 +765,11 @@ static bool perf_sched_next_event(struct* Assign a counter for each event. */ int perf_assign_events (struct event_constraint **constraints,int n, -int wmin,int wmax,int *assign) +int wmin,int wmax,int gpmax,int *assign) {struct perf_sched sched; -perf_sched_init (&sched, constraints, n, wmin, wmax); +perf_sched_init (&sched, constraints, n, wmin, wmax, gpmax); do { if (!perf_sched_find_counter(&sched))
@@ -822,8 +830,24 @@ int x86_schedule_events(struct cpu_hw_ev/* slow path */ if (i != n) { +int gpmax = x86_pmu.num_counters; + +/* +* Do not allow scheduling of more than half the available +* generic counters. +* +* This helps avoid counter starvation of sibling thread by +* ensuring at most half the counters cannot be in exclusive +* mode. There is no designated counters for the limits. Any +* N/2 counters can be used. This helps with events with +* specific counter constraints. +*/ +if (is_ht_workaround_enabled () && !cpuc->is_fake && +READ_ONCE (cpuc->excl_cntrs->exclusive_present)) + gpmax /=2 ; + unsched =perf_assign_events (cpuc->event_constraint, n, wmin, - wmax, assign); + wmax, gpmax, assign); }/*
arch/x86/kernel/cpu/perf_event.h +8 -3
@@ -74,6 +74,7 @@ struct event_constraint {#define PERF_X86_EVENT_EXCL 0x0040/* HT exclusivity on counter */ #define PERF_X86_EVENT_DYNAMIC 0x0080/* dynamic alloc'd constraint */ #define PERF_X86_EVENT_RDPMC_ALLOWED 0x0100/* grant rdpmc permission */ +#define PERF_X86_EVENT_EXCL_ACCT 0x0200/* accounted EXCL event */ struct amd_nb {
@@ -134,8 +135,6 @@ enum intel_excl_state_type {struct intel_excl_states {enum intel_excl_state_type init_state[X86_PMC_IDX_MAX ];enum intel_excl_state_type state[X86_PMC_IDX_MAX ]; -int num_alloc_cntrs;/* #counters allocated */ -int max_alloc_cntrs;/* max #counters allowed */ bool sched_started;/* true if scheduling has started */ };
@@ -144,6 +143,11 @@ struct intel_excl_cntrs {struct intel_excl_states states[2 ]; +union { +u16 has_exclusive[2 ]; +u32 exclusive_present; + }; +int refcnt;/* per-core: #HT threads */ unsigned core_id;/* per-core: core id */ };
@@ -176,6 +180,7 @@ struct cpu_hw_events {struct perf_event *event_list[X86_PMC_IDX_MAX ];/* in enabled order */ struct event_constraint *event_constraint[X86_PMC_IDX_MAX ]; +int n_excl;/* the number of exclusive events */ unsigned int group_flag;int is_fake;
@@ -719,7 +724,7 @@ static inline void __x86_pmu_enable_evenvoid x86_pmu_enable_all (int added);int perf_assign_events (struct event_constraint **constraints,int n, -int wmin,int wmax,int *assign); +int wmin,int wmax,int gpmax,int *assign);int x86_schedule_events (struct cpu_hw_events *cpuc,int n,int *assign);void x86_pmu_stop (struct perf_event *event,int flags);
arch/x86/kernel/cpu/perf_event_intel.c +10 -20
@@ -1923,7 +1923,6 @@ intel_start_scheduling(struct cpu_hw_eve xl = &excl_cntrs->states[tid]; xl->sched_started = true; - xl->num_alloc_cntrs =0 ;/* * lock shared state until we are done scheduling * in stop_event_scheduling()
@@ -2000,6 +1999,11 @@ intel_get_excl_constraints(struct cpu_hw* across HT threads */ is_excl = c->flags &PERF_X86_EVENT_EXCL ; +if (is_excl && !(event->hw.flags &PERF_X86_EVENT_EXCL_ACCT )) { + event->hw.flags |=PERF_X86_EVENT_EXCL_ACCT ; +if (!cpuc->n_excl++) +WRITE_ONCE (excl_cntrs->has_exclusive[tid],1 ); + }/* * xl = state of current HT
@@ -2008,18 +2012,6 @@ intel_get_excl_constraints(struct cpu_hw xl = &excl_cntrs->states[tid]; xlo = &excl_cntrs->states[o_tid]; -/* -* do not allow scheduling of more than max_alloc_cntrs -* which is set to half the available generic counters. -* this helps avoid counter starvation of sibling thread -* by ensuring at most half the counters cannot be in -* exclusive mode. There is not designated counters for the -* limits. Any N/2 counters can be used. This helps with -* events with specifix counter constraints -*/ -if (xl->num_alloc_cntrs++ == xl->max_alloc_cntrs) -return &emptyconstraint; - cx = c;/*
@@ -2150,6 +2142,11 @@ static void intel_put_excl_constraints(s xl = &excl_cntrs->states[tid]; xlo = &excl_cntrs->states[o_tid]; +if (hwc->flags &PERF_X86_EVENT_EXCL_ACCT ) { + hwc->flags &= ~PERF_X86_EVENT_EXCL_ACCT ; +if (!--cpuc->n_excl) +WRITE_ONCE (excl_cntrs->has_exclusive[tid],0 ); + }/* * put_constraint may be called from x86_schedule_events()
@@ -2632,8 +2629,6 @@ static void intel_pmu_cpu_starting(int c cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR ];if (x86_pmu.flags &PMU_FL_EXCL_CNTRS ) { -int h = x86_pmu.num_counters >>1 ; -for_each_cpu (i,topology_thread_cpumask (cpu)) {struct intel_excl_cntrs *c;
@@ -2647,11 +2642,6 @@ static void intel_pmu_cpu_starting(int c } cpuc->excl_cntrs->core_id = core_id; cpuc->excl_cntrs->refcnt++; -/* -* set hard limit to half the number of generic counters -*/ - cpuc->excl_cntrs->states[0 ].max_alloc_cntrs = h; - cpuc->excl_cntrs->states[1 ].max_alloc_cntrs = h; } }
arch/x86/kernel/cpu/perf_event_intel_uncore.c +1 -1
@@ -394,7 +394,7 @@ static int uncore_assign_events(struct i/* slow path */ if (i != n) ret =perf_assign_events (box->event_constraint, n, - wmin, wmax, assign); + wmin, wmax, n, assign);if (!assign || ret) {for (i =0 ; i < n; i++)
Peter Zijlstra
maintainerRe: [PATCH v2 02/11] perf/x86: Improve HT workaround GP counter constraint
arch/x86/kernel/cpu/perf_event.c static bool __perf_sched_find_counter(st · line 708
Quoted 11 lines · Peter Zijlstra
arch/x86/kernel/cpu/perf_event.c +4
-0@@ -696,11 +699,16 @@ static bool __perf_sched_find_counter(st goto done; } } +/* Grab the first unused counter starting with idx */ idx = sched->state.counter;for_each_set_bit_from (idx, c->idxmsk,INTEL_PMC_IDX_FIXED ) { +if (!__test_and_set_bit (idx, sched->state.used)) { +if (sched->state.nr_gp++ >= sched->max_gp) +return false;
Note the placement inside the success path of the GP allocation, instead of the attempt a GP place we had before.
Quoted 12 lines
arch/x86/kernel/cpu/perf_event.c +2
-0+ goto done; + } }return false;@@ -2000,6 +1999,11 @@ intel_get_excl_constraints(struct cpu_hw* across HT threads
arch/x86/kernel/cpu/perf_event.c intel_get_excl_constraints(struct cpu_hw · line 2009
Quoted 10 lines
arch/x86/kernel/cpu/perf_event.c +5
-0*/ is_excl = c->flags &PERF_X86_EVENT_EXCL ; +if (is_excl && !(event->hw.flags &PERF_X86_EVENT_EXCL_ACCT )) { + event->hw.flags |=PERF_X86_EVENT_EXCL_ACCT ; +if (!cpuc->n_excl++) +WRITE_ONCE (excl_cntrs->has_exclusive[tid],1 ); + }/* * xl = state of current HT
And that is what keeps repeated get_event_constraints() calls from ever increasing our n_excl count.
Re: [PATCH v2 02/11] perf/x86: Improve HT workaround GP counter constraint
In reply to Peter ZijlstraQuoted 97 lines · Peter Zijlstra
The (SNB/IVB/HSW) HT bug only affects events that can be programmed onto GP counters, therefore we should only limit the number of GP counters that can be used per cpu -- iow we should not constrain the FP counters.
Furthermore, we should only enfore such a limit when there are in fact exclusive events being scheduled on either sibling.
Reported-by Vince Weaver <[email protected]>Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/kernel/cpu/perf_event.c | 36 +++++++++++++++++++++----- arch/x86/kernel/cpu/perf_event.h | 11 +++++-- arch/x86/kernel/cpu/perf_event_intel.c | 30 +++++++-------------- arch/x86/kernel/cpu/perf_event_intel_uncore.c | 2 - 4 files changed, 49 insertions(+), 30 deletions(-)arch/x86/kernel/cpu/perf_event.c +19
-5@@ -611,6 +611,7 @@ struct sched_state {int event;/* event index */ int counter;/* counter index */ int unassigned;/* number of events to be assigned left */ +int nr_gp;/* number of GP counters used */ unsigned long used[BITS_TO_LONGS (X86_PMC_IDX_MAX )]; };@@ -620,9 +621,10 @@ struct sched_state {struct perf_sched {int max_weight;int max_events; +int max_gp; +int saved_states;struct event_constraint **constraints;struct sched_state state; -int saved_states;struct sched_state saved[SCHED_STATES_MAX ]; };@@ -630,13 +632,14 @@ struct perf_sched {* Initialize interator that runs through all events and counters. */ static void perf_sched_init (struct perf_sched *sched,struct event_constraint **constraints, -int num,int wmin,int wmax) +int num,int wmin,int wmax,int gpmax) {int idx;memset (sched,0 ,sizeof (*sched)); sched->max_events = num; sched->max_weight = wmax; + sched->max_gp = gpmax; sched->constraints = constraints;for (idx =0 ; idx < num; idx++) {@@ -696,11 +699,16 @@ static bool __perf_sched_find_counter(st goto done; } } +/* Grab the first unused counter starting with idx */ idx = sched->state.counter;for_each_set_bit_from (idx, c->idxmsk,INTEL_PMC_IDX_FIXED ) { -if (!__test_and_set_bit (idx, sched->state.used)) +if (!__test_and_set_bit (idx, sched->state.used)) { +if (sched->state.nr_gp++ >= sched->max_gp) +return false; + goto done; + } }return false;@@ -757,11 +765,11 @@ static bool perf_sched_next_event(struct* Assign a counter for each event. */ int perf_assign_events (struct event_constraint **constraints,int n, -int wmin,int wmax,int *assign) +int wmin,int wmax,int gpmax,int *assign) {struct perf_sched sched; -perf_sched_init (&sched, constraints, n, wmin, wmax); +perf_sched_init (&sched, constraints, n, wmin, wmax, gpmax); do { if (!perf_sched_find_counter(&sched))@@ -822,8 +830,24 @@ int x86_schedule_events(struct cpu_hw_ev/* slow path */ if (i != n) { +int gpmax = x86_pmu.num_counters; + +/* +* Do not allow scheduling of more than half the available +* generic counters. +*
arch/x86/kernel/cpu/perf_event.c int x86_schedule_events(struct cpu_hw_ev · line 848
Quoted 10 lines
arch/x86/kernel/cpu/perf_event.c +10
-0+* This helps avoid counter starvation of sibling thread by +* ensuring at most half the counters cannot be in exclusive +* mode. There is no designated counters for the limits. Any +* N/2 counters can be used. This helps with events with +* specific counter constraints. +*/ +if (is_ht_workaround_enabled () && !cpuc->is_fake && +READ_ONCE (cpuc->excl_cntrs->exclusive_present)) + gpmax /=2 ; +
What I don't like about this part is that this is a hack to work around a bug on some limited Intel CPUs and yet it is in the middle of generic x86 code. I understand it will be inoperative on AMD PMU and is not used by Intel uncore. On KNC or P6, you will not have is_ht_workaround_enabled(). Could this be made a x86_pmu callback? x86_pmu.counter_limit()?
Quoted 141 lines
arch/x86/kernel/cpu/perf_event.c +1
-1unsched =perf_assign_events (cpuc->event_constraint, n, wmin, - wmax, assign); + wmax, gpmax, assign); }/* arch/x86/kernel/cpu/perf_event.h +8
-3@@ -74,6 +74,7 @@ struct event_constraint {#define PERF_X86_EVENT_EXCL 0x0040/* HT exclusivity on counter */ #define PERF_X86_EVENT_DYNAMIC 0x0080/* dynamic alloc'd constraint */ #define PERF_X86_EVENT_RDPMC_ALLOWED 0x0100/* grant rdpmc permission */ +#define PERF_X86_EVENT_EXCL_ACCT 0x0200/* accounted EXCL event */ struct amd_nb {@@ -134,8 +135,6 @@ enum intel_excl_state_type {struct intel_excl_states {enum intel_excl_state_type init_state[X86_PMC_IDX_MAX ];enum intel_excl_state_type state[X86_PMC_IDX_MAX ]; -int num_alloc_cntrs;/* #counters allocated */ -int max_alloc_cntrs;/* max #counters allowed */ bool sched_started;/* true if scheduling has started */ };@@ -144,6 +143,11 @@ struct intel_excl_cntrs {struct intel_excl_states states[2 ]; +union { +u16 has_exclusive[2 ]; +u32 exclusive_present; + }; +int refcnt;/* per-core: #HT threads */ unsigned core_id;/* per-core: core id */ };@@ -176,6 +180,7 @@ struct cpu_hw_events {struct perf_event *event_list[X86_PMC_IDX_MAX ];/* in enabled order */ struct event_constraint *event_constraint[X86_PMC_IDX_MAX ]; +int n_excl;/* the number of exclusive events */ unsigned int group_flag;int is_fake;@@ -719,7 +724,7 @@ static inline void __x86_pmu_enable_evenvoid x86_pmu_enable_all (int added);int perf_assign_events (struct event_constraint **constraints,int n, -int wmin,int wmax,int *assign); +int wmin,int wmax,int gpmax,int *assign);int x86_schedule_events (struct cpu_hw_events *cpuc,int n,int *assign);void x86_pmu_stop (struct perf_event *event,int flags);arch/x86/kernel/cpu/perf_event_intel.c +10
-20@@ -1923,7 +1923,6 @@ intel_start_scheduling(struct cpu_hw_eve xl = &excl_cntrs->states[tid]; xl->sched_started = true; - xl->num_alloc_cntrs =0 ;/* * lock shared state until we are done scheduling * in stop_event_scheduling() @@ -2000,6 +1999,11 @@ intel_get_excl_constraints(struct cpu_hw* across HT threads */ is_excl = c->flags &PERF_X86_EVENT_EXCL ; +if (is_excl && !(event->hw.flags &PERF_X86_EVENT_EXCL_ACCT )) { + event->hw.flags |=PERF_X86_EVENT_EXCL_ACCT ; +if (!cpuc->n_excl++) +WRITE_ONCE (excl_cntrs->has_exclusive[tid],1 ); + }/* * xl = state of current HT @@ -2008,18 +2012,6 @@ intel_get_excl_constraints(struct cpu_hw xl = &excl_cntrs->states[tid]; xlo = &excl_cntrs->states[o_tid]; -/* -* do not allow scheduling of more than max_alloc_cntrs -* which is set to half the available generic counters. -* this helps avoid counter starvation of sibling thread -* by ensuring at most half the counters cannot be in -* exclusive mode. There is not designated counters for the -* limits. Any N/2 counters can be used. This helps with -* events with specifix counter constraints -*/ -if (xl->num_alloc_cntrs++ == xl->max_alloc_cntrs) -return &emptyconstraint; - cx = c;/* @@ -2150,6 +2142,11 @@ static void intel_put_excl_constraints(s xl = &excl_cntrs->states[tid]; xlo = &excl_cntrs->states[o_tid]; +if (hwc->flags &PERF_X86_EVENT_EXCL_ACCT ) { + hwc->flags &= ~PERF_X86_EVENT_EXCL_ACCT ; +if (!--cpuc->n_excl) +WRITE_ONCE (excl_cntrs->has_exclusive[tid],0 ); + }/* * put_constraint may be called from x86_schedule_events() @@ -2632,8 +2629,6 @@ static void intel_pmu_cpu_starting(int c cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR ];if (x86_pmu.flags &PMU_FL_EXCL_CNTRS ) { -int h = x86_pmu.num_counters >>1 ; -for_each_cpu (i,topology_thread_cpumask (cpu)) {struct intel_excl_cntrs *c;@@ -2647,11 +2642,6 @@ static void intel_pmu_cpu_starting(int c } cpuc->excl_cntrs->core_id = core_id; cpuc->excl_cntrs->refcnt++; -/* -* set hard limit to half the number of generic counters -*/ - cpuc->excl_cntrs->states[0 ].max_alloc_cntrs = h; - cpuc->excl_cntrs->states[1 ].max_alloc_cntrs = h; } }arch/x86/kernel/cpu/perf_event_intel_uncore.c +1
-1@@ -394,7 +394,7 @@ static int uncore_assign_events(struct i/* slow path */ if (i != n) ret =perf_assign_events (box->event_constraint, n, - wmin, wmax, assign); + wmin, wmax, n, assign);if (!assign || ret) {for (i =0 ; i < n; i++)
Peter Zijlstra
maintainerRe: [PATCH v2 02/11] perf/x86: Improve HT workaround GP counter constraint
Please trim your email.
Quoted 17 lines · Stephane Eranian
Quoted 17 lines
arch/x86/kernel/cpu/perf_event.c +13
-0@@ -822,8 +830,24 @@ int x86_schedule_events(struct cpu_hw_ev/* slow path */ if (i != n) { +int gpmax = x86_pmu.num_counters; + +/* +* Do not allow scheduling of more than half the available +* generic counters. +* +* This helps avoid counter starvation of sibling thread by +* ensuring at most half the counters cannot be in exclusive +* mode. There is no designated counters for the limits. Any +* N/2 counters can be used. This helps with events with +* specific counter constraints. +*/ + if (is_ht_workaround_enabled() && !cpuc->is_fake &&
Quoted 8 lines
arch/x86/kernel/cpu/perf_event.c int x86_schedule_events(struct cpu_hw_ev · line 848
Quoted 3 lines
arch/x86/kernel/cpu/perf_event.c +3
-0+ READ_ONCE(cpuc->excl_cntrs->exclusive_present)) + gpmax /=2 ; +What I don't like about this part is that this is a hack to work around a bug on some limited Intel CPUs and yet it is in the middle of generic x86 code. I understand it will be inoperative on AMD PMU and is not used by Intel uncore. On KNC or P6, you will not have
is_ht_workaround_enabled(). Could this be made ax86_pmucallback?x86_pmu.counter_limit()?
It'll be slower though. You get an indirect function call in there.
But sure we can clean that up later if you like; there's other things needing to be fixed here first.
I'm going to overhaul the whole get/put constraints stuff first.
Re: [PATCH v2 02/11] perf/x86: Improve HT workaround GP counter constraint
Quoted 29 lines · Peter Zijlstra
Please trim your email.
Quoted 17 lines · Stephane Eranian
Quoted 17 lines
arch/x86/kernel/cpu/perf_event.c +13
-0@@ -822,8 +830,24 @@ int x86_schedule_events(struct cpu_hw_ev/* slow path */ if (i != n) { +int gpmax = x86_pmu.num_counters; + +/* +* Do not allow scheduling of more than half the available +* generic counters. +* +* This helps avoid counter starvation of sibling thread by +* ensuring at most half the counters cannot be in exclusive +* mode. There is no designated counters for the limits. Any +* N/2 counters can be used. This helps with events with +* specific counter constraints. +*/ + if (is_ht_workaround_enabled() && !cpuc->is_fake &&Quoted 8 lines
arch/x86/kernel/cpu/perf_event.c int x86_schedule_events(struct cpu_hw_ev · line 848
Quoted 3 lines
arch/x86/kernel/cpu/perf_event.c +3
-0+ READ_ONCE(cpuc->excl_cntrs->exclusive_present)) + gpmax /=2 ; +What I don't like about this part is that this is a hack to work around a bug on some limited Intel CPUs and yet it is in the middle of generic x86 code. I understand it will be inoperative on AMD PMU and is not used by Intel uncore. On KNC or P6, you will not have
is_ht_workaround_enabled(). Could this be made ax86_pmucallback?x86_pmu.counter_limit()?
Quoted 6 lines
It'll be slower though. You get an indirect function call in there.
But sure we can clean that up later if you like; there's other things needing to be fixed here first.
I'm going to overhaul the whole get/put constraints stuff first.
Ok, I think it would be good to balance to number of get/put. It would avoid the confusion. Is that what you are thinking about?
Peter Zijlstra
maintainerRe: [PATCH v2 02/11] perf/x86: Improve HT workaround GP counter constraint
Quoted 4 lines · Stephane Eranian
Quoted 1 line
I'm going to overhaul the whole get/put constraints stuff first.
Ok, I think it would be good to balance to number of get/put. It would avoid the confusion. Is that what you are thinking about?
Yes, and remove the few associated modifications to events.
Peter Zijlstra
maintainerRe: [PATCH v2 02/11] perf/x86: Improve HT workaround GP counter constraint
Quoted 7 lines · Peter Zijlstra
Quoted 4 lines · Stephane Eranian
Quoted 1 line
I'm going to overhaul the whole get/put constraints stuff first.
Ok, I think it would be good to balance to number of get/put. It would avoid the confusion. Is that what you are thinking about?
Yes, and remove the few associated modifications to events.
I have the below (in 4 patches); compile tested only so far.
--- perf_event.c | 51 +++++++++++++++++++++++++-------------------------- perf_event.h | 4 ++-- perf_event_intel.c | 40 ++++------------------------------------ 3 files changed, 31 insertions(+), 64 deletions(-)
arch/x86/kernel/cpu/perf_event.c +25 -26
@@ -810,9 +810,15 @@ int x86_schedule_events(struct cpu_hw_ev x86_pmu.start_scheduling (cpuc);for (i =0 , wmin =X86_PMC_IDX_MAX , wmax =0 ; i < n; i++) { - cpuc->event_constraint[i] =NULL ; - c = x86_pmu.get_event_constraints (cpuc, i, cpuc->event_list[i]); - cpuc->event_constraint[i] = c; +/* +* Only call get_event_constraints() once! +*/ + c = cpuc->event_constraint[i]; +if (!c) { + e = cpuc->event_list[i]; + c = x86_pmu.get_event_constraints (cpuc, i, e); + cpuc->event_constraint[i] = c; + } wmin =min (wmin, c->weight); wmax =max (wmax, c->weight);
@@ -875,27 +881,23 @@ int x86_schedule_events(struct cpu_hw_ev* validate an event group (assign == NULL) */ if (!unsched && assign) { -for (i =0 ; i < n; i++) { - e = cpuc->event_list[i]; - e->hw.flags |=PERF_X86_EVENT_COMMITTED ; -if (x86_pmu.commit_scheduling) +if (x86_pmu.commit_scheduling) { +for (i =0 ; i < n; i++) { x86_pmu.commit_scheduling (cpuc, i, assign[i]); + } } - }else { -for (i =0 ; i < n; i++) { - e = cpuc->event_list[i]; -/* -* do not put_constraint() on comitted events, -* because they are good to go -*/ -if ((e->hw.flags &PERF_X86_EVENT_COMMITTED )) -continue ; + }else if (x86_pmu.put_event_constraints) { +/* x86_pmu_add() will not yet have updated n_events */ + i = cpuc->n_events; + +/* x86_pmu_commit_txn() relies on n_txn */ +if (cpuc->group_flag &PERF_EVENT_TXN ) + i -= cpuc->n_txn; -/* -* release events that failed scheduling -*/ -if (x86_pmu.put_event_constraints) - x86_pmu.put_event_constraints (cpuc, e); +for (; i < n; i++) { + e = cpuc->event_list[i]; +/* release events that failed scheduling */ + x86_pmu.put_event_constraints (cpuc, e); } }
@@ -923,6 +925,7 @@ static int collect_events(struct cpu_hw_if (n >= max_count)return -EINVAL ; cpuc->event_list[n] = leader; + cpuc->event_constraint[n] =NULL ; n++; }if (!dogrp)
@@ -937,6 +940,7 @@ static int collect_events(struct cpu_hw_return -EINVAL ; cpuc->event_list[n] = event; + cpuc->event_constraint[n] =NULL ; n++; }return n;
@@ -1295,11 +1299,6 @@ static void x86_pmu_del(struct perf_evenint i;/* -* event is descheduled -*/ - event->hw.flags &= ~PERF_X86_EVENT_COMMITTED ; - -/* * If we're called during a txn, we don't need to do anything. * The events never got scheduled and ->cancel_txn will truncate * the event_list.
arch/x86/kernel/cpu/perf_event.h +2 -2
@@ -68,13 +68,13 @@ struct event_constraint {#define PERF_X86_EVENT_PEBS_LDLAT 0x0001/* ld+ldlat data address sampling */ #define PERF_X86_EVENT_PEBS_ST 0x0002/* st data address sampling */ #define PERF_X86_EVENT_PEBS_ST_HSW 0x0004/* haswell style datala, store */ -#define PERF_X86_EVENT_COMMITTED 0x0008/* event passed commit_txn */ +#define PERF_X86_EVENT_PEBS_LD_HSW 0x0010/* haswell style datala, load */ #define PERF_X86_EVENT_PEBS_NA_HSW 0x0020/* haswell style datala, unknown */ #define PERF_X86_EVENT_EXCL 0x0040/* HT exclusivity on counter */ #define PERF_X86_EVENT_DYNAMIC 0x0080/* dynamic alloc'd constraint */ #define PERF_X86_EVENT_RDPMC_ALLOWED 0x0100/* grant rdpmc permission */ -#define PERF_X86_EVENT_EXCL_ACCT 0x0200/* accounted EXCL event */ +#define PERF_X86_EVENT_AUTO_RELOAD 0x0400/* use PEBS auto-reload */ #define PERF_X86_EVENT_FREERUNNING 0x0800/* use freerunning PEBS */
arch/x86/kernel/cpu/perf_event_intel.c +4 -36
@@ -1955,14 +1955,6 @@ __intel_shared_reg_get_constraints(strucunsigned long flags;int idx = reg->idx; -/* -* reg->alloc can be set due to existing state, so for fake cpuc we -* need to ignore this, otherwise we might fail to allocate proper fake -* state for this extra reg constraint. Also see the comment below. -*/ -if (reg->alloc && !cpuc->is_fake) -return NULL ;/* call x86_get_event_constraint() */ - again: era = &cpuc->shared_regs->regs[idx];/*
@@ -1986,14 +1978,6 @@ __intel_shared_reg_get_constraints(strucif (!cpuc->is_fake) {if (idx != reg->idx)intel_fixup_er (event, idx); - -/* -* x86_schedule_events() can call get_event_constraints() -* multiple times on events in the case of incremental -* scheduling(). reg->alloc ensures we only do the ER -* allocation once. -*/ - reg->alloc =1 ; }/* lock in msr value */
@@ -2026,24 +2010,12 @@ __intel_shared_reg_put_constraints(struc {struct er_account *era; -/* -* Only put constraint if extra reg was actually allocated. Also takes -* care of event which do not use an extra shared reg. -* -* Also, if this is a fake cpuc we shouldn't touch any event state -* (reg->alloc) and we don't care about leaving inconsistent cpuc state -* either since it'll be thrown out. -*/ -if (!reg->alloc || cpuc->is_fake) -return ; +WARN_ON_ONCE (cpuc->is_fake); era = &cpuc->shared_regs->regs[reg->idx];/* one fewer user */ atomic_dec (&era->ref); - -/* allocate again next time */ - reg->alloc =0 ; } static struct event_constraint *
@@ -2261,8 +2233,7 @@ intel_get_excl_constraints(struct cpu_hw* across HT threads */ is_excl = c->flags &PERF_X86_EVENT_EXCL ; -if (is_excl && !(event->hw.flags &PERF_X86_EVENT_EXCL_ACCT )) { - event->hw.flags |=PERF_X86_EVENT_EXCL_ACCT ; +if (is_excl) {if (!cpuc->n_excl++)WRITE_ONCE (excl_cntrs->has_exclusive[tid],1 ); }
@@ -2350,11 +2321,8 @@ static void intel_put_excl_constraints(sif (WARN_ON_ONCE (!excl_cntrs))return ; -if (hwc->flags &PERF_X86_EVENT_EXCL_ACCT ) { - hwc->flags &= ~PERF_X86_EVENT_EXCL_ACCT ; -if (!--cpuc->n_excl) -WRITE_ONCE (excl_cntrs->has_exclusive[tid],0 ); - } +if ((hwc->flags &PERF_X86_EVENT_EXCL ) && !--cpuc->n_excl) +WRITE_ONCE (excl_cntrs->has_exclusive[tid],0 );/* * If event was actually assigned, then mark the counter state as
Re: [PATCH v2 02/11] perf/x86: Improve HT workaround GP counter constraint
Quoted 27 lines · Peter Zijlstra
Quoted 7 lines · Peter Zijlstra
Quoted 4 lines · Stephane Eranian
Quoted 1 line
I'm going to overhaul the whole get/put constraints stuff first.
Ok, I think it would be good to balance to number of get/put. It would avoid the confusion. Is that what you are thinking about?
Yes, and remove the few associated modifications to events.
I have the below (in 4 patches); compile tested only so far.
--- perf_event.c | 51 +++++++++++++++++++++++++-------------------------- perf_event.h | 4 ++-- perf_event_intel.c | 40 ++++------------------------------------ 3 files changed, 31 insertions(+), 64 deletions(-)arch/x86/kernel/cpu/perf_event.c +0
-3@@ -810,9 +810,15 @@ int x86_schedule_events(struct cpu_hw_ev x86_pmu.start_scheduling (cpuc);for (i =0 , wmin =X86_PMC_IDX_MAX , wmax =0 ; i < n; i++) { - cpuc->event_constraint[i] =NULL ; - c = x86_pmu.get_event_constraints (cpuc, i, cpuc->event_list[i]); - cpuc->event_constraint[i] = c;
arch/x86/kernel/cpu/perf_event.c int x86_schedule_events(struct cpu_hw_ev · line 822
Quoted 10 lines
arch/x86/kernel/cpu/perf_event.c +9
-0+/* +* Only call get_event_constraints() once! +*/ + c = cpuc->event_constraint[i]; +if (!c) { + e = cpuc->event_list[i]; + c = x86_pmu.get_event_constraints (cpuc, i, e); + cpuc->event_constraint[i] = c; + }
But are you removing the incremental calls from the upper layer via x86_pmu.add()?
If not, then you are saying the dynamic constraint you got for offcore_response, LBR
or the HT workaround is still the best avail now.
Quoted 24 lines
arch/x86/kernel/cpu/perf_event.c +3
-11wmin =min (wmin, c->weight); wmax =max (wmax, c->weight);@@ -875,27 +881,23 @@ int x86_schedule_events(struct cpu_hw_ev* validate an event group (assign == NULL) */ if (!unsched && assign) { -for (i =0 ; i < n; i++) { - e = cpuc->event_list[i]; - e->hw.flags |=PERF_X86_EVENT_COMMITTED ; -if (x86_pmu.commit_scheduling) +if (x86_pmu.commit_scheduling) { +for (i =0 ; i < n; i++) { x86_pmu.commit_scheduling (cpuc, i, assign[i]); + } } - }else { -for (i =0 ; i < n; i++) { - e = cpuc->event_list[i]; -/* -* do not put_constraint() on comitted events, -* because they are good to go -*/
arch/x86/kernel/cpu/perf_event.c int x86_schedule_events(struct cpu_hw_ev · line 896
Quoted 10 lines
arch/x86/kernel/cpu/perf_event.c +7
-2-if ((e->hw.flags &PERF_X86_EVENT_COMMITTED )) -continue ; + }else if (x86_pmu.put_event_constraints) { +/* x86_pmu_add() will not yet have updated n_events */ + i = cpuc->n_events; + +/* x86_pmu_commit_txn() relies on n_txn */ +if (cpuc->group_flag &PERF_EVENT_TXN ) + i -= cpuc->n_txn;
Is that to say, you're doing only one pass over all active events?
Quoted 49 lines
arch/x86/kernel/cpu/perf_event.c +6
-10-/* -* release events that failed scheduling -*/ -if (x86_pmu.put_event_constraints) - x86_pmu.put_event_constraints (cpuc, e); +for (; i < n; i++) { + e = cpuc->event_list[i]; +/* release events that failed scheduling */ + x86_pmu.put_event_constraints (cpuc, e); } }@@ -923,6 +925,7 @@ static int collect_events(struct cpu_hw_if (n >= max_count)return -EINVAL ; cpuc->event_list[n] = leader; + cpuc->event_constraint[n] =NULL ; n++; }if (!dogrp)@@ -937,6 +940,7 @@ static int collect_events(struct cpu_hw_return -EINVAL ; cpuc->event_list[n] = event; + cpuc->event_constraint[n] =NULL ; n++; }return n;@@ -1295,11 +1299,6 @@ static void x86_pmu_del(struct perf_evenint i;/* -* event is descheduled -*/ - event->hw.flags &= ~PERF_X86_EVENT_COMMITTED ; - -/* * If we're called during a txn, we don't need to do anything. * The events never got scheduled and ->cancel_txn will truncate * the event_list. arch/x86/kernel/cpu/perf_event.h +1
-1@@ -68,13 +68,13 @@ struct event_constraint {#define PERF_X86_EVENT_PEBS_LDLAT 0x0001/* ld+ldlat data address sampling */ #define PERF_X86_EVENT_PEBS_ST 0x0002/* st data address sampling */ #define PERF_X86_EVENT_PEBS_ST_HSW 0x0004/* haswell style datala, store */ -#define PERF_X86_EVENT_COMMITTED 0x0008/* event passed commit_txn */ +
arch/x86/kernel/cpu/perf_event.h struct event_constraint { · line 80
Quoted 10 lines
arch/x86/kernel/cpu/perf_event.h +1
-1#define PERF_X86_EVENT_PEBS_LD_HSW 0x0010/* haswell style datala, load */ #define PERF_X86_EVENT_PEBS_NA_HSW 0x0020/* haswell style datala, unknown */ #define PERF_X86_EVENT_EXCL 0x0040/* HT exclusivity on counter */ #define PERF_X86_EVENT_DYNAMIC 0x0080/* dynamic alloc'd constraint */ #define PERF_X86_EVENT_RDPMC_ALLOWED 0x0100/* grant rdpmc permission */ -#define PERF_X86_EVENT_EXCL_ACCT 0x0200/* accounted EXCL event */ +#define PERF_X86_EVENT_AUTO_RELOAD 0x0400/* use PEBS auto-reload */ #define PERF_X86_EVENT_FREERUNNING 0x0800/* use freerunning PEBS */
What's free running PEBS? ;->
Quoted 83 lines
arch/x86/kernel/cpu/perf_event_intel.c +4
-36@@ -1955,14 +1955,6 @@ __intel_shared_reg_get_constraints(strucunsigned long flags;int idx = reg->idx; -/* -* reg->alloc can be set due to existing state, so for fake cpuc we -* need to ignore this, otherwise we might fail to allocate proper fake -* state for this extra reg constraint. Also see the comment below. -*/ -if (reg->alloc && !cpuc->is_fake) -return NULL ;/* call x86_get_event_constraint() */ - again: era = &cpuc->shared_regs->regs[idx];/* @@ -1986,14 +1978,6 @@ __intel_shared_reg_get_constraints(strucif (!cpuc->is_fake) {if (idx != reg->idx)intel_fixup_er (event, idx); - -/* -* x86_schedule_events() can call get_event_constraints() -* multiple times on events in the case of incremental -* scheduling(). reg->alloc ensures we only do the ER -* allocation once. -*/ - reg->alloc =1 ; }/* lock in msr value */ @@ -2026,24 +2010,12 @@ __intel_shared_reg_put_constraints(struc {struct er_account *era; -/* -* Only put constraint if extra reg was actually allocated. Also takes -* care of event which do not use an extra shared reg. -* -* Also, if this is a fake cpuc we shouldn't touch any event state -* (reg->alloc) and we don't care about leaving inconsistent cpuc state -* either since it'll be thrown out. -*/ -if (!reg->alloc || cpuc->is_fake) -return ; +WARN_ON_ONCE (cpuc->is_fake); era = &cpuc->shared_regs->regs[reg->idx];/* one fewer user */ atomic_dec (&era->ref); - -/* allocate again next time */ - reg->alloc =0 ; } static struct event_constraint *@@ -2261,8 +2233,7 @@ intel_get_excl_constraints(struct cpu_hw* across HT threads */ is_excl = c->flags &PERF_X86_EVENT_EXCL ; -if (is_excl && !(event->hw.flags &PERF_X86_EVENT_EXCL_ACCT )) { - event->hw.flags |=PERF_X86_EVENT_EXCL_ACCT ; +if (is_excl) {if (!cpuc->n_excl++)WRITE_ONCE (excl_cntrs->has_exclusive[tid],1 ); }@@ -2350,11 +2321,8 @@ static void intel_put_excl_constraints(sif (WARN_ON_ONCE (!excl_cntrs))return ; -if (hwc->flags &PERF_X86_EVENT_EXCL_ACCT ) { - hwc->flags &= ~PERF_X86_EVENT_EXCL_ACCT ; -if (!--cpuc->n_excl) -WRITE_ONCE (excl_cntrs->has_exclusive[tid],0 ); - } +if ((hwc->flags &PERF_X86_EVENT_EXCL ) && !--cpuc->n_excl) +WRITE_ONCE (excl_cntrs->has_exclusive[tid],0 );/* * If event was actually assigned, then mark the counter state as
Peter Zijlstra
maintainerRe: [PATCH v2 02/11] perf/x86: Improve HT workaround GP counter constraint
Quoted 5 lines · Stephane Eranian
But are you removing the incremental calls from the upper layer via
x86_pmu.add()?
If not, then you are saying the dynamic constraint you got foroffcore_response, LBR
or the HT workaround is still the best avail now.
sigh, see I knew I was missing something :/
So then for c->flag & DYNAMIC we should put and get again, right?
Re: [PATCH v2 02/11] perf/x86: Improve HT workaround GP counter constraint
Quoted 10 lines · Peter Zijlstra
Quoted 5 lines · Stephane Eranian
But are you removing the incremental calls from the upper layer via
x86_pmu.add()?
If not, then you are saying the dynamic constraint you got foroffcore_response, LBR
or the HT workaround is still the best avail now.sigh, see I knew I was missing something :/
So then for
c->flag& DYNAMIC we should put and get again, right?
I think it would still work. The constraint, even dynamic, you got on first call is "locked" to you. So if you call x86_schedule_events multiple times to add events like for E1, E2, E3 which causes at most 3 calls to x86_schedule_events9). Then you'd get the same constraint but
it would still be valid. Just need to make sure you release it if scheduling fails at event N and it has the dynamic constraint.
Peter Zijlstra
maintainerRe: [PATCH v2 02/11] perf/x86: Improve HT workaround GP counter constraint
In reply to Stephane EranianQuoted 4 lines · Stephane Eranian
Quoted 3 lines
#define
PERF_X86_EVENT_AUTO_RELOAD0x0400/* use PEBS auto-reload */
#definePERF_X86_EVENT_FREERUNNING0x0800/* use freerunning PEBS */What's free running PEBS? ;->
The series here:
Re: [PATCH v2 02/11] perf/x86: Improve HT workaround GP counter constraint
Quoted 9 lines · Peter Zijlstra
Quoted 4 lines · Stephane Eranian
Quoted 3 lines
#define
PERF_X86_EVENT_AUTO_RELOAD0x0400/* use PEBS auto-reload */
#definePERF_X86_EVENT_FREERUNNING0x0800/* use freerunning PEBS */What's free running PEBS? ;->
The series here:
Ok, I assume this goes with more patches that enable depth > 1 for the PEBS buffer, otherwise
I am not so sure about the added value. PEBS micro-code has to reprogram that MSR
no matter what, hasn't it?
Re: [PATCH v2 02/11] perf/x86: Improve HT workaround GP counter constraint
In reply to Peter Zijlstraarch/x86/kernel/cpu/perf_event.c int x86_schedule_events(struct cpu_hw_ev · line 845
Quoted 2 lines · Peter Zijlstra
arch/x86/kernel/cpu/perf_event.c +2
-0+*/ + if (is_ht_workaround_enabled() && !cpuc->is_fake &&
Could this function also check if at least one leaking event is scheduled somewhere? (e.g. from a global count)
-Andi
Signature · 1 line
[email protected] -- Speaking for myself only
Peter Zijlstra
maintainerRe: [PATCH v2 02/11] perf/x86: Improve HT workaround GP counter constraint
Quoted 6 lines · Andi Kleen
arch/x86/kernel/cpu/perf_event.c int x86_schedule_events(struct cpu_hw_ev · line 845
Quoted 2 lines · Peter Zijlstra
arch/x86/kernel/cpu/perf_event.c +2
-0+*/ + if (is_ht_workaround_enabled() && !cpuc->is_fake &&Could this function also check if at least one leaking event is scheduled somewhere? (e.g. from a global count)
You truncated one line too many:
+ if (is_ht_workaround_enabled() && !cpuc->is_fake &&
+ READ_ONCE(cpuc->excl_cntrs->exclusive_present))
+ gpmax /= 2;
Guess what that READ_ONCE() does? It checks if there's one such leaky event on the current core.
Re: [PATCH v2 02/11] perf/x86: Improve HT workaround GP counter constraint
Quoted 14 lines · Peter Zijlstra
Quoted 6 lines · Andi Kleen
arch/x86/kernel/cpu/perf_event.c int x86_schedule_events(struct cpu_hw_ev · line 845
Quoted 2 lines · Peter Zijlstra
arch/x86/kernel/cpu/perf_event.c +2
-0+*/ + if (is_ht_workaround_enabled() && !cpuc->is_fake &&Could this function also check if at least one leaking event is scheduled somewhere? (e.g. from a global count)
You truncated one line too many:
arch/x86/kernel/cpu/perf_event.c +3
-0+if (is_ht_workaround_enabled () && !cpuc->is_fake && +READ_ONCE (cpuc->excl_cntrs->exclusive_present)) + gpmax /=2 ;
Quoted 2 lines
Guess what that
READ_ONCE()does? It checks if there's one such leaky event on the current core.
Great. Thanks.
-Andi
Signature · 1 line
[email protected] -- Speaking for myself only.
Peter Zijlstra
maintainerPATCH v2 3/11 perf/x86: Correct local vs remote sibling state
In reply to Peter ZijlstraFor some obscure reason the current code accounts the current SMT thread's state on the remote thread and reads the remote's state on the local SMT thread.
While internally consistent, and 'correct' its pointless confusion we can do without.
Flip them the right way around.
--- arch/x86/kernel/cpu/perf_event_intel.c | 79 +++++++++++++-------------------- 1 file changed, 33 insertions(+), 46 deletions(-)
arch/x86/kernel/cpu/perf_event_intel.c +33 -46
@@ -1903,9 +1903,8 @@ static void intel_start_scheduling(struct cpu_hw_events *cpuc) {struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs; -struct intel_excl_states *xl, *xlo; +struct intel_excl_states *xl;int tid = cpuc->excl_thread_id; -int o_tid =1 - tid;/* sibling thread */ /* * nothing needed if in group validation mode
@@ -1919,7 +1918,6 @@ intel_start_scheduling(struct cpu_hw_eveif (!excl_cntrs)return ; - xlo = &excl_cntrs->states[o_tid]; xl = &excl_cntrs->states[tid]; xl->sched_started = true;
@@ -1932,18 +1930,17 @@ intel_start_scheduling(struct cpu_hw_everaw_spin_lock (&excl_cntrs->lock);/* -* save initial state of sibling thread +* Save a copy of our state to work on. */ -memcpy (xlo->init_state, xlo->state,sizeof (xlo->init_state)); +memcpy (xl->init_state, xl->state,sizeof (xl->init_state)); }static void intel_stop_scheduling (struct cpu_hw_events *cpuc) {struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs; -struct intel_excl_states *xl, *xlo; +struct intel_excl_states *xl;int tid = cpuc->excl_thread_id; -int o_tid =1 - tid;/* sibling thread */ /* * nothing needed if in group validation mode
@@ -1956,13 +1953,12 @@ intel_stop_scheduling(struct cpu_hw_evenif (!excl_cntrs)return ; - xlo = &excl_cntrs->states[o_tid]; xl = &excl_cntrs->states[tid];/* -* make new sibling thread state visible +* Commit the working state. */ -memcpy (xlo->state, xlo->init_state,sizeof (xlo->state)); +memcpy (xl->state, xl->init_state,sizeof (xl->state)); xl->sched_started = false;/*
@@ -1977,10 +1973,9 @@ intel_get_excl_constraints(struct cpu_hw {struct event_constraint *cx;struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs; -struct intel_excl_states *xl, *xlo; -int is_excl, i; +struct intel_excl_states *xlo;int tid = cpuc->excl_thread_id; -int o_tid =1 - tid;/* alternate */ +int is_excl, i;/* * validating a group does not require
@@ -1994,23 +1989,6 @@ intel_get_excl_constraints(struct cpu_hw*/ if (!excl_cntrs)return c; -/* -* event requires exclusive counter access -* across HT threads -*/ - is_excl = c->flags &PERF_X86_EVENT_EXCL ; -if (is_excl && !(event->hw.flags &PERF_X86_EVENT_EXCL_ACCT )) { - event->hw.flags |=PERF_X86_EVENT_EXCL_ACCT ; -if (!cpuc->n_excl++) -WRITE_ONCE (excl_cntrs->has_exclusive[tid],1 ); - } - -/* -* xl = state of current HT -* xlo = state of sibling HT -*/ - xl = &excl_cntrs->states[tid]; - xlo = &excl_cntrs->states[o_tid]; cx = c;
@@ -2054,6 +2032,22 @@ intel_get_excl_constraints(struct cpu_hw*/ /* +* state of sibling HT +*/ + xlo = &excl_cntrs->states[tid ^1 ]; + +/* +* event requires exclusive counter access +* across HT threads +*/ + is_excl = c->flags &PERF_X86_EVENT_EXCL ; +if (is_excl && !(event->hw.flags &PERF_X86_EVENT_EXCL_ACCT )) { + event->hw.flags |=PERF_X86_EVENT_EXCL_ACCT ; +if (!cpuc->n_excl++) +WRITE_ONCE (excl_cntrs->has_exclusive[tid],1 ); + } + +/* * Modify static constraint with current dynamic * state of thread *
@@ -2067,14 +2061,14 @@ intel_get_excl_constraints(struct cpu_hw* our corresponding counter cannot be used * regardless of our event */ -if (xl->state[i] ==INTEL_EXCL_EXCLUSIVE ) +if (xlo->state[i] ==INTEL_EXCL_EXCLUSIVE )__clear_bit (i, cx->idxmsk);/* * if measuring an exclusive event, sibling * measuring non-exclusive, then counter cannot * be used */ -if (is_excl && xl->state[i] ==INTEL_EXCL_SHARED ) +if (is_excl && xlo->state[i] ==INTEL_EXCL_SHARED )__clear_bit (i, cx->idxmsk); }
@@ -2124,10 +2118,9 @@ static void intel_put_excl_constraints(s {struct hw_perf_event *hwc = &event->hw;struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs; -struct intel_excl_states *xlo, *xl; -unsigned long flags =0 ;/* keep compiler happy */ int tid = cpuc->excl_thread_id; -int o_tid =1 - tid; +struct intel_excl_states *xl; +unsigned long flags =0 ;/* keep compiler happy */ /* * nothing needed if in group validation mode
@@ -2141,7 +2134,6 @@ static void intel_put_excl_constraints(sreturn ; xl = &excl_cntrs->states[tid]; - xlo = &excl_cntrs->states[o_tid];if (hwc->flags &PERF_X86_EVENT_EXCL_ACCT ) { hwc->flags &= ~PERF_X86_EVENT_EXCL_ACCT ;if (!--cpuc->n_excl)
@@ -2161,7 +2153,7 @@ static void intel_put_excl_constraints(s* counter state as unused now */ if (hwc->idx >=0 ) - xlo->state[hwc->idx] =INTEL_EXCL_UNUSED ; + xl->state[hwc->idx] =INTEL_EXCL_UNUSED ;if (!xl->sched_started)raw_spin_unlock_irqrestore (&excl_cntrs->lock, flags);
@@ -2200,16 +2192,12 @@ static void intel_commit_scheduling(stru {struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;struct event_constraint *c = cpuc->event_constraint[idx]; -struct intel_excl_states *xlo, *xl; +struct intel_excl_states *xl;int tid = cpuc->excl_thread_id; -int o_tid =1 - tid; -int is_excl;if (cpuc->is_fake || !c)return ; - is_excl = c->flags &PERF_X86_EVENT_EXCL ; -if (!(c->flags &PERF_X86_EVENT_DYNAMIC ))return ;
@@ -2219,15 +2207,14 @@ static void intel_commit_scheduling(strureturn ; xl = &excl_cntrs->states[tid]; - xlo = &excl_cntrs->states[o_tid];WARN_ON_ONCE (!raw_spin_is_locked (&excl_cntrs->lock));if (cntr >=0 ) { -if (is_excl) - xlo->init_state[cntr] =INTEL_EXCL_EXCLUSIVE ; +if (c->flags &PERF_X86_EVENT_EXCL ) + xl->init_state[cntr] =INTEL_EXCL_EXCLUSIVE ;else - xlo->init_state[cntr] =INTEL_EXCL_SHARED ; + xl->init_state[cntr] =INTEL_EXCL_SHARED ; } }
Re: [PATCH v2 03/11] perf/x86: Correct local vs remote sibling state
Quoted 112 lines · Peter Zijlstra
For some obscure reason the current code accounts the current SMT thread's state on the remote thread and reads the remote's state on the local SMT thread.
While internally consistent, and 'correct' its pointless confusion we can do without.
Flip them the right way around.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>--- arch/x86/kernel/cpu/perf_event_intel.c | 79 +++++++++++++-------------------- 1 file changed, 33 insertions(+), 46 deletions(-)arch/x86/kernel/cpu/perf_event_intel.c +8
-30@@ -1903,9 +1903,8 @@ static void intel_start_scheduling(struct cpu_hw_events *cpuc) {struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs; -struct intel_excl_states *xl, *xlo; +struct intel_excl_states *xl;int tid = cpuc->excl_thread_id; -int o_tid =1 - tid;/* sibling thread */ /* * nothing needed if in group validation mode @@ -1919,7 +1918,6 @@ intel_start_scheduling(struct cpu_hw_eveif (!excl_cntrs)return ; - xlo = &excl_cntrs->states[o_tid]; xl = &excl_cntrs->states[tid]; xl->sched_started = true;@@ -1932,18 +1930,17 @@ intel_start_scheduling(struct cpu_hw_everaw_spin_lock (&excl_cntrs->lock);/* -* save initial state of sibling thread +* Save a copy of our state to work on. */ -memcpy (xlo->init_state, xlo->state,sizeof (xlo->init_state)); +memcpy (xl->init_state, xl->state,sizeof (xl->init_state)); }static void intel_stop_scheduling (struct cpu_hw_events *cpuc) {struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs; -struct intel_excl_states *xl, *xlo; +struct intel_excl_states *xl;int tid = cpuc->excl_thread_id; -int o_tid =1 - tid;/* sibling thread */ /* * nothing needed if in group validation mode @@ -1956,13 +1953,12 @@ intel_stop_scheduling(struct cpu_hw_evenif (!excl_cntrs)return ; - xlo = &excl_cntrs->states[o_tid]; xl = &excl_cntrs->states[tid];/* -* make new sibling thread state visible +* Commit the working state. */ -memcpy (xlo->state, xlo->init_state,sizeof (xlo->state)); +memcpy (xl->state, xl->init_state,sizeof (xl->state)); xl->sched_started = false;/* @@ -1977,10 +1973,9 @@ intel_get_excl_constraints(struct cpu_hw {struct event_constraint *cx;struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs; -struct intel_excl_states *xl, *xlo; -int is_excl, i; +struct intel_excl_states *xlo;int tid = cpuc->excl_thread_id; -int o_tid =1 - tid;/* alternate */ +int is_excl, i;/* * validating a group does not require @@ -1994,23 +1989,6 @@ intel_get_excl_constraints(struct cpu_hw*/ if (!excl_cntrs)return c; -/* -* event requires exclusive counter access -* across HT threads -*/ - is_excl = c->flags &PERF_X86_EVENT_EXCL ; -if (is_excl && !(event->hw.flags &PERF_X86_EVENT_EXCL_ACCT )) { - event->hw.flags |=PERF_X86_EVENT_EXCL_ACCT ; -if (!cpuc->n_excl++) -WRITE_ONCE (excl_cntrs->has_exclusive[tid],1 ); - } - -/* -* xl = state of current HT -* xlo = state of sibling HT -*/ - xl = &excl_cntrs->states[tid]; - xlo = &excl_cntrs->states[o_tid]; cx = c;@@ -2054,6 +2032,22 @@ intel_get_excl_constraints(struct cpu_hw*/
arch/x86/kernel/cpu/perf_event_intel.c intel_get_excl_constraints(struct cpu_hw · line 2042
Quoted 10 lines
arch/x86/kernel/cpu/perf_event_intel.c +8
-0/* +* state of sibling HT +*/ + xlo = &excl_cntrs->states[tid ^1 ]; + +/* +* event requires exclusive counter access +* across HT threads +*/
I think the comment needs to be changed to reflect what the test is doing. I would say:
/*
* account for exclusive counter usage. Needed to avoid
* cross thread counter starvation problem with exclusive events. */
Quoted 97 lines
arch/x86/kernel/cpu/perf_event_intel.c +17
-16+ is_excl = c->flags &PERF_X86_EVENT_EXCL ; +if (is_excl && !(event->hw.flags &PERF_X86_EVENT_EXCL_ACCT )) { + event->hw.flags |=PERF_X86_EVENT_EXCL_ACCT ; +if (!cpuc->n_excl++) +WRITE_ONCE (excl_cntrs->has_exclusive[tid],1 ); + } + +/* * Modify static constraint with current dynamic * state of thread * @@ -2067,14 +2061,14 @@ intel_get_excl_constraints(struct cpu_hw* our corresponding counter cannot be used * regardless of our event */ -if (xl->state[i] ==INTEL_EXCL_EXCLUSIVE ) +if (xlo->state[i] ==INTEL_EXCL_EXCLUSIVE )__clear_bit (i, cx->idxmsk);/* * if measuring an exclusive event, sibling * measuring non-exclusive, then counter cannot * be used */ -if (is_excl && xl->state[i] ==INTEL_EXCL_SHARED ) +if (is_excl && xlo->state[i] ==INTEL_EXCL_SHARED )__clear_bit (i, cx->idxmsk); }@@ -2124,10 +2118,9 @@ static void intel_put_excl_constraints(s {struct hw_perf_event *hwc = &event->hw;struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs; -struct intel_excl_states *xlo, *xl; -unsigned long flags =0 ;/* keep compiler happy */ int tid = cpuc->excl_thread_id; -int o_tid =1 - tid; +struct intel_excl_states *xl; +unsigned long flags =0 ;/* keep compiler happy */ /* * nothing needed if in group validation mode @@ -2141,7 +2134,6 @@ static void intel_put_excl_constraints(sreturn ; xl = &excl_cntrs->states[tid]; - xlo = &excl_cntrs->states[o_tid];if (hwc->flags &PERF_X86_EVENT_EXCL_ACCT ) { hwc->flags &= ~PERF_X86_EVENT_EXCL_ACCT ;if (!--cpuc->n_excl)@@ -2161,7 +2153,7 @@ static void intel_put_excl_constraints(s* counter state as unused now */ if (hwc->idx >=0 ) - xlo->state[hwc->idx] =INTEL_EXCL_UNUSED ; + xl->state[hwc->idx] =INTEL_EXCL_UNUSED ;if (!xl->sched_started)raw_spin_unlock_irqrestore (&excl_cntrs->lock, flags);@@ -2200,16 +2192,12 @@ static void intel_commit_scheduling(stru {struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;struct event_constraint *c = cpuc->event_constraint[idx]; -struct intel_excl_states *xlo, *xl; +struct intel_excl_states *xl;int tid = cpuc->excl_thread_id; -int o_tid =1 - tid; -int is_excl;if (cpuc->is_fake || !c)return ; - is_excl = c->flags &PERF_X86_EVENT_EXCL ; -if (!(c->flags &PERF_X86_EVENT_DYNAMIC ))return ;@@ -2219,15 +2207,14 @@ static void intel_commit_scheduling(strureturn ; xl = &excl_cntrs->states[tid]; - xlo = &excl_cntrs->states[o_tid];WARN_ON_ONCE (!raw_spin_is_locked (&excl_cntrs->lock));if (cntr >=0 ) { -if (is_excl) - xlo->init_state[cntr] =INTEL_EXCL_EXCLUSIVE ; +if (c->flags &PERF_X86_EVENT_EXCL ) + xl->init_state[cntr] =INTEL_EXCL_EXCLUSIVE ;else - xlo->init_state[cntr] =INTEL_EXCL_SHARED ; + xl->init_state[cntr] =INTEL_EXCL_SHARED ; } }
Peter Zijlstra
maintainerPATCH v2 4/11 perf/x86: Use lockdep
In reply to Peter ZijlstraLockdep is very good at finding incorrect IRQ state while locking and is far better at telling us if we hold a lock than the _is_locked() API. It also generates less code for the !DEBUG kernels.
--- arch/x86/kernel/cpu/perf_event_intel.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
arch/x86/kernel/cpu/perf_event_intel.c +1 -2
@@ -1926,7 +1926,6 @@ intel_start_scheduling(struct cpu_hw_eve* in stop_event_scheduling() * makes scheduling appear as a transaction */ -WARN_ON_ONCE (!irqs_disabled ());raw_spin_lock (&excl_cntrs->lock);/*
@@ -2198,7 +2197,7 @@ static void intel_commit_scheduling(stru xl = &excl_cntrs->states[tid]; -WARN_ON_ONCE (!raw_spin_is_locked (&excl_cntrs->lock)); +lockdep_assert_held (&excl_cntrs->lock);if (cntr >=0 ) {if (c->flags &PERF_X86_EVENT_EXCL )
Peter Zijlstra
maintainerPATCH v2 5/11 perf/x86: Simplify dynamic constraint code somewhat
In reply to Peter ZijlstraInstead of using cx after the dynamic allocation, put all cx inside the dynamic allocation block and use c outside of it.
Also use direct assignment to copy the structure; let the compiler figure it out.
--- arch/x86/kernel/cpu/perf_event_intel.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-)
arch/x86/kernel/cpu/perf_event_intel.c +10 -11
@@ -1970,7 +1970,6 @@ static struct event_constraint * intel_get_excl_constraints(struct cpu_hw_events *cpuc, struct perf_event *event, int idx, struct event_constraint *c) { - struct event_constraint *cx; struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;struct intel_excl_states *xlo;int tid = cpuc->excl_thread_id;
@@ -1989,8 +1988,6 @@ intel_get_excl_constraints(struct cpu_hwif (!excl_cntrs)return c; - cx = c; -/* * because we modify the constraint, we need * to make a copy. Static constraints come
@@ -2000,6 +1997,7 @@ intel_get_excl_constraints(struct cpu_hw* been cloned (marked dynamic) */ if (!(c->flags &PERF_X86_EVENT_DYNAMIC )) { +struct event_constraint *cx;/* sanity check */ if (idx <0 )
@@ -2014,13 +2012,14 @@ intel_get_excl_constraints(struct cpu_hw* initialize dynamic constraint * with static constraint */ -memcpy (cx, c,sizeof (*cx)); + *cx = *c;/* * mark constraint as dynamic, so we * can free it later on */ cx->flags |=PERF_X86_EVENT_DYNAMIC ; + c = cx; }/*
@@ -2049,37 +2048,37 @@ intel_get_excl_constraints(struct cpu_hw* SHARED : sibling counter measuring non-exclusive event * UNUSED : sibling counter unused */ -for_each_set_bit (i, cx->idxmsk,X86_PMC_IDX_MAX ) { +for_each_set_bit (i, c->idxmsk,X86_PMC_IDX_MAX ) {/* * exclusive event in sibling counter * our corresponding counter cannot be used * regardless of our event */ if (xlo->state[i] ==INTEL_EXCL_EXCLUSIVE ) -__clear_bit (i, cx->idxmsk); +__clear_bit (i, c->idxmsk);/* * if measuring an exclusive event, sibling * measuring non-exclusive, then counter cannot * be used */ if (is_excl && xlo->state[i] ==INTEL_EXCL_SHARED ) -__clear_bit (i, cx->idxmsk); +__clear_bit (i, c->idxmsk); }/* * recompute actual bit weight for scheduling algorithm */ - cx->weight =hweight64 (cx->idxmsk64); + c->weight =hweight64 (c->idxmsk64);/* * if we return an empty mask, then switch * back to static empty constraint to avoid * the cost of freeing later on */ -if (cx->weight ==0 ) - cx = &emptyconstraint; +if (c->weight ==0 ) + c = &emptyconstraint; -return cx; +return c; } static struct event_constraint *
Peter Zijlstra
maintainerPATCH v2 6/11 perf/x86: Make WARNs consistent
In reply to Peter ZijlstraThe commit_scheduling hook is pointlessly different from the start and stop scheduling hook.
Furthermore, the constraint should never be NULL, so remove that test. Even though we'll never get called (because we NULL the callbacks) when !is_ht_workaround_enabled() put that test in.
Collapse the (pointless) WARN_ON_ONCE and bail on !cpuc->excl_cntrs -- this is doubly pointless, because its the same condition as is_ht_workaround_enabled() which was already pointless because the whole method won't ever be called.
Furthremore, make all the !excl_cntrs test WARN_ON_ONCE; they're all pointless, because the above, either the function ({get,put}_excl_constraint) are already predicated on it existing or the is_ht_workaround_enabled() thing is the same test.
--- arch/x86/kernel/cpu/perf_event_intel.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-)
arch/x86/kernel/cpu/perf_event_intel.c +7 -11
@@ -1915,7 +1915,7 @@ intel_start_scheduling(struct cpu_hw_eve/* * no exclusion needed */ -if (!excl_cntrs) +if (WARN_ON_ONCE (!excl_cntrs))return ; xl = &excl_cntrs->states[tid];
@@ -1949,7 +1949,7 @@ intel_stop_scheduling(struct cpu_hw_even/* * no exclusion needed */ -if (!excl_cntrs) +if (WARN_ON_ONCE (!excl_cntrs))return ; xl = &excl_cntrs->states[tid];
@@ -1985,7 +1985,7 @@ intel_get_excl_constraints(struct cpu_hw/* * no exclusion needed */ -if (!excl_cntrs) +if (WARN_ON_ONCE (!excl_cntrs))return c;/*
@@ -2121,9 +2121,7 @@ static void intel_put_excl_constraints(sif (cpuc->is_fake)return ; -WARN_ON_ONCE (!excl_cntrs); - -if (!excl_cntrs) +if (WARN_ON_ONCE (!excl_cntrs))return ; xl = &excl_cntrs->states[tid];
@@ -2190,15 +2188,13 @@ static void intel_commit_scheduling(strustruct intel_excl_states *xl;int tid = cpuc->excl_thread_id; -if (cpuc->is_fake || !c) +if (cpuc->is_fake || !is_ht_workaround_enabled ())return ; -if (!(c->flags &PERF_X86_EVENT_DYNAMIC )) +if (WARN_ON_ONCE (!excl_cntrs))return ; -WARN_ON_ONCE (!excl_cntrs); - -if (!excl_cntrs) +if (!(c->flags &PERF_X86_EVENT_DYNAMIC ))return ; xl = &excl_cntrs->states[tid];
Peter Zijlstra
maintainerPATCH v2 7/11 perf/x86: Move intel_commit_scheduling()
In reply to Peter ZijlstraPlace intel_commit_scheduling() in the right place, which is in between start and stop.
--- arch/x86/kernel/cpu/perf_event.h | 4 +- arch/x86/kernel/cpu/perf_event_intel.c | 60 ++++++++++++++++----------------- 2 files changed, 32 insertions(+), 32 deletions(-)
arch/x86/kernel/cpu/perf_event.h +2 -2
@@ -520,10 +520,10 @@ struct x86_pmu {void (*put_event_constraints)(struct cpu_hw_events *cpuc,struct perf_event *event); -void (*commit_scheduling)(struct cpu_hw_events *cpuc,int idx,int cntr); -void (*start_scheduling)(struct cpu_hw_events *cpuc); +void (*commit_scheduling)(struct cpu_hw_events *cpuc,int idx,int cntr); +void (*stop_scheduling)(struct cpu_hw_events *cpuc);struct event_constraint *event_constraints;
arch/x86/kernel/cpu/perf_event_intel.c +30 -30
86 lines not on this page · raw message
Peter Zijlstra
maintainerPATCH v2 8/11 perf/x86: Remove pointless tests
In reply to Peter ZijlstraBoth intel_commit_scheduling() and intel_get_excl_contraints() test for cntr < 0.
The only way that can happen (aside from a bug) is through validate_event(), however that is already captured by the cpuc->is_fake test.
So remove these test and simplify the code.
--- arch/x86/kernel/cpu/perf_event_intel.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-)
arch/x86/kernel/cpu/perf_event_intel.c +4 -10
@@ -1955,12 +1955,10 @@ static void intel_commit_scheduling(strulockdep_assert_held (&excl_cntrs->lock); -if (cntr >=0 ) { -if (c->flags &PERF_X86_EVENT_EXCL ) - xl->init_state[cntr] =INTEL_EXCL_EXCLUSIVE ; -else - xl->init_state[cntr] =INTEL_EXCL_SHARED ; - } +if (c->flags &PERF_X86_EVENT_EXCL ) + xl->init_state[cntr] =INTEL_EXCL_EXCLUSIVE ; +else + xl->init_state[cntr] =INTEL_EXCL_SHARED ; } static void
@@ -2028,10 +2026,6 @@ intel_get_excl_constraints(struct cpu_hwif (!(c->flags &PERF_X86_EVENT_DYNAMIC )) {struct event_constraint *cx; -/* sanity check */ -if (idx <0 ) -return &emptyconstraint; -/* * grab pre-allocated constraint entry */
Peter Zijlstra
maintainerPATCH v2 9/11 perf/x86: Remove intel_excl_states::init_state
In reply to Peter ZijlstraFor some obscure reason intel_{start,stop}_scheduling() copy the HT state to an intermediate array. This would make sense if we ever were to make changes to it which we'd have to discard.
Except we don't. By the time we call intel_commit_scheduling() we're; as the name implies; committed to them. We'll never back out.
A further hint its pointless is that stop_scheduling() unconditionally publishes the state.
So the intermediate array is pointless, modify the state in place and kill the extra array.
And remove the pointless array initialization: INTEL_EXCL_UNUSED == 0.
Note; all is serialized by intel_excl_cntr::lock.
--- arch/x86/kernel/cpu/perf_event.c | 1 - arch/x86/kernel/cpu/perf_event.h | 1 - arch/x86/kernel/cpu/perf_event_intel.c | 22 ++-------------------- 3 files changed, 2 insertions(+), 22 deletions(-)
arch/x86/kernel/cpu/perf_event.c +0 -1
@@ -861,7 +861,6 @@ int x86_schedule_events(struct cpu_hw_ev }if (!assign || unsched) { -for (i =0 ; i < n; i++) { e = cpuc->event_list[i];/*
arch/x86/kernel/cpu/perf_event.h +0 -1
@@ -132,7 +132,6 @@ enum intel_excl_state_type { };struct intel_excl_states { -enum intel_excl_state_type init_state[X86_PMC_IDX_MAX ];enum intel_excl_state_type state[X86_PMC_IDX_MAX ];bool sched_started;/* true if scheduling has started */ };
arch/x86/kernel/cpu/perf_event_intel.c +2 -20
@@ -1927,11 +1927,6 @@ intel_start_scheduling(struct cpu_hw_eve* makes scheduling appear as a transaction */ raw_spin_lock (&excl_cntrs->lock); - -/* -* Save a copy of our state to work on. -*/ -memcpy (xl->init_state, xl->state,sizeof (xl->init_state)); }static void intel_commit_scheduling (struct cpu_hw_events *cpuc,int idx,int cntr)
@@ -1955,9 +1950,9 @@ static void intel_commit_scheduling(strulockdep_assert_held (&excl_cntrs->lock);if (c->flags &PERF_X86_EVENT_EXCL ) - xl->init_state[cntr] =INTEL_EXCL_EXCLUSIVE ; + xl->state[cntr] =INTEL_EXCL_EXCLUSIVE ;else - xl->init_state[cntr] =INTEL_EXCL_SHARED ; + xl->state[cntr] =INTEL_EXCL_SHARED ; } static void
@@ -1980,11 +1975,6 @@ intel_stop_scheduling(struct cpu_hw_even xl = &excl_cntrs->states[tid]; -/* -* Commit the working state. -*/ -memcpy (xl->state, xl->init_state,sizeof (xl->state)); - xl->sched_started = false;/* * release shared state lock (acquired in intel_start_scheduling())
@@ -2509,19 +2499,11 @@ struct intel_shared_regs *allocate_sharestatic struct intel_excl_cntrs *allocate_excl_cntrs (int cpu) {struct intel_excl_cntrs *c; -int i; c =kzalloc_node (sizeof (struct intel_excl_cntrs ),GFP_KERNEL ,cpu_to_node (cpu));if (c) {raw_spin_lock_init (&c->lock); -for (i =0 ; i <X86_PMC_IDX_MAX ; i++) { - c->states[0 ].state[i] =INTEL_EXCL_UNUSED ; - c->states[0 ].init_state[i] =INTEL_EXCL_UNUSED ; - - c->states[1 ].state[i] =INTEL_EXCL_UNUSED ; - c->states[1 ].init_state[i] =INTEL_EXCL_UNUSED ; - } c->core_id =-1 ; }return c;
Peter Zijlstra
maintainerPATCH v2 10/11 perf,x86: Simplify logic
In reply to Peter Zijlstra!x && y == ! (x || !y)
--- arch/x86/kernel/cpu/perf_event.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
arch/x86/kernel/cpu/perf_event.c +1 -3
@@ -858,9 +858,7 @@ int x86_schedule_events(struct cpu_hw_evif (x86_pmu.commit_scheduling) x86_pmu.commit_scheduling (cpuc, i, assign[i]); } - } - -if (!assign || unsched) { + }else {for (i =0 ; i < n; i++) { e = cpuc->event_list[i];/*
Peter Zijlstra
maintainerPATCH v2 11/11 perf/x86: Simplify put_exclusive_constraints
In reply to Peter ZijlstraDon't bother with taking locks if we're not actually going to do anything. Also, drop the _irqsave(), this is very much only called from IRQ-disabled context.
--- arch/x86/kernel/cpu/perf_event_intel.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-)
arch/x86/kernel/cpu/perf_event_intel.c +15 -14
@@ -2130,7 +2130,6 @@ static void intel_put_excl_constraints(sstruct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;int tid = cpuc->excl_thread_id;struct intel_excl_states *xl; -unsigned long flags =0 ;/* keep compiler happy */ /* * nothing needed if in group validation mode
@@ -2141,7 +2140,6 @@ static void intel_put_excl_constraints(sif (WARN_ON_ONCE (!excl_cntrs))return ; - xl = &excl_cntrs->states[tid];if (hwc->flags &PERF_X86_EVENT_EXCL_ACCT ) { hwc->flags &= ~PERF_X86_EVENT_EXCL_ACCT ;if (!--cpuc->n_excl)
@@ -2149,22 +2147,25 @@ static void intel_put_excl_constraints(s }/* -* put_constraint may be called from x86_schedule_events() -* which already has the lock held so here make locking -* conditional +* If event was actually assigned, then mark the counter state as +* unused now. */ -if (!xl->sched_started) -raw_spin_lock_irqsave (&excl_cntrs->lock, flags); +if (hwc->idx >=0 ) { + xl = &excl_cntrs->states[tid]; + +/* +* put_constraint may be called from x86_schedule_events() +* which already has the lock held so here make locking +* conditional. +*/ +if (!xl->sched_started) +raw_spin_lock (&excl_cntrs->lock); -/* -* if event was actually assigned, then mark the -* counter state as unused now -*/ -if (hwc->idx >=0 ) xl->state[hwc->idx] =INTEL_EXCL_UNUSED ; -if (!xl->sched_started) -raw_spin_unlock_irqrestore (&excl_cntrs->lock, flags); +if (!xl->sched_started) +raw_spin_unlock (&excl_cntrs->lock); + } } static void
Peter Zijlstra
maintainerRe: [PATCH v2 11/11] perf/x86: Simplify put_exclusive_constraints
Quoted 14 lines · Peter Zijlstra
arch/x86/kernel/cpu/perf_event_intel.c +9
-0@@ -2149,22 +2147,25 @@ static void intel_put_excl_constraints(s }/* +* If event was actually assigned, then mark the counter state as +* unused now. */ +if (hwc->idx >=0 ) { + xl = &excl_cntrs->states[tid]; + +/* +* put_constraint may be called from x86_schedule_events() +* which already has the lock held so here make locking +* conditional.
arch/x86/kernel/cpu/perf_event_intel.c static void intel_put_excl_constraints(s · line 2169
Quoted 10 lines
arch/x86/kernel/cpu/perf_event_intel.c +6
-0+*/ +if (!xl->sched_started) +raw_spin_lock (&excl_cntrs->lock); xl->state[hwc->idx] =INTEL_EXCL_UNUSED ; +if (!xl->sched_started) +raw_spin_unlock (&excl_cntrs->lock); + } }
Hmm, we could probably avoid the lock and use WRITE_ONCE() to clear this state, but I'm too tired to really think about it.