perf fixes
Subsystems (2)
Linus,
Please pull the latest perf-urgent-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git perf-urgent-for-linus
# HEAD: 1d54ad944074010609562da5c89e4f5df2f4e5db perf/core: Fix perf_event_disable_inatomic() race
Six kernel side fixes: three related to NMI handling on AMD systems, a race fix, a kexec initialization fix and a PEBS sampling fix.
Thanks,
Ingo
------------------>
Lendacky, Thomas (3):
x86/perf/amd: Resolve race condition when disabling PMC x86/perf/amd: Resolve NMI latency issues for active PMCs x86/perf/amd: Remove need to check "running" bit in NMI handler
Peter Zijlstra (2):
perf/x86/intel: Initialize TFA MSR perf/core: Fix perf_event_disable_inatomic() race
Stephane Eranian (1):
perf/x86/intel: Fix handling of wakeup_events for multi-entry PEBS
arch/x86/events/amd/core.c | 140 +++++++++++++++++++++++++++++++++++++++++-- arch/x86/events/core.c | 13 +--- arch/x86/events/intel/core.c | 8 ++- kernel/events/core.c | 52 +++++++++++++--- kernel/events/ring_buffer.c | 4 +- 5 files changed, 190 insertions(+), 27 deletions(-)
arch/x86/events/amd/core.c +135 -5
@@ -3,10 +3,14 @@#include <linux/types.h> #include <linux/init.h> #include <linux/slab.h> +#include <linux/delay.h> #include <asm/apicdef.h> +#include <asm/nmi.h> #include "../perf_event.h" +static DEFINE_PER_CPU(unsigned int, perf_nmi_counter); +static __initconst const u64 amd_hw_cache_event_ids [PERF_COUNT_HW_CACHE_MAX ] [PERF_COUNT_HW_CACHE_OP_MAX ]
@@ -429,6 +433,132 @@ static void amd_pmu_cpu_dead(int cpu) } } +/* +* When a PMC counter overflows, an NMI is used to process the event and +* reset the counter. NMI latency can result in the counter being updated +* before the NMI can run, which can result in what appear to be spurious +* NMIs. This function is intended to wait for the NMI to run and reset +* the counter to avoid possible unhandled NMI messages. +*/ +#define OVERFLOW_WAIT_COUNT 50 + +static void amd_pmu_wait_on_overflow (int idx) +{ +unsigned int i; +u64 counter; + +/* +* Wait for the counter to be reset if it has overflowed. This loop +* should exit very, very quickly, but just in case, don't wait +* forever... +*/ +for (i =0 ; i <OVERFLOW_WAIT_COUNT ; i++) { +rdmsrl (x86_pmu_event_addr (idx), counter); +if (counter & (1ULL << (x86_pmu.cntval_bits -1 ))) +break ; + +/* Might be in IRQ context, so can't sleep */ +udelay (1 ); + } +} + +static void amd_pmu_disable_all (void ) +{ +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); +int idx; + +x86_pmu_disable_all (); + +/* +* This shouldn't be called from NMI context, but add a safeguard here +* to return, since if we're in NMI context we can't wait for an NMI +* to reset an overflowed counter value. +*/ +if (in_nmi ()) +return ; + +/* +* Check each counter for overflow and wait for it to be reset by the +* NMI if it has overflowed. This relies on the fact that all active +* counters are always enabled when this function is caled and +* ARCH_PERFMON_EVENTSEL_INT is always set. +*/ +for (idx =0 ; idx < x86_pmu.num_counters; idx++) { +if (!test_bit (idx, cpuc->active_mask)) +continue ; + +amd_pmu_wait_on_overflow (idx); + } +} + +static void amd_pmu_disable_event (struct perf_event *event) +{ +x86_pmu_disable_event (event); + +/* +* This can be called from NMI context (via x86_pmu_stop). The counter +* may have overflowed, but either way, we'll never see it get reset +* by the NMI if we're already in the NMI. And the NMI latency support +* below will take care of any pending NMI that might have been +* generated by the overflow. +*/ +if (in_nmi ()) +return ; + +amd_pmu_wait_on_overflow (event->hw.idx); +} + +/* +* Because of NMI latency, if multiple PMC counters are active or other sources +* of NMIs are received, the perf NMI handler can handle one or more overflowed +* PMC counters outside of the NMI associated with the PMC overflow. If the NMI +* doesn't arrive at the LAPIC in time to become a pending NMI, then the kernel +* back-to-back NMI support won't be active. This PMC handler needs to take into +* account that this can occur, otherwise this could result in unknown NMI +* messages being issued. Examples of this is PMC overflow while in the NMI +* handler when multiple PMCs are active or PMC overflow while handling some +* other source of an NMI. +* +* Attempt to mitigate this by using the number of active PMCs to determine +* whether to return NMI_HANDLED if the perf NMI handler did not handle/reset +* any PMCs. The per-CPU perf_nmi_counter variable is set to a minimum of the +* number of active PMCs or 2. The value of 2 is used in case an NMI does not +* arrive at the LAPIC in time to be collapsed into an already pending NMI. +*/ +static int amd_pmu_handle_irq (struct pt_regs *regs) +{ +struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events); +int active, handled; + +/* +* Obtain the active count before calling x86_pmu_handle_irq() since +* it is possible that x86_pmu_handle_irq() may make a counter +* inactive (through x86_pmu_stop). +*/ + active =__bitmap_weight (cpuc->active_mask,X86_PMC_IDX_MAX ); + +/* Process any counter overflows */ + handled =x86_pmu_handle_irq (regs); + +/* +* If a counter was handled, record the number of possible remaining +* NMIs that can occur. +*/ +if (handled) { +this_cpu_write (perf_nmi_counter, + min_t(unsigned int, 2, active)); + +return handled; + } + +if (!this_cpu_read (perf_nmi_counter)) +return NMI_DONE ; + +this_cpu_dec (perf_nmi_counter); + +return NMI_HANDLED ; +} +static struct event_constraint *amd_get_event_constraints (struct cpu_hw_events *cpuc,int idx,struct perf_event *event)
@@ -621,11 +751,11 @@ static ssize_t amd_event_sysfs_show(char *page, u64 config)static __initconst const struct x86_pmu amd_pmu = { .name ="AMD" , - .handle_irq = x86_pmu_handle_irq, - .disable_all = x86_pmu_disable_all, + .handle_irq = amd_pmu_handle_irq, + .disable_all = amd_pmu_disable_all, .enable_all = x86_pmu_enable_all, .enable = x86_pmu_enable_event, - .disable = x86_pmu_disable_event, + .disable = amd_pmu_disable_event, .hw_config = amd_pmu_hw_config, .schedule_events = x86_schedule_events, .eventsel =MSR_K7_EVNTSEL0 ,
@@ -732,7 +862,7 @@ void amd_pmu_enable_virt(void) cpuc->perf_ctr_virt_mask =0 ;/* Reload all events */ -x86_pmu_disable_all (); +amd_pmu_disable_all ();x86_pmu_enable_all (0 ); }EXPORT_SYMBOL_GPL (amd_pmu_enable_virt);
@@ -750,7 +880,7 @@ void amd_pmu_disable_virt(void) cpuc->perf_ctr_virt_mask =AMD64_EVENTSEL_HOSTONLY ;/* Reload all events */ -x86_pmu_disable_all (); +amd_pmu_disable_all ();x86_pmu_enable_all (0 ); }EXPORT_SYMBOL_GPL (amd_pmu_disable_virt);
arch/x86/events/core.c +3 -10
@@ -1349,8 +1349,9 @@ void x86_pmu_stop(struct perf_event *event, int flags)struct cpu_hw_events *cpuc =this_cpu_ptr (&cpu_hw_events);struct hw_perf_event *hwc = &event->hw; -if (__test_and_clear_bit (hwc->idx, cpuc->active_mask)) { +if (test_bit (hwc->idx, cpuc->active_mask)) { x86_pmu.disable (event); +__clear_bit (hwc->idx, cpuc->active_mask); cpuc->events[hwc->idx] =NULL ;WARN_ON_ONCE (hwc->state &PERF_HES_STOPPED ); hwc->state |=PERF_HES_STOPPED ;
@@ -1447,16 +1448,8 @@ int x86_pmu_handle_irq(struct pt_regs *regs)apic_write (APIC_LVTPC ,APIC_DM_NMI );for (idx =0 ; idx < x86_pmu.num_counters; idx++) { -if (!test_bit (idx, cpuc->active_mask)) { -/* -* Though we deactivated the counter some cpus -* might still deliver spurious interrupts still -* in flight. Catch them: -*/ -if (__test_and_clear_bit (idx, cpuc->running)) - handled++; +if (!test_bit (idx, cpuc->active_mask))continue ; - } event = cpuc->events[idx];
arch/x86/events/intel/core.c +7 -1
@@ -3185,7 +3185,7 @@ static int intel_pmu_hw_config(struct perf_event *event)return ret;if (event->attr.precise_ip) { -if (!event->attr.freq) { +if (!(event->attr.freq || event->attr.wakeup_events)) { event->hw.flags |=PERF_X86_EVENT_AUTO_RELOAD ;if (!(event->attr.sample_type & ~intel_pmu_large_pebs_flags (event)))
@@ -3575,6 +3575,12 @@ static void intel_pmu_cpu_starting(int cpu) cpuc->lbr_sel =NULL ; +if (x86_pmu.flags &PMU_FL_TFA ) { +WARN_ON_ONCE (cpuc->tfa_shadow); + cpuc->tfa_shadow = ~0ULL ; +intel_set_tfa (cpuc, false); + } +if (x86_pmu.version >1 )flip_smm_bit (&x86_pmu.attr_freeze_on_smi);
kernel/events/core.c +43 -9
84 lines not on this page · raw message
kernel/events/ring_buffer.c +2 -2
@@ -392,7 +392,7 @@ void *perf_aux_output_begin(struct perf_output_handle *handle,* store that will be enabled on successful return */ if (!handle->size) {/* A, matches D */ - event->pending_disable =1 ; + event->pending_disable =smp_processor_id ();perf_output_wakeup (handle);local_set (&rb->aux_nest,0 ); goto err_put;
@@ -480,7 +480,7 @@ void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size)if (wakeup) {if (handle->aux_flags &PERF_AUX_FLAG_TRUNCATED ) - handle->event->pending_disable =1 ; + handle->event->pending_disable =smp_processor_id ();perf_output_wakeup (handle); }
The pull request you sent on Fri, 12 Apr 2019 15:06:42 +0200:
The pull request you sent on Fri, 12 Apr 2019 15:06:42 +0200:
Quoted 1 line
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.gitperf-urgent-for-linus
has been merged into torvalds/linux.git:
https://git.kernel.org/torvalds/c/73fdb2c908c64a32e11c72c029d636f556859c0d
Thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.wiki.kernel.org/userdoc/prtracker