arch-PEBS enabling for Intel platforms

When intel_pmu_drain_pebs_icl() is called to drain PEBS records, the perf_event_overflow() could be called to process the last PEBS record.

While perf_event_overflow() could trigger the interrupt throttle and stop all events of the group, like what the below call-chain shows.

perf_event_overflow()
-> __perf_event_overflow()

    ->__perf_event_account_interrupt()
      -> perf_event_throttle_group()
        -> perf_event_throttle()
          -> event->pmu->stop()
            -> x86_pmu_stop()

The side effect of stopping the events is that all corresponding event pointers in cpuc->events[] array are cleared to NULL.

Assume there are two PEBS events (event a and event b) in a group. When intel_pmu_drain_pebs_icl() calls perf_event_overflow() to process the last PEBS record of PEBS event a, interrupt throttle is triggered and all pointers of event a and event b are cleared to NULL. Then intel_pmu_drain_pebs_icl() tries to process the last PEBS record of event b and encounters NULL pointer access.

To avoid this issue, move cpuc->events[] clearing from x86_pmu_stop() to x86_pmu_del(). It's safe since cpuc->active_mask or cpuc->pebs_enabled is always checked before access the event pointer from cpuc->events[].

Reported-by kernel test robot <[email protected]>

Closes https://lore.kernel.org/oe-lkp/[email protected] open here

Fixes 9734e25fbf5a ("perf: Fix the throttle logic for a group")

Suggested-by Peter Zijlstra <[email protected]>

Signed-off-by Dapeng Mi <[email protected]>

---
 arch/x86/events/core.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

arch/x86/events/core.c +3 -2

index 745caa6c15a3..74479f9d6eed 100644
@@ -1344,6 +1344,7 @@ static void x86_pmu_enable(struct pmu *pmu)
 				hwc->state |= PERF_HES_ARCH;
 
 			x86_pmu_stop(event, PERF_EF_UPDATE);
+			cpuc->events[hwc->idx] = NULL;
 		}
 
 		/*

@@ -1365,6 +1366,7 @@ static void x86_pmu_enable(struct pmu *pmu)
 			 * if cpuc->enabled = 0, then no wrmsr as
 			 * per x86_pmu_enable_event()
 			 */
+			cpuc->events[hwc->idx] = event;
 			x86_pmu_start(event, PERF_EF_RELOAD);
 		}
 		cpuc->n_added = 0;

@@ -1531,7 +1533,6 @@ static void x86_pmu_start(struct perf_event *event, int flags)
 
 	event->hw.state = 0;
 
-	cpuc->events[idx] = event;
 	__set_bit(idx, cpuc->active_mask);
 	static_call(x86_pmu_enable)(event);
 	perf_event_update_userpage(event);

@@ -1610,7 +1611,6 @@ void x86_pmu_stop(struct perf_event *event, int flags)
 	if (test_bit(hwc->idx, cpuc->active_mask)) {
 		static_call(x86_pmu_disable)(event);
 		__clear_bit(hwc->idx, cpuc->active_mask);
-		cpuc->events[hwc->idx] = NULL;
 		WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
 		hwc->state |= PERF_HES_STOPPED;
 	}

@@ -1648,6 +1648,7 @@ static void x86_pmu_del(struct perf_event *event, int flags)
 	 * Not a TXN, therefore cleanup properly.
 	 */
 	x86_pmu_stop(event, PERF_EF_UPDATE);
+	cpuc->events[event->hw.idx] = NULL;
 
 	for (i = 0; i < cpuc->n_events; i++) {
 		if (event == cpuc->event_list[i])

Signature · 1 line

2.34.1

arch-PEBS leverages CPUID.23H.4/5 sub-leaves enumerate arch-PEBS supported capabilities and counters bitmap. This patch parses these 2 sub-leaves and initializes arch-PEBS capabilities and corresponding structures.

Since IA32_PEBS_ENABLE and MSR_PEBS_DATA_CFG MSRs are no longer existed for arch-PEBS, arch-PEBS doesn't need to manipulate these MSRs. Thus add a simple pair of __intel_pmu_pebs_enable/disable() callbacks for arch-PEBS.

Signed-off-by Dapeng Mi <[email protected]>

---
 arch/x86/events/core.c            | 21 ++++++++---
 arch/x86/events/intel/core.c      | 60 ++++++++++++++++++++++---------
 arch/x86/events/intel/ds.c        | 52 ++++++++++++++++++++++-----
 arch/x86/events/perf_event.h      | 25 +++++++++++--
 arch/x86/include/asm/perf_event.h |  7 +++-
 5 files changed, 132 insertions(+), 33 deletions(-)

arch/x86/events/core.c +16 -5

index 74479f9d6eed..f2402ae3ffa0 100644
@@ -554,14 +554,22 @@ static inline int precise_br_compat(struct perf_event *event)
        return m == b;
 }
 
-int x86_pmu_max_precise(void)
+int x86_pmu_max_precise(struct pmu *pmu)
 {
        int precise = 0;
 
-       /* Support for constant skid */
        if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
-               precise++;
+               /* arch PEBS */
+               if (x86_pmu.arch_pebs) {
+                       precise = 2;
+                       if (hybrid(pmu, arch_pebs_cap).pdists)
+                               precise++;
+
+                       return precise;
+               }
 
+               /* legacy PEBS - support for constant skid */
+               precise++;
                /* Support for IP fixup */
                if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)
                        precise++;

@@ -569,13 +577,14 @@ int x86_pmu_max_precise(void)
                if (x86_pmu.pebs_prec_dist)
                        precise++;
        }
+
        return precise;
 }
 
 int x86_pmu_hw_config(struct perf_event *event)
 {
        if (event->attr.precise_ip) {
-               int precise = x86_pmu_max_precise();
+               int precise = x86_pmu_max_precise(event->pmu);
 
                if (event->attr.precise_ip > precise)
                        return -EOPNOTSUPP;

@@ -2630,7 +2639,9 @@ static ssize_t max_precise_show(struct device *cdev,
                                  struct device_attribute *attr,
                                  char *buf)
 {
-       return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise());
+       struct pmu *pmu = dev_get_drvdata(cdev);
+
+       return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise(pmu));
 }
 
 static DEVICE_ATTR_RO(max_precise);

arch/x86/events/intel/core.c +19 -14

index c88bcd5d2bc4..9ce27b326923 100644
@@ -5271,34 +5271,59 @@ static inline bool intel_pmu_broken_perf_cap(void)
        return false;
 }
 
+#define counter_mask(_gp, _fixed) ((_gp) | ((u64)(_fixed) << INTEL_PMC_IDX_FIXED))
+
 static void update_pmu_cap(struct pmu *pmu)
 {
-       unsigned int cntr, fixed_cntr, ecx, edx;
-       union cpuid35_eax eax;
-       union cpuid35_ebx ebx;
+       unsigned int eax, ebx, ecx, edx;
+       union cpuid35_eax eax_0;
+       union cpuid35_ebx ebx_0;
+       u64 cntrs_mask = 0;
+       u64 pebs_mask = 0;
+       u64 pdists_mask = 0;
 
-       cpuid(ARCH_PERFMON_EXT_LEAF, &eax.full, &ebx.full, &ecx, &edx);
+       cpuid(ARCH_PERFMON_EXT_LEAF, &eax_0.full, &ebx_0.full, &ecx, &edx);
 
-       if (ebx.split.umask2)
+       if (ebx_0.split.umask2)
                hybrid(pmu, config_mask) |= ARCH_PERFMON_EVENTSEL_UMASK2;
-       if (ebx.split.eq)
+       if (ebx_0.split.eq)
                hybrid(pmu, config_mask) |= ARCH_PERFMON_EVENTSEL_EQ;
 
-       if (eax.split.cntr_subleaf) {
+       if (eax_0.split.cntr_subleaf) {
                cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_NUM_COUNTER_LEAF,
-                           &cntr, &fixed_cntr, &ecx, &edx);
-               hybrid(pmu, cntr_mask64) = cntr;
-               hybrid(pmu, fixed_cntr_mask64) = fixed_cntr;
+                           &eax, &ebx, &ecx, &edx);
+               hybrid(pmu, cntr_mask64) = eax;
+               hybrid(pmu, fixed_cntr_mask64) = ebx;
+               cntrs_mask = counter_mask(eax, ebx);
        }
 
-       if (eax.split.acr_subleaf) {
+       if (eax_0.split.acr_subleaf) {
                cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_ACR_LEAF,
-                           &cntr, &fixed_cntr, &ecx, &edx);
+                           &eax, &ebx, &ecx, &edx);
                /* The mask of the counters which can be reloaded */
-               hybrid(pmu, acr_cntr_mask64) = cntr | ((u64)fixed_cntr << INTEL_PMC_IDX_FIXED);
-
+               hybrid(pmu, acr_cntr_mask64) = counter_mask(eax, ebx);

arch/x86/events/intel/core.c +17 -1

+
+               cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_PEBS_COUNTER_LEAF,
+                           &eax, &ebx, &ecx, &edx);
+               pebs_mask   = counter_mask(eax, ecx);
+               pdists_mask = counter_mask(ebx, edx);
+               hybrid(pmu, arch_pebs_cap).counters = pebs_mask;
+               hybrid(pmu, arch_pebs_cap).pdists = pdists_mask;
+
+               if (WARN_ON((pebs_mask | pdists_mask) & ~cntrs_mask))
+                       x86_pmu.arch_pebs = 0;
+       } else {
+               WARN_ON(x86_pmu.arch_pebs == 1);
+               x86_pmu.arch_pebs = 0;
        }
 
        if (!intel_pmu_broken_perf_cap()) {

@@ -6252,7 +6277,7 @@ tsx_is_visible(struct kobject *kobj, struct attribute *attr, int i)
 static umode_t
 pebs_is_visible(struct kobject *kobj, struct attribute *attr, int i)
 {
-       return x86_pmu.ds_pebs ? attr->mode : 0;
+       return intel_pmu_has_pebs() ? attr->mode : 0;
 }
 
 static umode_t

@@ -7728,6 +7753,9 @@ __init int intel_pmu_init(void)
        if (!is_hybrid() && boot_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT))
                update_pmu_cap(NULL);
 
+       if (x86_pmu.arch_pebs)
+               pr_cont("Architectural PEBS, ");
+
        intel_pmu_check_counters_mask(&x86_pmu.cntr_mask64,
                                      &x86_pmu.fixed_cntr_mask64,
                                      &x86_pmu.intel_ctrl);

arch/x86/events/intel/ds.c +44 -8

index c0b7ac1c7594..26e485eca0a0 100644
@@ -1531,6 +1531,15 @@ static inline void intel_pmu_drain_large_pebs(struct cpu_hw_events *cpuc)
                intel_pmu_drain_pebs_buffer();
 }
 
+static void __intel_pmu_pebs_enable(struct perf_event *event)
+{
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+       struct hw_perf_event *hwc = &event->hw;
+
+       hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
+       cpuc->pebs_enabled |= 1ULL << hwc->idx;
+}
+
 void intel_pmu_pebs_enable(struct perf_event *event)
 {
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

@@ -1539,9 +1548,7 @@ void intel_pmu_pebs_enable(struct perf_event *event)
        struct debug_store *ds = cpuc->ds;
        unsigned int idx = hwc->idx;
 
-       hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
-
-       cpuc->pebs_enabled |= 1ULL << hwc->idx;
+       __intel_pmu_pebs_enable(event);
 
        if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) && (x86_pmu.version < 5))
                cpuc->pebs_enabled |= 1ULL << (hwc->idx + 32);

@@ -1603,14 +1610,22 @@ void intel_pmu_pebs_del(struct perf_event *event)
        pebs_update_state(needed_cb, cpuc, event, false);
 }
 
-void intel_pmu_pebs_disable(struct perf_event *event)
+static void __intel_pmu_pebs_disable(struct perf_event *event)
 {
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct hw_perf_event *hwc = &event->hw;
 
        intel_pmu_drain_large_pebs(cpuc);
-
        cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
+       hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
+}
+
+void intel_pmu_pebs_disable(struct perf_event *event)
+{
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+       struct hw_perf_event *hwc = &event->hw;
+
+       __intel_pmu_pebs_disable(event);
 
        if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) &&
            (x86_pmu.version < 5))

@@ -1622,8 +1637,6 @@ void intel_pmu_pebs_disable(struct perf_event *event)
 
        if (cpuc->enabled)
                wrmsrq(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
-
-       hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
 }
 
 void intel_pmu_pebs_enable_all(void)

@@ -2669,11 +2682,26 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d
        }
 }
 
+static void __init intel_arch_pebs_init(void)
+{
+       /*
+        * Current hybrid platforms always both support arch-PEBS or not
+        * on all kinds of cores. So directly set x86_pmu.arch_pebs flag
+        * if boot cpu supports arch-PEBS.
+        */
+       x86_pmu.arch_pebs = 1;
+       x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
+       x86_pmu.pebs_capable = ~0ULL;
+
+       x86_pmu.pebs_enable = __intel_pmu_pebs_enable;
+       x86_pmu.pebs_disable = __intel_pmu_pebs_disable;
+}
+
 /*
  * PEBS probe and setup
  */
 
-void __init intel_pebs_init(void)
+static void __init intel_ds_pebs_init(void)
 {
        /*
         * No support for 32bit formats

@@ -2788,6 +2816,14 @@ void __init intel_pebs_init(void)
        }
 }
 
+void __init intel_pebs_init(void)
+{
+       if (x86_pmu.intel_cap.pebs_format == 0xf)
+               intel_arch_pebs_init();
+       else
+               intel_ds_pebs_init();
+}
+
 void perf_restore_debug_store(void)
 {
        struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);

arch/x86/events/perf_event.h +22 -3

index 285779c73479..ca5289980b52 100644
@@ -708,6 +708,12 @@ enum hybrid_pmu_type {
        hybrid_big_small_tiny   = hybrid_big   | hybrid_small_tiny,
 };
 
+struct arch_pebs_cap {
+       u64 caps;
+       u64 counters;
+       u64 pdists;
+};
+
 struct x86_hybrid_pmu {
        struct pmu                      pmu;
        const char                      *name;

@@ -752,6 +758,8 @@ struct x86_hybrid_pmu {
                                        mid_ack         :1,
                                        enabled_ack     :1;
 
+       struct arch_pebs_cap            arch_pebs_cap;
+
        u64                             pebs_data_source[PERF_PEBS_DATA_SOURCE_MAX];
 };
 

@@ -906,7 +914,7 @@ struct x86_pmu {
        union perf_capabilities intel_cap;
 
        /*
-        * Intel DebugStore bits
+        * Intel DebugStore and PEBS bits
         */
        unsigned int    bts                     :1,
                        bts_active              :1,

@@ -917,7 +925,8 @@ struct x86_pmu {
                        pebs_no_tlb             :1,
                        pebs_no_isolation       :1,
                        pebs_block              :1,
-                       pebs_ept                :1;
+                       pebs_ept                :1,
+                       arch_pebs               :1;
        int             pebs_record_size;
        int             pebs_buffer_size;
        u64             pebs_events_mask;

@@ -929,6 +938,11 @@ struct x86_pmu {
        u64             rtm_abort_event;
        u64             pebs_capable;
 
+       /*
+        * Intel Architectural PEBS
+        */
+       struct arch_pebs_cap arch_pebs_cap;
+
        /*
         * Intel LBR
         */

@@ -1216,7 +1230,7 @@ int x86_reserve_hardware(void);
 
 void x86_release_hardware(void);
 
-int x86_pmu_max_precise(void);
+int x86_pmu_max_precise(struct pmu *pmu);
 
 void hw_perf_lbr_event_destroy(struct perf_event *event);
 

@@ -1791,6 +1805,11 @@ static inline int intel_pmu_max_num_pebs(struct pmu *pmu)
        return fls((u32)hybrid(pmu, pebs_events_mask));
 }
 
+static inline bool intel_pmu_has_pebs(void)
+{
+       return x86_pmu.ds_pebs || x86_pmu.arch_pebs;
+}
+
 #else /* CONFIG_CPU_SUP_INTEL */
 
 static inline void reserve_ds_buffers(void)

arch/x86/include/asm/perf_event.h +6 -2

index 49a4d442f3fc..0dfa06722bab 100644
@@ -200,6 +200,8 @@ union cpuid10_edx {
 #define ARCH_PERFMON_EXT_LEAF                  0x00000023
 #define ARCH_PERFMON_NUM_COUNTER_LEAF          0x1
 #define ARCH_PERFMON_ACR_LEAF                  0x2
+#define ARCH_PERFMON_PEBS_CAP_LEAF             0x4
+#define ARCH_PERFMON_PEBS_COUNTER_LEAF         0x5
 
 union cpuid35_eax {
        struct {

@@ -210,7 +212,10 @@ union cpuid35_eax {
                unsigned int    acr_subleaf:1;
                /* Events Sub-Leaf */
                unsigned int    events_subleaf:1;
-               unsigned int    reserved:28;
+               /* arch-PEBS Sub-Leaves */
+               unsigned int    pebs_caps_subleaf:1;
+               unsigned int    pebs_cnts_subleaf:1;
+               unsigned int    reserved:26;
        } split;
        unsigned int            full;
 };
--

2.34.1

Quoted 133 lines · Dapeng Mi

arch-PEBS leverages CPUID.23H.4/5 sub-leaves enumerate arch-PEBS supported capabilities and counters bitmap. This patch parses these 2 sub-leaves and initializes arch-PEBS capabilities and corresponding structures.

Since IA32_PEBS_ENABLE and MSR_PEBS_DATA_CFG MSRs are no longer existed for arch-PEBS, arch-PEBS doesn't need to manipulate these MSRs. Thus add a simple pair of __intel_pmu_pebs_enable/disable() callbacks for arch-PEBS.

Signed-off-by Dapeng Mi <[email protected]>

---
 arch/x86/events/core.c            | 21 ++++++++---
 arch/x86/events/intel/core.c      | 60 ++++++++++++++++++++++---------
 arch/x86/events/intel/ds.c        | 52 ++++++++++++++++++++++-----
 arch/x86/events/perf_event.h      | 25 +++++++++++--
 arch/x86/include/asm/perf_event.h |  7 +++-
 5 files changed, 132 insertions(+), 33 deletions(-)

arch/x86/events/core.c +16 -5

index 74479f9d6eed..f2402ae3ffa0 100644
@@ -554,14 +554,22 @@ static inline int precise_br_compat(struct perf_event *event)
        return m == b;
 }
 
-int x86_pmu_max_precise(void)
+int x86_pmu_max_precise(struct pmu *pmu)
 {
        int precise = 0;
 
-       /* Support for constant skid */
        if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
-               precise++;
+               /* arch PEBS */
+               if (x86_pmu.arch_pebs) {
+                       precise = 2;
+                       if (hybrid(pmu, arch_pebs_cap).pdists)
+                               precise++;
+
+                       return precise;
+               }
 
+               /* legacy PEBS - support for constant skid */
+               precise++;
                /* Support for IP fixup */
                if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)
                        precise++;

@@ -569,13 +577,14 @@ int x86_pmu_max_precise(void)
                if (x86_pmu.pebs_prec_dist)
                        precise++;
        }
+
        return precise;
 }
 
 int x86_pmu_hw_config(struct perf_event *event)
 {
        if (event->attr.precise_ip) {
-               int precise = x86_pmu_max_precise();
+               int precise = x86_pmu_max_precise(event->pmu);
 
                if (event->attr.precise_ip > precise)
                        return -EOPNOTSUPP;

@@ -2630,7 +2639,9 @@ static ssize_t max_precise_show(struct device *cdev,
                                  struct device_attribute *attr,
                                  char *buf)
 {
-       return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise());
+       struct pmu *pmu = dev_get_drvdata(cdev);
+
+       return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise(pmu));
 }
 
 static DEVICE_ATTR_RO(max_precise);

arch/x86/events/intel/core.c +21 -15

index c88bcd5d2bc4..9ce27b326923 100644
@@ -5271,34 +5271,59 @@ static inline bool intel_pmu_broken_perf_cap(void)
        return false;
 }
 
+#define counter_mask(_gp, _fixed) ((_gp) | ((u64)(_fixed) << INTEL_PMC_IDX_FIXED))
+
 static void update_pmu_cap(struct pmu *pmu)
 {
-       unsigned int cntr, fixed_cntr, ecx, edx;
-       union cpuid35_eax eax;
-       union cpuid35_ebx ebx;
+       unsigned int eax, ebx, ecx, edx;
+       union cpuid35_eax eax_0;
+       union cpuid35_ebx ebx_0;
+       u64 cntrs_mask = 0;
+       u64 pebs_mask = 0;
+       u64 pdists_mask = 0;
 
-       cpuid(ARCH_PERFMON_EXT_LEAF, &eax.full, &ebx.full, &ecx, &edx);
+       cpuid(ARCH_PERFMON_EXT_LEAF, &eax_0.full, &ebx_0.full, &ecx, &edx);
 
-       if (ebx.split.umask2)
+       if (ebx_0.split.umask2)
                hybrid(pmu, config_mask) |= ARCH_PERFMON_EVENTSEL_UMASK2;
-       if (ebx.split.eq)
+       if (ebx_0.split.eq)
                hybrid(pmu, config_mask) |= ARCH_PERFMON_EVENTSEL_EQ;
 
-       if (eax.split.cntr_subleaf) {
+       if (eax_0.split.cntr_subleaf) {
                cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_NUM_COUNTER_LEAF,
-                           &cntr, &fixed_cntr, &ecx, &edx);
-               hybrid(pmu, cntr_mask64) = cntr;
-               hybrid(pmu, fixed_cntr_mask64) = fixed_cntr;
+                           &eax, &ebx, &ecx, &edx);
+               hybrid(pmu, cntr_mask64) = eax;
+               hybrid(pmu, fixed_cntr_mask64) = ebx;
+               cntrs_mask = counter_mask(eax, ebx);
        }
 
-       if (eax.split.acr_subleaf) {
+       if (eax_0.split.acr_subleaf) {
                cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_ACR_LEAF,
-                           &cntr, &fixed_cntr, &ecx, &edx);
+                           &eax, &ebx, &ecx, &edx);
                /* The mask of the counters which can be reloaded */
-               hybrid(pmu, acr_cntr_mask64) = cntr | ((u64)fixed_cntr << INTEL_PMC_IDX_FIXED);
-
+               hybrid(pmu, acr_cntr_mask64) = counter_mask(eax, ebx);
                /* The mask of the counters which can cause a reload of reloadable counters */
-               hybrid(pmu, acr_cause_mask64) = ecx | ((u64)edx << INTEL_PMC_IDX_FIXED);
+               hybrid(pmu, acr_cause_mask64) = counter_mask(ecx, edx);
+       }

arch/x86/events/intel/core.c static inline bool intel_pmu_broken_perf_cap(void) · line 5313

Quoted 6 lines

arch/x86/events/intel/core.c +6 -0

+
+       /* Bits[5:4] should be set simultaneously if arch-PEBS is supported */
+       if (eax_0.split.pebs_caps_subleaf && eax_0.split.pebs_cnts_subleaf) {
+               cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_PEBS_CAP_LEAF,
+                           &eax, &ebx, &ecx, &edx);
+               hybrid(pmu, arch_pebs_cap).caps = (u64)ebx << 32;

nit: It seems strange to use a u64 for caps but only use the top 32 bits. Did you intend to use the low 32-bits for eax?

Thanks,
Ian

Quoted 249 lines

arch/x86/events/intel/core.c +17 -1

+
+               cpuid_count(ARCH_PERFMON_EXT_LEAF, ARCH_PERFMON_PEBS_COUNTER_LEAF,
+                           &eax, &ebx, &ecx, &edx);
+               pebs_mask   = counter_mask(eax, ecx);
+               pdists_mask = counter_mask(ebx, edx);
+               hybrid(pmu, arch_pebs_cap).counters = pebs_mask;
+               hybrid(pmu, arch_pebs_cap).pdists = pdists_mask;
+
+               if (WARN_ON((pebs_mask | pdists_mask) & ~cntrs_mask))
+                       x86_pmu.arch_pebs = 0;
+       } else {
+               WARN_ON(x86_pmu.arch_pebs == 1);
+               x86_pmu.arch_pebs = 0;
        }
 
        if (!intel_pmu_broken_perf_cap()) {

@@ -6252,7 +6277,7 @@ tsx_is_visible(struct kobject *kobj, struct attribute *attr, int i)
 static umode_t
 pebs_is_visible(struct kobject *kobj, struct attribute *attr, int i)
 {
-       return x86_pmu.ds_pebs ? attr->mode : 0;
+       return intel_pmu_has_pebs() ? attr->mode : 0;
 }
 
 static umode_t

@@ -7728,6 +7753,9 @@ __init int intel_pmu_init(void)
        if (!is_hybrid() && boot_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT))
                update_pmu_cap(NULL);
 
+       if (x86_pmu.arch_pebs)
+               pr_cont("Architectural PEBS, ");
+
        intel_pmu_check_counters_mask(&x86_pmu.cntr_mask64,
                                      &x86_pmu.fixed_cntr_mask64,
                                      &x86_pmu.intel_ctrl);

arch/x86/events/intel/ds.c +44 -8

index c0b7ac1c7594..26e485eca0a0 100644
@@ -1531,6 +1531,15 @@ static inline void intel_pmu_drain_large_pebs(struct cpu_hw_events *cpuc)
                intel_pmu_drain_pebs_buffer();
 }
 
+static void __intel_pmu_pebs_enable(struct perf_event *event)
+{
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+       struct hw_perf_event *hwc = &event->hw;
+
+       hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
+       cpuc->pebs_enabled |= 1ULL << hwc->idx;
+}
+
 void intel_pmu_pebs_enable(struct perf_event *event)
 {
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);

@@ -1539,9 +1548,7 @@ void intel_pmu_pebs_enable(struct perf_event *event)
        struct debug_store *ds = cpuc->ds;
        unsigned int idx = hwc->idx;
 
-       hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
-
-       cpuc->pebs_enabled |= 1ULL << hwc->idx;
+       __intel_pmu_pebs_enable(event);
 
        if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) && (x86_pmu.version < 5))
                cpuc->pebs_enabled |= 1ULL << (hwc->idx + 32);

@@ -1603,14 +1610,22 @@ void intel_pmu_pebs_del(struct perf_event *event)
        pebs_update_state(needed_cb, cpuc, event, false);
 }
 
-void intel_pmu_pebs_disable(struct perf_event *event)
+static void __intel_pmu_pebs_disable(struct perf_event *event)
 {
        struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct hw_perf_event *hwc = &event->hw;
 
        intel_pmu_drain_large_pebs(cpuc);
-
        cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
+       hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
+}
+
+void intel_pmu_pebs_disable(struct perf_event *event)
+{
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+       struct hw_perf_event *hwc = &event->hw;
+
+       __intel_pmu_pebs_disable(event);
 
        if ((event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT) &&
            (x86_pmu.version < 5))

@@ -1622,8 +1637,6 @@ void intel_pmu_pebs_disable(struct perf_event *event)
 
        if (cpuc->enabled)
                wrmsrq(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
-
-       hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
 }
 
 void intel_pmu_pebs_enable_all(void)

@@ -2669,11 +2682,26 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d
        }
 }
 
+static void __init intel_arch_pebs_init(void)
+{
+       /*
+        * Current hybrid platforms always both support arch-PEBS or not
+        * on all kinds of cores. So directly set x86_pmu.arch_pebs flag
+        * if boot cpu supports arch-PEBS.
+        */
+       x86_pmu.arch_pebs = 1;
+       x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
+       x86_pmu.pebs_capable = ~0ULL;
+
+       x86_pmu.pebs_enable = __intel_pmu_pebs_enable;
+       x86_pmu.pebs_disable = __intel_pmu_pebs_disable;
+}
+
 /*
  * PEBS probe and setup
  */
 
-void __init intel_pebs_init(void)
+static void __init intel_ds_pebs_init(void)
 {
        /*
         * No support for 32bit formats

@@ -2788,6 +2816,14 @@ void __init intel_pebs_init(void)
        }
 }
 
+void __init intel_pebs_init(void)
+{
+       if (x86_pmu.intel_cap.pebs_format == 0xf)
+               intel_arch_pebs_init();
+       else
+               intel_ds_pebs_init();
+}
+
 void perf_restore_debug_store(void)
 {
        struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds);

arch/x86/events/perf_event.h +22 -3

index 285779c73479..ca5289980b52 100644
@@ -708,6 +708,12 @@ enum hybrid_pmu_type {
        hybrid_big_small_tiny   = hybrid_big   | hybrid_small_tiny,
 };
 
+struct arch_pebs_cap {
+       u64 caps;
+       u64 counters;
+       u64 pdists;
+};
+
 struct x86_hybrid_pmu {
        struct pmu                      pmu;
        const char                      *name;

@@ -752,6 +758,8 @@ struct x86_hybrid_pmu {
                                        mid_ack         :1,
                                        enabled_ack     :1;
 
+       struct arch_pebs_cap            arch_pebs_cap;
+
        u64                             pebs_data_source[PERF_PEBS_DATA_SOURCE_MAX];
 };
 

@@ -906,7 +914,7 @@ struct x86_pmu {
        union perf_capabilities intel_cap;
 
        /*
-        * Intel DebugStore bits
+        * Intel DebugStore and PEBS bits
         */
        unsigned int    bts                     :1,
                        bts_active              :1,

@@ -917,7 +925,8 @@ struct x86_pmu {
                        pebs_no_tlb             :1,
                        pebs_no_isolation       :1,
                        pebs_block              :1,
-                       pebs_ept                :1;
+                       pebs_ept                :1,
+                       arch_pebs               :1;
        int             pebs_record_size;
        int             pebs_buffer_size;
        u64             pebs_events_mask;

@@ -929,6 +938,11 @@ struct x86_pmu {
        u64             rtm_abort_event;
        u64             pebs_capable;
 
+       /*
+        * Intel Architectural PEBS
+        */
+       struct arch_pebs_cap arch_pebs_cap;
+
        /*
         * Intel LBR
         */

@@ -1216,7 +1230,7 @@ int x86_reserve_hardware(void);
 
 void x86_release_hardware(void);
 
-int x86_pmu_max_precise(void);
+int x86_pmu_max_precise(struct pmu *pmu);
 
 void hw_perf_lbr_event_destroy(struct perf_event *event);
 

@@ -1791,6 +1805,11 @@ static inline int intel_pmu_max_num_pebs(struct pmu *pmu)
        return fls((u32)hybrid(pmu, pebs_events_mask));
 }
 
+static inline bool intel_pmu_has_pebs(void)
+{
+       return x86_pmu.ds_pebs || x86_pmu.arch_pebs;
+}
+
 #else /* CONFIG_CPU_SUP_INTEL */
 
 static inline void reserve_ds_buffers(void)

arch/x86/include/asm/perf_event.h +6 -2

index 49a4d442f3fc..0dfa06722bab 100644
@@ -200,6 +200,8 @@ union cpuid10_edx {
 #define ARCH_PERFMON_EXT_LEAF                  0x00000023
 #define ARCH_PERFMON_NUM_COUNTER_LEAF          0x1
 #define ARCH_PERFMON_ACR_LEAF                  0x2
+#define ARCH_PERFMON_PEBS_CAP_LEAF             0x4
+#define ARCH_PERFMON_PEBS_COUNTER_LEAF         0x5
 
 union cpuid35_eax {
        struct {

@@ -210,7 +212,10 @@ union cpuid35_eax {
                unsigned int    acr_subleaf:1;
                /* Events Sub-Leaf */
                unsigned int    events_subleaf:1;
-               unsigned int    reserved:28;
+               /* arch-PEBS Sub-Leaves */
+               unsigned int    pebs_caps_subleaf:1;
+               unsigned int    pebs_cnts_subleaf:1;
+               unsigned int    reserved:26;
        } split;
        unsigned int            full;
 };
--

2.34.1

A significant difference with adaptive PEBS is that arch-PEBS record supports fragments which means an arch-PEBS record could be split into several independent fragments which have its own arch-PEBS header in each fragment.

This patch defines architectural PEBS record layout structures and add helpers to process arch-PEBS records or fragments. Only legacy PEBS groups like basic, GPR, XMM and LBR groups are supported in this patch, the new added YMM/ZMM/OPMASK vector registers capturing would be supported in the future.

Signed-off-by Dapeng Mi <[email protected]>

---
 arch/x86/events/intel/core.c      |  13 +++
 arch/x86/events/intel/ds.c        | 184 ++++++++++++++++++++++++++++++
 arch/x86/include/asm/msr-index.h  |   6 +
 arch/x86/include/asm/perf_event.h |  96 ++++++++++++++++
 4 files changed, 299 insertions(+)

arch/x86/events/intel/core.c +13 -0

index 9ce27b326923..de4dbde28adc 100644
@@ -3215,6 +3215,19 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
                        status &= ~GLOBAL_STATUS_PERF_METRICS_OVF_BIT;
        }
 
+       /*
+        * Arch PEBS sets bit 54 in the global status register
+        */
+       if (__test_and_clear_bit(GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT,
+                                (unsigned long *)&status)) {
+               handled++;
+               static_call(x86_pmu_drain_pebs)(regs, &data);
+
+               if (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS] &&
+                   is_pebs_counter_event_group(cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS]))
+                       status &= ~GLOBAL_STATUS_PERF_METRICS_OVF_BIT;
+       }
+
        /*
         * Intel PT
         */

arch/x86/events/intel/ds.c +97 -0

index 68664526443f..fe1bf373409e 100644
@@ -2270,6 +2270,117 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event,
                        format_group);
 }
 
+static inline bool arch_pebs_record_continued(struct arch_pebs_header *header)
+{
+       /* Continue bit or null PEBS record indicates fragment follows. */
+       return header->cont || !(header->format & GENMASK_ULL(63, 16));
+}
+
+static void setup_arch_pebs_sample_data(struct perf_event *event,
+                                       struct pt_regs *iregs,
+                                       void *__pebs,
+                                       struct perf_sample_data *data,
+                                       struct pt_regs *regs)
+{
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+       u64 sample_type = event->attr.sample_type;
+       struct arch_pebs_header *header = NULL;
+       struct arch_pebs_aux *meminfo = NULL;
+       struct arch_pebs_gprs *gprs = NULL;
+       struct x86_perf_regs *perf_regs;
+       void *next_record;
+       void *at = __pebs;
+
+       if (at == NULL)
+               return;
+
+       perf_regs = container_of(regs, struct x86_perf_regs, regs);
+       perf_regs->xmm_regs = NULL;
+
+       __setup_perf_sample_data(event, iregs, data);
+
+       *regs = *iregs;
+
+again:
+       header = at;
+       next_record = at + sizeof(struct arch_pebs_header);
+       if (header->basic) {
+               struct arch_pebs_basic *basic = next_record;
+               u16 retire = 0;
+
+               next_record = basic + 1;
+
+               if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT)
+                       retire = basic->valid ? basic->retire : 0;
+               __setup_pebs_basic_group(event, regs, data, sample_type,
+                                basic->ip, basic->tsc, retire);
+       }
+
+       /*
+        * The record for MEMINFO is in front of GP
+        * But PERF_SAMPLE_TRANSACTION needs gprs->ax.
+        * Save the pointer here but process later.
+        */
+       if (header->aux) {
+               meminfo = next_record;
+               next_record = meminfo + 1;
+       }
+
+       if (header->gpr) {
+               gprs = next_record;
+               next_record = gprs + 1;
+
+               __setup_pebs_gpr_group(event, regs,
+                                      (struct pebs_gprs *)gprs,
+                                      sample_type);
+       }
+
+       if (header->aux) {
+               u64 ax = gprs ? gprs->ax : 0;
+
+               __setup_pebs_meminfo_group(event, data, sample_type,
+                                          meminfo->cache_latency,
+                                          meminfo->instr_latency,
+                                          meminfo->address, meminfo->aux,
+                                          meminfo->tsx_tuning, ax);
+       }
+
+       if (header->xmm) {
+               struct pebs_xmm *xmm;
+
+               next_record += sizeof(struct arch_pebs_xer_header);
+
+               xmm = next_record;
+               perf_regs->xmm_regs = xmm->xmm;
+               next_record = xmm + 1;
+       }
+
+       if (header->lbr) {
+               struct arch_pebs_lbr_header *lbr_header = next_record;
+               struct lbr_entry *lbr;
+               int num_lbr;
+
+               next_record = lbr_header + 1;
+               lbr = next_record;
+
+               num_lbr = header->lbr == ARCH_PEBS_LBR_NUM_VAR ?
+                               lbr_header->depth :
+                               header->lbr * ARCH_PEBS_BASE_LBR_ENTRIES;
+               next_record += num_lbr * sizeof(struct lbr_entry);

arch/x86/events/intel/ds.c +77 -0

+               goto again;
+       }
+}
+
 static inline void *
 get_next_pebs_record_by_bit(void *base, void *top, int bit)
 {

@@ -2753,6 +2864,78 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d
                                            setup_pebs_adaptive_sample_data);
 }
 
+static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs,
+                                     struct perf_sample_data *data)
+{
+       short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
+       void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS];
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+       union arch_pebs_index index;
+       struct x86_perf_regs perf_regs;
+       struct pt_regs *regs = &perf_regs.regs;
+       void *base, *at, *top;
+       u64 mask;
+
+       rdmsrq(MSR_IA32_PEBS_INDEX, index.whole);
+
+       if (unlikely(!index.wr)) {
+               intel_pmu_pebs_event_update_no_drain(cpuc, X86_PMC_IDX_MAX);
+               return;
+       }
+
+       base = cpuc->ds_pebs_vaddr;
+       top = (void *)((u64)cpuc->ds_pebs_vaddr +
+                      (index.wr << ARCH_PEBS_INDEX_WR_SHIFT));
+
+       index.wr = 0;
+       index.full = 0;
+       wrmsrq(MSR_IA32_PEBS_INDEX, index.whole);
+
+       mask = hybrid(cpuc->pmu, arch_pebs_cap).counters & cpuc->pebs_enabled;
+
+       if (!iregs)
+               iregs = &dummy_iregs;
+
+       /* Process all but the last event for each counter. */
+       for (at = base; at < top;) {
+               struct arch_pebs_header *header;
+               struct arch_pebs_basic *basic;
+               u64 pebs_status;
+
+               header = at;
+
+               if (WARN_ON_ONCE(!header->size))
+                       break;
+
+               /* 1st fragment or single record must have basic group */
+               if (!header->basic) {
+                       at += header->size;
+                       continue;
+               }
+
+               basic = at + sizeof(struct arch_pebs_header);
+               pebs_status = mask & basic->applicable_counters;
+               __intel_pmu_handle_pebs_record(iregs, regs, data, at,
+                                              pebs_status, counts, last,
+                                              setup_arch_pebs_sample_data);
+
+               /* Skip non-last fragments */
+               while (arch_pebs_record_continued(header)) {
+                       if (!header->size)
+                               break;
+                       at += header->size;
+                       header = at;
+               }
+
+               /* Skip last fragment or the single record */
+               at += header->size;
+       }
+
+       __intel_pmu_handle_last_pebs_record(iregs, regs, data, mask,
+                                           counts, last,
+                                           setup_arch_pebs_sample_data);
+}
+
 static void __init intel_arch_pebs_init(void)
 {
        /*

@@ -2762,6 +2945,7 @@ static void __init intel_arch_pebs_init(void)
         */
        x86_pmu.arch_pebs = 1;
        x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
+       x86_pmu.drain_pebs = intel_pmu_drain_arch_pebs;
        x86_pmu.pebs_capable = ~0ULL;
 
        x86_pmu.pebs_enable = __intel_pmu_pebs_enable;

arch/x86/include/asm/msr-index.h +6 -0

index 9e1720d73244..fc7a4e7c718d 100644
@@ -327,6 +327,12 @@
                                         PERF_CAP_PEBS_FORMAT | PERF_CAP_PEBS_BASELINE | \
                                         PERF_CAP_PEBS_TIMING_INFO)
 
+/* Arch PEBS */
+#define MSR_IA32_PEBS_BASE             0x000003f4
+#define MSR_IA32_PEBS_INDEX            0x000003f5
+#define ARCH_PEBS_OFFSET_MASK          0x7fffff
+#define ARCH_PEBS_INDEX_WR_SHIFT       4
+
 #define MSR_IA32_RTIT_CTL              0x00000570
 #define RTIT_CTL_TRACEEN               BIT(0)
 #define RTIT_CTL_CYCLEACC              BIT(1)

arch/x86/include/asm/perf_event.h +96 -1

index 0dfa06722bab..3b3848f0d339 100644
@@ -437,6 +437,8 @@ static inline bool is_topdown_idx(int idx)
 #define GLOBAL_STATUS_LBRS_FROZEN              BIT_ULL(GLOBAL_STATUS_LBRS_FROZEN_BIT)
 #define GLOBAL_STATUS_TRACE_TOPAPMI_BIT                55
 #define GLOBAL_STATUS_TRACE_TOPAPMI            BIT_ULL(GLOBAL_STATUS_TRACE_TOPAPMI_BIT)
+#define GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT  54
+#define GLOBAL_STATUS_ARCH_PEBS_THRESHOLD      BIT_ULL(GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT)
 #define GLOBAL_STATUS_PERF_METRICS_OVF_BIT     48
 
 #define GLOBAL_CTRL_EN_PERF_METRICS            BIT_ULL(48)

@@ -507,6 +509,100 @@ struct pebs_cntr_header {
 
 #define INTEL_CNTR_METRICS             0x3
 
+/*
+ * Arch PEBS
+ */
+union arch_pebs_index {
+       struct {
+               u64 rsvd:4,
+                   wr:23,
+                   rsvd2:4,
+                   full:1,
+                   en:1,
+                   rsvd3:3,
+                   thresh:23,
+                   rsvd4:5;
+       };
+       u64 whole;
+};
+
+struct arch_pebs_header {
+       union {
+               u64 format;
+               struct {
+                       u64 size:16,    /* Record size */
+                           rsvd:14,
+                           mode:1,     /* 64BIT_MODE */
+                           cont:1,
+                           rsvd2:3,
+                           cntr:5,
+                           lbr:2,
+                           rsvd3:7,
+                           xmm:1,
+                           ymmh:1,
+                           rsvd4:2,
+                           opmask:1,
+                           zmmh:1,
+                           h16zmm:1,
+                           rsvd5:5,
+                           gpr:1,
+                           aux:1,
+                           basic:1;
+               };
+       };
+       u64 rsvd6;
+};
+
+struct arch_pebs_basic {
+       u64 ip;
+       u64 applicable_counters;
+       u64 tsc;
+       u64 retire      :16,    /* Retire Latency */
+           valid       :1,
+           rsvd        :47;
+       u64 rsvd2;
+       u64 rsvd3;
+};
+
+struct arch_pebs_aux {
+       u64 address;
+       u64 rsvd;
+       u64 rsvd2;
+       u64 rsvd3;
+       u64 rsvd4;
+       u64 aux;
+       u64 instr_latency       :16,
+           pad2                :16,
+           cache_latency       :16,
+           pad3                :16;
+       u64 tsx_tuning;
+};
+
+struct arch_pebs_gprs {
+       u64 flags, ip, ax, cx, dx, bx, sp, bp, si, di;
+       u64 r8, r9, r10, r11, r12, r13, r14, r15, ssp;
+       u64 rsvd;
+};
+
+struct arch_pebs_xer_header {
+       u64 xstate;
+       u64 rsvd;
+};
+
+#define ARCH_PEBS_LBR_NAN              0x0
+#define ARCH_PEBS_LBR_NUM_8            0x1
+#define ARCH_PEBS_LBR_NUM_16           0x2
+#define ARCH_PEBS_LBR_NUM_VAR          0x3
+#define ARCH_PEBS_BASE_LBR_ENTRIES     8
+struct arch_pebs_lbr_header {
+       u64 rsvd;
+       u64 ctl;
+       u64 depth;
+       u64 ler_from;
+       u64 ler_to;
+       u64 ler_info;
+};
+
 /*
  * AMD Extended Performance Monitoring and Debug cpuid feature detection
  */
--

2.34.1

Quoted 152 lines · Dapeng Mi

A significant difference with adaptive PEBS is that arch-PEBS record supports fragments which means an arch-PEBS record could be split into several independent fragments which have its own arch-PEBS header in each fragment.

This patch defines architectural PEBS record layout structures and add helpers to process arch-PEBS records or fragments. Only legacy PEBS groups like basic, GPR, XMM and LBR groups are supported in this patch, the new added YMM/ZMM/OPMASK vector registers capturing would be supported in the future.

Signed-off-by Dapeng Mi <[email protected]>

---
 arch/x86/events/intel/core.c      |  13 +++
 arch/x86/events/intel/ds.c        | 184 ++++++++++++++++++++++++++++++
 arch/x86/include/asm/msr-index.h  |   6 +
 arch/x86/include/asm/perf_event.h |  96 ++++++++++++++++
 4 files changed, 299 insertions(+)

arch/x86/events/intel/core.c +13 -0

index 9ce27b326923..de4dbde28adc 100644
@@ -3215,6 +3215,19 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status)
                        status &= ~GLOBAL_STATUS_PERF_METRICS_OVF_BIT;
        }
 
+       /*
+        * Arch PEBS sets bit 54 in the global status register
+        */
+       if (__test_and_clear_bit(GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT,
+                                (unsigned long *)&status)) {
+               handled++;
+               static_call(x86_pmu_drain_pebs)(regs, &data);
+
+               if (cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS] &&
+                   is_pebs_counter_event_group(cpuc->events[INTEL_PMC_IDX_FIXED_SLOTS]))
+                       status &= ~GLOBAL_STATUS_PERF_METRICS_OVF_BIT;
+       }
+
        /*
         * Intel PT
         */

arch/x86/events/intel/ds.c +101 -0

index 68664526443f..fe1bf373409e 100644
@@ -2270,6 +2270,117 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event,
                        format_group);
 }
 
+static inline bool arch_pebs_record_continued(struct arch_pebs_header *header)
+{
+       /* Continue bit or null PEBS record indicates fragment follows. */
+       return header->cont || !(header->format & GENMASK_ULL(63, 16));
+}
+
+static void setup_arch_pebs_sample_data(struct perf_event *event,
+                                       struct pt_regs *iregs,
+                                       void *__pebs,
+                                       struct perf_sample_data *data,
+                                       struct pt_regs *regs)
+{
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+       u64 sample_type = event->attr.sample_type;
+       struct arch_pebs_header *header = NULL;
+       struct arch_pebs_aux *meminfo = NULL;
+       struct arch_pebs_gprs *gprs = NULL;
+       struct x86_perf_regs *perf_regs;
+       void *next_record;
+       void *at = __pebs;
+
+       if (at == NULL)
+               return;
+
+       perf_regs = container_of(regs, struct x86_perf_regs, regs);
+       perf_regs->xmm_regs = NULL;
+
+       __setup_perf_sample_data(event, iregs, data);
+
+       *regs = *iregs;
+
+again:
+       header = at;
+       next_record = at + sizeof(struct arch_pebs_header);
+       if (header->basic) {
+               struct arch_pebs_basic *basic = next_record;
+               u16 retire = 0;
+
+               next_record = basic + 1;
+
+               if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT)
+                       retire = basic->valid ? basic->retire : 0;
+               __setup_pebs_basic_group(event, regs, data, sample_type,
+                                basic->ip, basic->tsc, retire);
+       }
+
+       /*
+        * The record for MEMINFO is in front of GP
+        * But PERF_SAMPLE_TRANSACTION needs gprs->ax.
+        * Save the pointer here but process later.
+        */
+       if (header->aux) {
+               meminfo = next_record;
+               next_record = meminfo + 1;
+       }
+
+       if (header->gpr) {
+               gprs = next_record;
+               next_record = gprs + 1;
+
+               __setup_pebs_gpr_group(event, regs,
+                                      (struct pebs_gprs *)gprs,
+                                      sample_type);
+       }
+
+       if (header->aux) {
+               u64 ax = gprs ? gprs->ax : 0;
+
+               __setup_pebs_meminfo_group(event, data, sample_type,
+                                          meminfo->cache_latency,
+                                          meminfo->instr_latency,
+                                          meminfo->address, meminfo->aux,
+                                          meminfo->tsx_tuning, ax);
+       }
+
+       if (header->xmm) {
+               struct pebs_xmm *xmm;
+
+               next_record += sizeof(struct arch_pebs_xer_header);
+
+               xmm = next_record;
+               perf_regs->xmm_regs = xmm->xmm;
+               next_record = xmm + 1;
+       }
+
+       if (header->lbr) {
+               struct arch_pebs_lbr_header *lbr_header = next_record;
+               struct lbr_entry *lbr;
+               int num_lbr;
+
+               next_record = lbr_header + 1;
+               lbr = next_record;
+
+               num_lbr = header->lbr == ARCH_PEBS_LBR_NUM_VAR ?
+                               lbr_header->depth :
+                               header->lbr * ARCH_PEBS_BASE_LBR_ENTRIES;
+               next_record += num_lbr * sizeof(struct lbr_entry);
+
+               if (has_branch_stack(event)) {
+                       intel_pmu_store_pebs_lbrs(lbr);
+                       intel_pmu_lbr_save_brstack(data, cpuc, event);

arch/x86/events/intel/ds.c static void setup_pebs_adaptive_sample_data(struct perf_event *event, · line 2379

Quoted 6 lines

arch/x86/events/intel/ds.c +6 -0

+               }
+       }
+
+       /* Parse followed fragments if there are. */
+       if (arch_pebs_record_continued(header)) {
+               at = at + header->size;

If the header->size is 0, will it cause infinite loop? I can see a 0 check below but not here.

Thanks,
CT

Quoted 229 lines

arch/x86/events/intel/ds.c +77 -0

+               goto again;
+       }
+}
+
 static inline void *
 get_next_pebs_record_by_bit(void *base, void *top, int bit)
 {

@@ -2753,6 +2864,78 @@ static void intel_pmu_drain_pebs_icl(struct pt_regs *iregs, struct perf_sample_d
                                            setup_pebs_adaptive_sample_data);
 }
 
+static void intel_pmu_drain_arch_pebs(struct pt_regs *iregs,
+                                     struct perf_sample_data *data)
+{
+       short counts[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS] = {};
+       void *last[INTEL_PMC_IDX_FIXED + MAX_FIXED_PEBS_EVENTS];
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+       union arch_pebs_index index;
+       struct x86_perf_regs perf_regs;
+       struct pt_regs *regs = &perf_regs.regs;
+       void *base, *at, *top;
+       u64 mask;
+
+       rdmsrq(MSR_IA32_PEBS_INDEX, index.whole);
+
+       if (unlikely(!index.wr)) {
+               intel_pmu_pebs_event_update_no_drain(cpuc, X86_PMC_IDX_MAX);
+               return;
+       }
+
+       base = cpuc->ds_pebs_vaddr;
+       top = (void *)((u64)cpuc->ds_pebs_vaddr +
+                      (index.wr << ARCH_PEBS_INDEX_WR_SHIFT));
+
+       index.wr = 0;
+       index.full = 0;
+       wrmsrq(MSR_IA32_PEBS_INDEX, index.whole);
+
+       mask = hybrid(cpuc->pmu, arch_pebs_cap).counters & cpuc->pebs_enabled;
+
+       if (!iregs)
+               iregs = &dummy_iregs;
+
+       /* Process all but the last event for each counter. */
+       for (at = base; at < top;) {
+               struct arch_pebs_header *header;
+               struct arch_pebs_basic *basic;
+               u64 pebs_status;
+
+               header = at;
+
+               if (WARN_ON_ONCE(!header->size))
+                       break;
+
+               /* 1st fragment or single record must have basic group */
+               if (!header->basic) {
+                       at += header->size;
+                       continue;
+               }
+
+               basic = at + sizeof(struct arch_pebs_header);
+               pebs_status = mask & basic->applicable_counters;
+               __intel_pmu_handle_pebs_record(iregs, regs, data, at,
+                                              pebs_status, counts, last,
+                                              setup_arch_pebs_sample_data);
+
+               /* Skip non-last fragments */
+               while (arch_pebs_record_continued(header)) {
+                       if (!header->size)
+                               break;
+                       at += header->size;
+                       header = at;
+               }
+
+               /* Skip last fragment or the single record */
+               at += header->size;
+       }
+
+       __intel_pmu_handle_last_pebs_record(iregs, regs, data, mask,
+                                           counts, last,
+                                           setup_arch_pebs_sample_data);
+}
+
 static void __init intel_arch_pebs_init(void)
 {
        /*

@@ -2762,6 +2945,7 @@ static void __init intel_arch_pebs_init(void)
         */
        x86_pmu.arch_pebs = 1;
        x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
+       x86_pmu.drain_pebs = intel_pmu_drain_arch_pebs;
        x86_pmu.pebs_capable = ~0ULL;
 
        x86_pmu.pebs_enable = __intel_pmu_pebs_enable;

arch/x86/include/asm/msr-index.h +6 -0

index 9e1720d73244..fc7a4e7c718d 100644
@@ -327,6 +327,12 @@
                                         PERF_CAP_PEBS_FORMAT | PERF_CAP_PEBS_BASELINE | \
                                         PERF_CAP_PEBS_TIMING_INFO)
 
+/* Arch PEBS */
+#define MSR_IA32_PEBS_BASE             0x000003f4
+#define MSR_IA32_PEBS_INDEX            0x000003f5
+#define ARCH_PEBS_OFFSET_MASK          0x7fffff
+#define ARCH_PEBS_INDEX_WR_SHIFT       4
+
 #define MSR_IA32_RTIT_CTL              0x00000570
 #define RTIT_CTL_TRACEEN               BIT(0)
 #define RTIT_CTL_CYCLEACC              BIT(1)

arch/x86/include/asm/perf_event.h +96 -1

index 0dfa06722bab..3b3848f0d339 100644
@@ -437,6 +437,8 @@ static inline bool is_topdown_idx(int idx)
 #define GLOBAL_STATUS_LBRS_FROZEN              BIT_ULL(GLOBAL_STATUS_LBRS_FROZEN_BIT)
 #define GLOBAL_STATUS_TRACE_TOPAPMI_BIT                55
 #define GLOBAL_STATUS_TRACE_TOPAPMI            BIT_ULL(GLOBAL_STATUS_TRACE_TOPAPMI_BIT)
+#define GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT  54
+#define GLOBAL_STATUS_ARCH_PEBS_THRESHOLD      BIT_ULL(GLOBAL_STATUS_ARCH_PEBS_THRESHOLD_BIT)
 #define GLOBAL_STATUS_PERF_METRICS_OVF_BIT     48
 
 #define GLOBAL_CTRL_EN_PERF_METRICS            BIT_ULL(48)

@@ -507,6 +509,100 @@ struct pebs_cntr_header {
 
 #define INTEL_CNTR_METRICS             0x3
 
+/*
+ * Arch PEBS
+ */
+union arch_pebs_index {
+       struct {
+               u64 rsvd:4,
+                   wr:23,
+                   rsvd2:4,
+                   full:1,
+                   en:1,
+                   rsvd3:3,
+                   thresh:23,
+                   rsvd4:5;
+       };
+       u64 whole;
+};
+
+struct arch_pebs_header {
+       union {
+               u64 format;
+               struct {
+                       u64 size:16,    /* Record size */
+                           rsvd:14,
+                           mode:1,     /* 64BIT_MODE */
+                           cont:1,
+                           rsvd2:3,
+                           cntr:5,
+                           lbr:2,
+                           rsvd3:7,
+                           xmm:1,
+                           ymmh:1,
+                           rsvd4:2,
+                           opmask:1,
+                           zmmh:1,
+                           h16zmm:1,
+                           rsvd5:5,
+                           gpr:1,
+                           aux:1,
+                           basic:1;
+               };
+       };
+       u64 rsvd6;
+};
+
+struct arch_pebs_basic {
+       u64 ip;
+       u64 applicable_counters;
+       u64 tsc;
+       u64 retire      :16,    /* Retire Latency */
+           valid       :1,
+           rsvd        :47;
+       u64 rsvd2;
+       u64 rsvd3;
+};
+
+struct arch_pebs_aux {
+       u64 address;
+       u64 rsvd;
+       u64 rsvd2;
+       u64 rsvd3;
+       u64 rsvd4;
+       u64 aux;
+       u64 instr_latency       :16,
+           pad2                :16,
+           cache_latency       :16,
+           pad3                :16;
+       u64 tsx_tuning;
+};
+
+struct arch_pebs_gprs {
+       u64 flags, ip, ax, cx, dx, bx, sp, bp, si, di;
+       u64 r8, r9, r10, r11, r12, r13, r14, r15, ssp;
+       u64 rsvd;
+};
+
+struct arch_pebs_xer_header {
+       u64 xstate;
+       u64 rsvd;
+};
+
+#define ARCH_PEBS_LBR_NAN              0x0
+#define ARCH_PEBS_LBR_NUM_8            0x1
+#define ARCH_PEBS_LBR_NUM_16           0x2
+#define ARCH_PEBS_LBR_NUM_VAR          0x3
+#define ARCH_PEBS_BASE_LBR_ENTRIES     8
+struct arch_pebs_lbr_header {
+       u64 rsvd;
+       u64 ctl;
+       u64 depth;
+       u64 ler_from;
+       u64 ler_to;
+       u64 ler_info;
+};
+
 /*
  * AMD Extended Performance Monitoring and Debug cpuid feature detection
  */
--

2.34.1

arch-PEBS provides CPUIDs to enumerate which counters support PEBS sampling and precise distribution PEBS sampling. Thus PEBS constraints should be dynamically configured base on these counter and precise distribution bitmap instead of defining them statically.

Update event dyn_constraint base on PEBS event precise level.

Signed-off-by Dapeng Mi <[email protected]>

---
 arch/x86/events/intel/core.c | 11 +++++++++++
 arch/x86/events/intel/ds.c   |  1 +
 2 files changed, 12 insertions(+)

arch/x86/events/intel/core.c +11 -0

index 6e04d73dfae5..40ccfd80d554 100644
@@ -4252,6 +4252,8 @@ static int intel_pmu_hw_config(struct perf_event *event)
 	}
 
 	if (event->attr.precise_ip) {
+		struct arch_pebs_cap pebs_cap = hybrid(event->pmu, arch_pebs_cap);
+
 		if ((event->attr.config & INTEL_ARCH_EVENT_MASK) == INTEL_FIXED_VLBR_EVENT)
 			return -EINVAL;
 

@@ -4265,6 +4267,15 @@ static int intel_pmu_hw_config(struct perf_event *event)
 		}
 		if (x86_pmu.pebs_aliases)
 			x86_pmu.pebs_aliases(event);
+
+		if (x86_pmu.arch_pebs) {
+			u64 cntr_mask = hybrid(event->pmu, intel_ctrl) &
+						~GLOBAL_CTRL_EN_PERF_METRICS;
+			u64 pebs_mask = event->attr.precise_ip >= 3 ?
+						pebs_cap.pdists : pebs_cap.counters;
+			if (cntr_mask != pebs_mask)
+				event->hw.dyn_constraint &= pebs_mask;
+		}
 	}
 
 	if (needs_branch_stack(event)) {

arch/x86/events/intel/ds.c +1 -0

index 5c26a5235f94..1179980f795b 100644
@@ -3005,6 +3005,7 @@ static void __init intel_arch_pebs_init(void)
 	x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
 	x86_pmu.drain_pebs = intel_pmu_drain_arch_pebs;
 	x86_pmu.pebs_capable = ~0ULL;
+	x86_pmu.flags |= PMU_FL_PEBS_ALL;
 
 	x86_pmu.pebs_enable = __intel_pmu_pebs_enable;
 	x86_pmu.pebs_disable = __intel_pmu_pebs_disable;

Signature · 1 line

2.34.1

Quoted 6 lines · Dapeng Mi

arch-PEBS provides CPUIDs to enumerate which counters support PEBS sampling and precise distribution PEBS sampling. Thus PEBS constraints should be dynamically configured base on these counter and precise distribution bitmap instead of defining them statically.

Update event dyn_constraint base on PEBS event precise level.

What happened to this:

https://lore.kernel.org/all/[email protected]/

Quoted 52 lines

Signed-off-by Dapeng Mi <[email protected]>

---
 arch/x86/events/intel/core.c | 11 +++++++++++
 arch/x86/events/intel/ds.c   |  1 +
 2 files changed, 12 insertions(+)

arch/x86/events/intel/core.c +11 -0

index 6e04d73dfae5..40ccfd80d554 100644
@@ -4252,6 +4252,8 @@ static int intel_pmu_hw_config(struct perf_event *event)
 	}
 
 	if (event->attr.precise_ip) {
+		struct arch_pebs_cap pebs_cap = hybrid(event->pmu, arch_pebs_cap);
+
 		if ((event->attr.config & INTEL_ARCH_EVENT_MASK) == INTEL_FIXED_VLBR_EVENT)
 			return -EINVAL;
 

@@ -4265,6 +4267,15 @@ static int intel_pmu_hw_config(struct perf_event *event)
 		}
 		if (x86_pmu.pebs_aliases)
 			x86_pmu.pebs_aliases(event);
+
+		if (x86_pmu.arch_pebs) {
+			u64 cntr_mask = hybrid(event->pmu, intel_ctrl) &
+						~GLOBAL_CTRL_EN_PERF_METRICS;
+			u64 pebs_mask = event->attr.precise_ip >= 3 ?
+						pebs_cap.pdists : pebs_cap.counters;
+			if (cntr_mask != pebs_mask)
+				event->hw.dyn_constraint &= pebs_mask;
+		}
 	}
 
 	if (needs_branch_stack(event)) {

arch/x86/events/intel/ds.c +1 -0

index 5c26a5235f94..1179980f795b 100644
@@ -3005,6 +3005,7 @@ static void __init intel_arch_pebs_init(void)
 	x86_pmu.pebs_buffer_size = PEBS_BUFFER_SIZE;
 	x86_pmu.drain_pebs = intel_pmu_drain_arch_pebs;
 	x86_pmu.pebs_capable = ~0ULL;
+	x86_pmu.flags |= PMU_FL_PEBS_ALL;
 
 	x86_pmu.pebs_enable = __intel_pmu_pebs_enable;
 	x86_pmu.pebs_disable = __intel_pmu_pebs_disable;

Signature · 1 line

2.34.1

Quoted 7 lines · Peter Zijlstra

Quoted 6 lines · Dapeng Mi

arch-PEBS provides CPUIDs to enumerate which counters support PEBS sampling and precise distribution PEBS sampling. Thus PEBS constraints should be dynamically configured base on these counter and precise distribution bitmap instead of defining them statically.

Update event dyn_constraint base on PEBS event precise level.

Quoted 3 lines

What happened to this:

https://lore.kernel.org/all/[email protected]/

About the issue, Kan ever posted a patch to mitigate the risk, but it seems the patch is not merged yet.

https://lore.kernel.org/all/[email protected]/

Quoted 14 lines · Dapeng

Quoted 10 lines · Peter Zijlstra

Quoted 6 lines · Dapeng Mi

arch-PEBS provides CPUIDs to enumerate which counters support PEBS sampling and precise distribution PEBS sampling. Thus PEBS constraints should be dynamically configured base on these counter and precise distribution bitmap instead of defining them statically.

Update event dyn_constraint base on PEBS event precise level.

What happened to this:

https://lore.kernel.org/all/[email protected]/

About the issue, Kan ever posted a patch to mitigate the risk, but it seems the patch is not merged yet.

Quoted 2 lines

https://lore.kernel.org/all/[email protected]/

Clearly it became a victim of some scatter brained maintainer or something.

Let me stick that near this set and go read the last few patches.

Quoted 7 lines · Peter Zijlstra

Quoted 6 lines · Dapeng Mi

arch-PEBS provides CPUIDs to enumerate which counters support PEBS sampling and precise distribution PEBS sampling. Thus PEBS constraints should be dynamically configured base on these counter and precise distribution bitmap instead of defining them statically.

Update event dyn_constraint base on PEBS event precise level.

Quoted 3 lines

What happened to this:

https://lore.kernel.org/all/[email protected]/

About the issue, Kan ever posted a patch to mitigate the risk, but it seems the patch is not merged yet.

https://lore.kernel.org/all/[email protected]/

Quoted 7 lines · Dapeng

Quoted 6 lines · Peter Zijlstra

Quoted 5 lines · Dapeng Mi

arch-PEBS provides CPUIDs to enumerate which counters support PEBS sampling and precise distribution PEBS sampling. Thus PEBS constraints should be dynamically configured base on these counter and precise distribution bitmap instead of defining them statically.

Quoted 8 lines

Quoted 4 lines

Quoted 1 line

Update event dyn_constraint base on PEBS event precise level.

What happened to this:

https://lore.kernel.org/all/[email protected]/

About the issue, Kan ever posted a patch to mitigate the risk, but it seems the patch is not merged yet.

https://lore.kernel.org/all/[email protected]/

IIUC the below is what is required handle this new dynamic case, right?

arch/x86/events/intel/core.c +5 -0

@@ -5423,6 +5423,8 @@ enum dyn_constr_type {
 	DYN_CONSTR_BR_CNTR,
 	DYN_CONSTR_ACR_CNTR,
 	DYN_CONSTR_ACR_CAUSE,
+	DYN_CONSTR_PEBS,
+	DYN_CONSTR_PDIST,
 
 	DYN_CONSTR_MAX,
 };

@@ -5432,6 +5434,8 @@ static const char * const dyn_constr_typ
 	[DYN_CONSTR_BR_CNTR] = "a branch counter logging event",
 	[DYN_CONSTR_ACR_CNTR] = "an auto-counter reload event",
 	[DYN_CONSTR_ACR_CAUSE] = "an auto-counter reload cause event",
+	[DYN_CONSTR_PEBS] = "a PEBS event",
+	[DYN_CONSTR_PDIST] = "a PEBS PDIST event",
 };
 
 static void __intel_pmu_check_dyn_constr(struct event_constraint *constr,

@@ -5536,6 +5540,14 @@ static void intel_pmu_check_dyn_constr(s
 				continue;
 			mask = hybrid(pmu, acr_cause_mask64) & GENMASK_ULL(INTEL_PMC_MAX_GENERIC - 1, 0);
 			break;
+		case DYN_CONSTR_PEBS: