Skip to contentSkip to conversation

perf/core: Update perf_adjust_freq_unthr_context()

PATCH Superseded Namhyung Kim 4 participants 15 messages Nov 20, 2023 – Dec 2, 2023

Subsystems (2)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Namhyung Kim

author maintainer
#0 lore raw

PATCH 1/3 perf/core: Update perf_adjust_freq_unthr_context()

It was unnecessarily disabling and enabling PMUs for each event. It should be done at PMU level. Add pmu_ctx->nr_freq counter to check it at each PMU. As pmu context has separate active lists for pinned group and flexible group, factor out a new function to do the job.

Another minor optimization is that it can skip PMUs w/ CAP_NO_INTERRUPT even if it needs to unthrottle sampling events.

---
 include/linux/perf_event.h |  1 +
 kernel/events/core.c       | 68 +++++++++++++++++++++++---------------
 2 files changed, 43 insertions(+), 26 deletions(-)
include/linux/perf_event.h +1 -0
index 0367d748fae0..3eb17dc89f5e 100644
@@ -879,6 +879,7 @@ struct perf_event_pmu_context {
 
 	unsigned int			nr_events;
 	unsigned int			nr_cgroups;
+	unsigned int			nr_freq;
 
 	atomic_t			refcount; /* event <-> epc */
 	struct rcu_head			rcu_head;
kernel/events/core.c +42 -26

112 lines not on this page · raw message

Signature · 1 line
2.43.0.rc1.413.gea7ed67945-goog
author maintainer
#1 lore raw

PATCH 2/3 perf/core: Reduce PMU access to adjust sample freq

For throttled events, it first starts the event and then stop unnecessarily. As it's already stopped, it can directly adjust the frequency and then move on.

---
 kernel/events/core.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)
kernel/events/core.c +10 -3
index 53e2ad73102d..fd3449e4d081 100644
@@ -4119,10 +4119,15 @@ static void perf_adjust_freq_unthr_events(struct list_head *event_list)
 		if (hwc->interrupts == MAX_INTERRUPTS) {
 			hwc->interrupts = 0;
 			perf_log_throttle(event, 1);
-			event->pmu->start(event, 0);
-		}
 
-		if (!event->attr.freq || !event->attr.sample_freq)
+			if (!event->attr.freq || !event->attr.sample_freq) {
+				delta = 0;
+				goto next;
+			}
+
+			if (event->hw.state & PERF_HES_STOPPED)
+				goto adjust;
+		} else if (!event->attr.freq || !event->attr.sample_freq)
 			continue;
 
 		/*
@@ -4130,6 +4135,7 @@ static void perf_adjust_freq_unthr_events(struct list_head *event_list)
 		 */
 		event->pmu->stop(event, PERF_EF_UPDATE);
 
+adjust:
 		now = local64_read(&event->count);
 		delta = now - hwc->freq_count_stamp;
 		hwc->freq_count_stamp = now;
@@ -4144,6 +4150,7 @@ static void perf_adjust_freq_unthr_events(struct list_head *event_list)
 		if (delta > 0)
 			perf_adjust_period(event, period, delta, false);
 
+next:
 		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
 	}
 }
Signature · 1 line
2.43.0.rc1.413.gea7ed67945-goog
Reviewed-by on v1 2/3
#2 lore raw

Re: [PATCH 2/3] perf/core: Reduce PMU access to adjust sample freq

Quoted 5 lines · Namhyung Kim

For throttled events, it first starts the event and then stop unnecessarily. As it's already stopped, it can directly adjust the frequency and then move on.

Thanks,
Kan

Quoted 43 lines
---
 kernel/events/core.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)
kernel/events/core.c +10 -3
index 53e2ad73102d..fd3449e4d081 100644
@@ -4119,10 +4119,15 @@ static void perf_adjust_freq_unthr_events(struct list_head *event_list)
 		if (hwc->interrupts == MAX_INTERRUPTS) {
 			hwc->interrupts = 0;
 			perf_log_throttle(event, 1);
-			event->pmu->start(event, 0);
-		}
 
-		if (!event->attr.freq || !event->attr.sample_freq)
+			if (!event->attr.freq || !event->attr.sample_freq) {
+				delta = 0;
+				goto next;
+			}
+
+			if (event->hw.state & PERF_HES_STOPPED)
+				goto adjust;
+		} else if (!event->attr.freq || !event->attr.sample_freq)
 			continue;
 
 		/*
@@ -4130,6 +4135,7 @@ static void perf_adjust_freq_unthr_events(struct list_head *event_list)
 		 */
 		event->pmu->stop(event, PERF_EF_UPDATE);
 
+adjust:
 		now = local64_read(&event->count);
 		delta = now - hwc->freq_count_stamp;
 		hwc->freq_count_stamp = now;
@@ -4144,6 +4150,7 @@ static void perf_adjust_freq_unthr_events(struct list_head *event_list)
 		if (delta > 0)
 			perf_adjust_period(event, period, delta, false);
 
+next:
 		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
 	}
 }
author maintainer
#3 lore raw

PATCH 3/3 perf/x86: Add CAP_NO_INTERRUPT for uncore PMUs

In reply to Namhyung Kim

It doesn't support sampling in uncore PMU events. While it's technically possible to generate interrupts, let's treat it as if it has no interrupt in order to skip the freq adjust/unthrottling logic in the timer handler which is only meaningful to sampling events.

Also remove the sampling event check because it'd be done in the general code in the perf_event_open syscall.

---
 arch/x86/events/intel/uncore.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
arch/x86/events/intel/uncore.c +6 -5
index 69043e02e8a7..f7e6228bd1b1 100644
@@ -744,10 +744,6 @@ static int uncore_pmu_event_init(struct perf_event *event)
 	if (pmu->func_id < 0)
 		return -ENOENT;
 
-	/* Sampling not supported yet */
-	if (hwc->sample_period)
-		return -EINVAL;
-
 	/*
 	 * Place all uncore events for a particular physical package
 	 * onto a single cpu
@@ -919,7 +915,12 @@ static int uncore_pmu_register(struct intel_uncore_pmu *pmu)
 			.stop		= uncore_pmu_event_stop,
 			.read		= uncore_pmu_event_read,
 			.module		= THIS_MODULE,
-			.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
+			/*
+			 * It doesn't allow sampling for uncore events, let's
+			 * treat the PMU has no interrupts to skip them in the
+			 * perf_adjust_freq_unthr_context().
+			 */
+			.capabilities	= PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
 			.attr_update	= pmu->type->attr_update,
 		};
 	} else {
Signature · 1 line
2.43.0.rc1.413.gea7ed67945-goog
#4 lore raw

Re: [PATCH 3/3] perf/x86: Add CAP_NO_INTERRUPT for uncore PMUs

Quoted 31 lines · Namhyung Kim

It doesn't support sampling in uncore PMU events. While it's technically possible to generate interrupts, let's treat it as if it has no interrupt in order to skip the freq adjust/unthrottling logic in the timer handler which is only meaningful to sampling events.

Also remove the sampling event check because it'd be done in the general code in the perf_event_open syscall.

---
 arch/x86/events/intel/uncore.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
arch/x86/events/intel/uncore.c +0 -4
index 69043e02e8a7..f7e6228bd1b1 100644
@@ -744,10 +744,6 @@ static int uncore_pmu_event_init(struct perf_event *event)
 	if (pmu->func_id < 0)
 		return -ENOENT;
 
-	/* Sampling not supported yet */
-	if (hwc->sample_period)
-		return -EINVAL;
-
 	/*
 	 * Place all uncore events for a particular physical package
 	 * onto a single cpu
@@ -919,7 +915,12 @@ static int uncore_pmu_register(struct intel_uncore_pmu *pmu)
 			.stop		= uncore_pmu_event_stop,
 			.read		= uncore_pmu_event_read,

arch/x86/events/intel/uncore.c static int uncore_pmu_register(struct intel_uncore_pmu *pmu) · line 925

Quoted 10 lines
arch/x86/events/intel/uncore.c +6 -1
 			.module		= THIS_MODULE,
-			.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
+			/*
+			 * It doesn't allow sampling for uncore events, let's
+			 * treat the PMU has no interrupts to skip them in the
+			 * perf_adjust_freq_unthr_context().
+			 */
+			.capabilities	= PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
 			.attr_update	= pmu->type->attr_update,
 		};

There is a special customized uncore PMU which needs the flag as well.

(quoted diff) +0 -0

b/arch/x86/events/intel/uncore_snb.c
index 7fd4334e12a1..46a63e291975 100644

arch/x86/events/intel/uncore_snb.c +1 -1
@@ -1013,7 +1013,7 @@ static struct pmu snb_uncore_imc_pmu = {
        .start          = uncore_pmu_event_start,
        .stop           = uncore_pmu_event_stop,
        .read           = uncore_pmu_event_read,
-       .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
+       .capabilities   = PERF_PMU_CAP_NO_EXCLUDE |

PERF_PMU_CAP_NO_INTERRUPT,
};

static struct intel_uncore_ops snb_uncore_imc_ops = {

Thanks,
Kan

Quoted 1 line
arch/x86/events/intel/uncore.c +0 -0
 	} else {
author maintainer
#5 lore raw

Re: [PATCH 3/3] perf/x86: Add CAP_NO_INTERRUPT for uncore PMUs

Hi Kan,

Quoted 40 lines · Kan
Quoted 36 lines · Namhyung Kim

It doesn't support sampling in uncore PMU events. While it's technically possible to generate interrupts, let's treat it as if it has no interrupt in order to skip the freq adjust/unthrottling logic in the timer handler which is only meaningful to sampling events.

Also remove the sampling event check because it'd be done in the general code in the perf_event_open syscall.

---
 arch/x86/events/intel/uncore.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
arch/x86/events/intel/uncore.c +3 -5
index 69043e02e8a7..f7e6228bd1b1 100644
@@ -744,10 +744,6 @@ static int uncore_pmu_event_init(struct perf_event *event)
      if (pmu->func_id < 0)
              return -ENOENT;
 
-     /* Sampling not supported yet */
-     if (hwc->sample_period)
-             return -EINVAL;
-
      /*
       * Place all uncore events for a particular physical package
       * onto a single cpu
@@ -919,7 +915,12 @@ static int uncore_pmu_register(struct intel_uncore_pmu *pmu)
                      .stop           = uncore_pmu_event_stop,
                      .read           = uncore_pmu_event_read,
                      .module         = THIS_MODULE,
-                     .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
+                     /*
+                      * It doesn't allow sampling for uncore events, let's
+                      * treat the PMU has no interrupts to skip them in the
Quoted 8 lines

arch/x86/events/intel/uncore.c static int uncore_pmu_register(struct intel_uncore_pmu *pmu) · line 925

Quoted 5 lines
arch/x86/events/intel/uncore.c +3 -0
+                      * perf_adjust_freq_unthr_context().
+                      */
+                     .capabilities   = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
                      .attr_update    = pmu->type->attr_update,
              };

There is a special customized uncore PMU which needs the flag as well.

Ok, I will add that too.

Btw, during the work I noticed many PMU drivers didn't set the CAP_NO_INTERRUPT flag even if they didn't support sampling and rejected the sampling events manually in the ->event_init() callback.

I guess it's because the name of the flag is somewhat misleading. As the PMU drivers handle IRQ (for overflows), they thought they had interrupts and didn't set the flag. I think it'd be better to rename it to CAP_NO_SAMPLING to reveal the intention. And then we could just set the flag in the pmu.capabilities and remove the manual checks.

The benefit is it can skip the PMUs in the timer tick handler even if it needs to unthrottle some events. What do you think?

Thanks,
Namhyung

Quoted 21 lines
arch/x86/events/intel/uncore_snb.c +1 -1
index 7fd4334e12a1..46a63e291975 100644
@@ -1013,7 +1013,7 @@ static struct pmu snb_uncore_imc_pmu = {
        .start          = uncore_pmu_event_start,
        .stop           = uncore_pmu_event_stop,
        .read           = uncore_pmu_event_read,
-       .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
+       .capabilities   = PERF_PMU_CAP_NO_EXCLUDE |
 PERF_PMU_CAP_NO_INTERRUPT,
 };
 

static struct intel_uncore_ops snb_uncore_imc_ops = {

Thanks,
Kan

Quoted 1 line
arch/x86/events/intel/uncore.c +0 -0
      } else {
#6 lore raw

Re: [PATCH 3/3] perf/x86: Add CAP_NO_INTERRUPT for uncore PMUs

Quoted 58 lines · Namhyung Kim

Hi Kan,

Quoted 40 lines · Kan
Quoted 36 lines · Namhyung Kim

It doesn't support sampling in uncore PMU events. While it's technically possible to generate interrupts, let's treat it as if it has no interrupt in order to skip the freq adjust/unthrottling logic in the timer handler which is only meaningful to sampling events.

Also remove the sampling event check because it'd be done in the general code in the perf_event_open syscall.

---
 arch/x86/events/intel/uncore.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
arch/x86/events/intel/uncore.c +3 -5
index 69043e02e8a7..f7e6228bd1b1 100644
@@ -744,10 +744,6 @@ static int uncore_pmu_event_init(struct perf_event *event)
      if (pmu->func_id < 0)
              return -ENOENT;
 
-     /* Sampling not supported yet */
-     if (hwc->sample_period)
-             return -EINVAL;
-
      /*
       * Place all uncore events for a particular physical package
       * onto a single cpu
@@ -919,7 +915,12 @@ static int uncore_pmu_register(struct intel_uncore_pmu *pmu)
                      .stop           = uncore_pmu_event_stop,
                      .read           = uncore_pmu_event_read,
                      .module         = THIS_MODULE,
-                     .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
+                     /*
+                      * It doesn't allow sampling for uncore events, let's
+                      * treat the PMU has no interrupts to skip them in the
Quoted 8 lines

arch/x86/events/intel/uncore.c static int uncore_pmu_register(struct intel_uncore_pmu *pmu) · line 925

Quoted 5 lines
arch/x86/events/intel/uncore.c +3 -0
+                      * perf_adjust_freq_unthr_context().
+                      */
+                     .capabilities   = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
                      .attr_update    = pmu->type->attr_update,
              };

There is a special customized uncore PMU which needs the flag as well.

Ok, I will add that too.

Btw, during the work I noticed many PMU drivers didn't set the CAP_NO_INTERRUPT flag even if they didn't support sampling and rejected the sampling events manually in the ->event_init() callback.

Quoted 9 lines

I guess it's because the name of the flag is somewhat misleading. As the PMU drivers handle IRQ (for overflows), they thought they had interrupts and didn't set the flag. I think it'd be better to rename it to CAP_NO_SAMPLING to reveal the intention. And then we could just set the flag in the pmu.capabilities and remove the manual checks.

The benefit is it can skip the PMUs in the timer tick handler even if it needs to unthrottle some events. What do you think?

I agree. The current name is kind of misleading.

The patch, which introduced the flag (commit id 53b25335dd60 ("perf:
Disable sampled events if no PMU interrupt")), also tried to disable the sampled events on a no-sampling supported platform.

The renaming sounds good to me.

Thanks,
Kan

author maintainer
#7 lore raw

Re: [PATCH 3/3] perf/x86: Add CAP_NO_INTERRUPT for uncore PMUs

Quoted 72 lines · Kan
Quoted 58 lines · Namhyung Kim

Hi Kan,

Quoted 40 lines · Kan
Quoted 36 lines · Namhyung Kim

It doesn't support sampling in uncore PMU events. While it's technically possible to generate interrupts, let's treat it as if it has no interrupt in order to skip the freq adjust/unthrottling logic in the timer handler which is only meaningful to sampling events.

Also remove the sampling event check because it'd be done in the general code in the perf_event_open syscall.

---
 arch/x86/events/intel/uncore.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)
arch/x86/events/intel/uncore.c +3 -5
index 69043e02e8a7..f7e6228bd1b1 100644
@@ -744,10 +744,6 @@ static int uncore_pmu_event_init(struct perf_event *event)
      if (pmu->func_id < 0)
              return -ENOENT;
 
-     /* Sampling not supported yet */
-     if (hwc->sample_period)
-             return -EINVAL;
-
      /*
       * Place all uncore events for a particular physical package
       * onto a single cpu
@@ -919,7 +915,12 @@ static int uncore_pmu_register(struct intel_uncore_pmu *pmu)
                      .stop           = uncore_pmu_event_stop,
                      .read           = uncore_pmu_event_read,
                      .module         = THIS_MODULE,
-                     .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
+                     /*
+                      * It doesn't allow sampling for uncore events, let's
+                      * treat the PMU has no interrupts to skip them in the
Quoted 8 lines

arch/x86/events/intel/uncore.c static int uncore_pmu_register(struct intel_uncore_pmu *pmu) · line 925

Quoted 5 lines
arch/x86/events/intel/uncore.c +3 -0
+                      * perf_adjust_freq_unthr_context().
+                      */
+                     .capabilities   = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
                      .attr_update    = pmu->type->attr_update,
              };

There is a special customized uncore PMU which needs the flag as well.

Ok, I will add that too.

Btw, during the work I noticed many PMU drivers didn't set the CAP_NO_INTERRUPT flag even if they didn't support sampling and rejected the sampling events manually in the ->event_init() callback.

Quoted 9 lines

I guess it's because the name of the flag is somewhat misleading. As the PMU drivers handle IRQ (for overflows), they thought they had interrupts and didn't set the flag. I think it'd be better to rename it to CAP_NO_SAMPLING to reveal the intention. And then we could just set the flag in the pmu.capabilities and remove the manual checks.

The benefit is it can skip the PMUs in the timer tick handler even if it needs to unthrottle some events. What do you think?

Quoted 7 lines

I agree. The current name is kind of misleading.

The patch, which introduced the flag (commit id 53b25335dd60 ("perf:
Disable sampled events if no PMU interrupt")), also tried to disable the sampled events on a no-sampling supported platform.

The renaming sounds good to me.

Thank Kan for the review.

Peter and Ingo, would you please pick up the first two patches if you don't have any concerns? Then I can work on the renaming on top.

Thanks,
Namhyung

reviewer Reviewed-by on v1 1/3
#8 lore raw
In reply to Namhyung Kim
Quoted 10 lines · Namhyung Kim

It was unnecessarily disabling and enabling PMUs for each event. It should be done at PMU level. Add pmu_ctx->nr_freq counter to check it at each PMU. As pmu context has separate active lists for pinned group and flexible group, factor out a new function to do the job.

Another minor optimization is that it can skip PMUs w/ CAP_NO_INTERRUPT even if it needs to unthrottle sampling events.

Series:

Thanks,
Ian

Quoted 142 lines
---
 include/linux/perf_event.h |  1 +
 kernel/events/core.c       | 68 +++++++++++++++++++++++---------------
 2 files changed, 43 insertions(+), 26 deletions(-)
include/linux/perf_event.h +1 -0
index 0367d748fae0..3eb17dc89f5e 100644
@@ -879,6 +879,7 @@ struct perf_event_pmu_context {
 
        unsigned int                    nr_events;
        unsigned int                    nr_cgroups;
+       unsigned int                    nr_freq;
 
        atomic_t                        refcount; /* event <-> epc */
        struct rcu_head                 rcu_head;
kernel/events/core.c +42 -27
index 3eb26c2c6e65..53e2ad73102d 100644
@@ -2275,8 +2275,10 @@ event_sched_out(struct perf_event *event, struct perf_event_context *ctx)
 
        if (!is_software_event(event))
                cpc->active_oncpu--;
-       if (event->attr.freq && event->attr.sample_freq)
+       if (event->attr.freq && event->attr.sample_freq) {
                ctx->nr_freq--;
+               epc->nr_freq--;
+       }
        if (event->attr.exclusive || !cpc->active_oncpu)
                cpc->exclusive = 0;
 
@@ -2531,9 +2533,10 @@ event_sched_in(struct perf_event *event, struct perf_event_context *ctx)
 
        if (!is_software_event(event))
                cpc->active_oncpu++;
-       if (event->attr.freq && event->attr.sample_freq)
+       if (event->attr.freq && event->attr.sample_freq) {
                ctx->nr_freq++;
-
+               epc->nr_freq++;
+       }
        if (event->attr.exclusive)
                cpc->exclusive = 1;
 
@@ -4096,30 +4099,14 @@ static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bo
        }
 }
 
-/*
- * combine freq adjustment with unthrottling to avoid two passes over the
- * events. At the same time, make sure, having freq events does not change
- * the rate of unthrottling as that would introduce bias.
- */
-static void
-perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
+static void perf_adjust_freq_unthr_events(struct list_head *event_list)
 {
        struct perf_event *event;
        struct hw_perf_event *hwc;
        u64 now, period = TICK_NSEC;
        s64 delta;
 
-       /*
-        * only need to iterate over all events iff:
-        * - context have events in frequency mode (needs freq adjust)
-        * - there are events to unthrottle on this cpu
-        */
-       if (!(ctx->nr_freq || unthrottle))
-               return;
-
-       raw_spin_lock(&ctx->lock);
-
-       list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
+       list_for_each_entry(event, event_list, active_list) {
                if (event->state != PERF_EVENT_STATE_ACTIVE)
                        continue;
 
@@ -4127,8 +4114,6 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
                if (!event_filter_match(event))
                        continue;
 
-               perf_pmu_disable(event->pmu);
-
                hwc = &event->hw;
 
                if (hwc->interrupts == MAX_INTERRUPTS) {
@@ -4138,7 +4123,7 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
                }
 
                if (!event->attr.freq || !event->attr.sample_freq)
-                       goto next;
+                       continue;
 
                /*
                 * stop the event and update event->count
@@ -4160,8 +4145,39 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
                        perf_adjust_period(event, period, delta, false);
 
                event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
-       next:
-               perf_pmu_enable(event->pmu);
+       }
+}
+
+/*
+ * combine freq adjustment with unthrottling to avoid two passes over the
+ * events. At the same time, make sure, having freq events does not change
+ * the rate of unthrottling as that would introduce bias.
+ */
+static void
+perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
+{
+       struct perf_event_pmu_context *pmu_ctx;
+
+       /*
+        * only need to iterate over all events iff:
+        * - context have events in frequency mode (needs freq adjust)
+        * - there are events to unthrottle on this cpu
+        */
+       if (!(ctx->nr_freq || unthrottle))
+               return;
+
+       raw_spin_lock(&ctx->lock);
+
+       list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
+               if (!(pmu_ctx->nr_freq || unthrottle))
+                       continue;
+               if (pmu_ctx->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT)
+                       continue;
+
+               perf_pmu_disable(pmu_ctx->pmu);
+               perf_adjust_freq_unthr_events(&pmu_ctx->pinned_active);
+               perf_adjust_freq_unthr_events(&pmu_ctx->flexible_active);
+               perf_pmu_enable(pmu_ctx->pmu);
        }
 
        raw_spin_unlock(&ctx->lock);
--

2.43.0.rc1.413.gea7ed67945-goog

#9 lore raw
Quoted 10 lines · Ian Rogers
Quoted 9 lines · Namhyung Kim

It was unnecessarily disabling and enabling PMUs for each event. It should be done at PMU level. Add pmu_ctx->nr_freq counter to check it at each PMU. As pmu context has separate active lists for pinned group and flexible group, factor out a new function to do the job.

Another minor optimization is that it can skip PMUs w/ CAP_NO_INTERRUPT even if it needs to unthrottle sampling events.

Quoted 8 lines
Quoted 1 line

Series:

Thanks,
Ian

Can we have "Cc: [email protected]" for the whole series? This series should have a great performance improvement for all VMs in which perf sampling events without specifying period.

The key point behind is that disabling/enabling PMU in virtualized environment is super heavyweight which can reaches up to 50% of the CPU time, ie., When multiplxing is used in the VM, a vCPU on a pCPU can only use 50% of the resource, the other half was entirely wasted in host PMU code doing the enabling/disabling PMU.

Thanks.
-Mingwei

Quoted 142 lines
Quoted 142 lines
---
 include/linux/perf_event.h |  1 +
 kernel/events/core.c       | 68 +++++++++++++++++++++++---------------
 2 files changed, 43 insertions(+), 26 deletions(-)
include/linux/perf_event.h +1 -0
index 0367d748fae0..3eb17dc89f5e 100644
@@ -879,6 +879,7 @@ struct perf_event_pmu_context {
 
        unsigned int                    nr_events;
        unsigned int                    nr_cgroups;
+       unsigned int                    nr_freq;
 
        atomic_t                        refcount; /* event <-> epc */
        struct rcu_head                 rcu_head;
kernel/events/core.c +42 -27
index 3eb26c2c6e65..53e2ad73102d 100644
@@ -2275,8 +2275,10 @@ event_sched_out(struct perf_event *event, struct perf_event_context *ctx)
 
        if (!is_software_event(event))
                cpc->active_oncpu--;
-       if (event->attr.freq && event->attr.sample_freq)
+       if (event->attr.freq && event->attr.sample_freq) {
                ctx->nr_freq--;
+               epc->nr_freq--;
+       }
        if (event->attr.exclusive || !cpc->active_oncpu)
                cpc->exclusive = 0;
 
@@ -2531,9 +2533,10 @@ event_sched_in(struct perf_event *event, struct perf_event_context *ctx)
 
        if (!is_software_event(event))
                cpc->active_oncpu++;
-       if (event->attr.freq && event->attr.sample_freq)
+       if (event->attr.freq && event->attr.sample_freq) {
                ctx->nr_freq++;
-
+               epc->nr_freq++;
+       }
        if (event->attr.exclusive)
                cpc->exclusive = 1;
 
@@ -4096,30 +4099,14 @@ static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bo
        }
 }
 
-/*
- * combine freq adjustment with unthrottling to avoid two passes over the
- * events. At the same time, make sure, having freq events does not change
- * the rate of unthrottling as that would introduce bias.
- */
-static void
-perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
+static void perf_adjust_freq_unthr_events(struct list_head *event_list)
 {
        struct perf_event *event;
        struct hw_perf_event *hwc;
        u64 now, period = TICK_NSEC;
        s64 delta;
 
-       /*
-        * only need to iterate over all events iff:
-        * - context have events in frequency mode (needs freq adjust)
-        * - there are events to unthrottle on this cpu
-        */
-       if (!(ctx->nr_freq || unthrottle))
-               return;
-
-       raw_spin_lock(&ctx->lock);
-
-       list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
+       list_for_each_entry(event, event_list, active_list) {
                if (event->state != PERF_EVENT_STATE_ACTIVE)
                        continue;
 
@@ -4127,8 +4114,6 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
                if (!event_filter_match(event))
                        continue;
 
-               perf_pmu_disable(event->pmu);
-
                hwc = &event->hw;
 
                if (hwc->interrupts == MAX_INTERRUPTS) {
@@ -4138,7 +4123,7 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
                }
 
                if (!event->attr.freq || !event->attr.sample_freq)
-                       goto next;
+                       continue;
 
                /*
                 * stop the event and update event->count
@@ -4160,8 +4145,39 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
                        perf_adjust_period(event, period, delta, false);
 
                event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
-       next:
-               perf_pmu_enable(event->pmu);
+       }
+}
+
+/*
+ * combine freq adjustment with unthrottling to avoid two passes over the
+ * events. At the same time, make sure, having freq events does not change
+ * the rate of unthrottling as that would introduce bias.
+ */
+static void
+perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
+{
+       struct perf_event_pmu_context *pmu_ctx;
+
+       /*
+        * only need to iterate over all events iff:
+        * - context have events in frequency mode (needs freq adjust)
+        * - there are events to unthrottle on this cpu
+        */
+       if (!(ctx->nr_freq || unthrottle))
+               return;
+
+       raw_spin_lock(&ctx->lock);
+
+       list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
+               if (!(pmu_ctx->nr_freq || unthrottle))
+                       continue;
+               if (pmu_ctx->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT)
+                       continue;
+
+               perf_pmu_disable(pmu_ctx->pmu);
+               perf_adjust_freq_unthr_events(&pmu_ctx->pinned_active);
+               perf_adjust_freq_unthr_events(&pmu_ctx->flexible_active);
+               perf_pmu_enable(pmu_ctx->pmu);
        }
 
        raw_spin_unlock(&ctx->lock);
--

2.43.0.rc1.413.gea7ed67945-goog

author maintainer
#10 lore raw

Hi Mingwei,

Quoted 17 lines · Mingwei Zhang
Quoted 15 lines · Ian Rogers
Quoted 10 lines · Namhyung Kim

It was unnecessarily disabling and enabling PMUs for each event. It should be done at PMU level. Add pmu_ctx->nr_freq counter to check it at each PMU. As pmu context has separate active lists for pinned group and flexible group, factor out a new function to do the job.

Another minor optimization is that it can skip PMUs w/ CAP_NO_INTERRUPT even if it needs to unthrottle sampling events.

Series:

Quoted 7 lines
Quoted 3 lines

Thanks,
Ian

Can we have "Cc: [email protected]" for the whole series? This series should have a great performance improvement for all VMs in which perf sampling events without specifying period.

I was not sure if it's ok to have this performance fix in the stable series.

Thanks,
Namhyung

#11 lore raw

On Tue, Nov 21, 2023, Namhyung Kim wrote:

Hi Namhyung,

Quoted 24 lines

Hi Mingwei,

Quoted 21 lines · Mingwei Zhang
Quoted 18 lines · Ian Rogers
Quoted 10 lines · Namhyung Kim

It was unnecessarily disabling and enabling PMUs for each event. It should be done at PMU level. Add pmu_ctx->nr_freq counter to check it at each PMU. As pmu context has separate active lists for pinned group and flexible group, factor out a new function to do the job.

Another minor optimization is that it can skip PMUs w/ CAP_NO_INTERRUPT even if it needs to unthrottle sampling events.

Series:

Thanks,
Ian

Quoted 6 lines
Quoted 3 lines

Can we have "Cc: [email protected]" for the whole series? This series should have a great performance improvement for all VMs in which perf sampling events without specifying period.

I was not sure if it's ok to have this performance fix in the stable series.

Critical performance bug fix is ok to be added to stable tree, as the requirements are mentioned here:

https://www.kernel.org/doc/Documentation/process/stable-kernel-rules.rst

In particular, this patch satisfies the 2nd sub-bullet of the forth bullet.

But let me step back. Only this patch is needed with stable tag instead of the whole series. This patch impact 69 lines of code. It satisfies the rule of within 100 lines (bullet 3).

I will give a try and test it today or tomorrow and make sure we satisfy bullet 2.

Once it gets in, bullet 1 will be satisfied as well.

Overall, the intention is to improve PMU performance in VM as early as we can since we don't control the schedule of distro kernel upgrade and we don't control when end customers upgrade their running kernel. So I presume even adding to the stable tree may take years to see the result change. But if we don't do it, it may take way longer (since it does not contain a "Fixes" tag as well).

Thanks.
-Mingwei

Quoted 2 lines

Thanks,
Namhyung

#12 lore raw
Quoted 59 lines · Mingwei Zhang

On Tue, Nov 21, 2023, Namhyung Kim wrote:

Hi Namhyung,

Quoted 23 lines

Hi Mingwei,

Quoted 20 lines · Mingwei Zhang
Quoted 17 lines · Namhyung Kim
Quoted 10 lines

It was unnecessarily disabling and enabling PMUs for each event. It should be done at PMU level. Add pmu_ctx->nr_freq counter to check it at each PMU. As pmu context has separate active lists for pinned group and flexible group, factor out a new function to do the job.

Another minor optimization is that it can skip PMUs w/ CAP_NO_INTERRUPT even if it needs to unthrottle sampling events.

Series:

Thanks,
Ian

Quoted 6 lines
Quoted 3 lines

Can we have "Cc: [email protected]" for the whole series? This series should have a great performance improvement for all VMs in which perf sampling events without specifying period.

I was not sure if it's ok to have this performance fix in the stable series.

Critical performance bug fix is ok to be added to stable tree, as the requirements are mentioned here:

https://www.kernel.org/doc/Documentation/process/stable-kernel-rules.rst

In particular, this patch satisfies the 2nd sub-bullet of the forth bullet.

But let me step back. Only this patch is needed with stable tag instead of the whole series. This patch impact 69 lines of code. It satisfies the rule of within 100 lines (bullet 3).

I will give a try and test it today or tomorrow and make sure we satisfy bullet 2.

Once it gets in, bullet 1 will be satisfied as well.

Overall, the intention is to improve PMU performance in VM as early as we can since we don't control the schedule of distro kernel upgrade and we don't control when end customers upgrade their running kernel. So I presume even adding to the stable tree may take years to see the result change. But if we don't do it, it may take way longer (since it does not contain a "Fixes" tag as well).

Quoted 3 lines

Thanks.
-Mingwei

I have tested the code. Yes profiling results in the VM shows that it removes perf_adjust_freq_unthr_contex() as the hot spot. However, when running perf with sufficient events in frequency mode that triggers multiplexing. The overall performance overhead still reaches 60% per CPU (this overhead is invisible to vCPU).

At the host level, I have been monitoring the write MSRs and found that the repeated writes to 0x38f disappeared, indicating that this patch is indeed working. But on the other hand, I have noticed more frequent overflows and PMIs.

The more frequent overflows is shown in the writes to MSR 0x390 and reads to MSR 0x38e. I infer the more frequent PMIs from the much longer execution of 'vmx_vmexit()' shown in flamegraph.

Because of the above observation, it seems to me that this patch no longer satisfies the requirements of Ccing "[email protected]". I will double check and follow up on this one.

Thanks.
-Mingwei

Reviewed-by on v1 1/3
#13 lore raw
In reply to Namhyung Kim
Quoted 9 lines · Namhyung Kim

It was unnecessarily disabling and enabling PMUs for each event. It should be done at PMU level. Add pmu_ctx->nr_freq counter to check it at each PMU. As pmu context has separate active lists for pinned group and flexible group, factor out a new function to do the job.

Another minor optimization is that it can skip PMUs w/ CAP_NO_INTERRUPT even if it needs to unthrottle sampling events.

Thanks,
Kan

Quoted 139 lines
---
 include/linux/perf_event.h |  1 +
 kernel/events/core.c       | 68 +++++++++++++++++++++++---------------
 2 files changed, 43 insertions(+), 26 deletions(-)
include/linux/perf_event.h +1 -0
index 0367d748fae0..3eb17dc89f5e 100644
@@ -879,6 +879,7 @@ struct perf_event_pmu_context {
 
 	unsigned int			nr_events;
 	unsigned int			nr_cgroups;
+	unsigned int			nr_freq;
 
 	atomic_t			refcount; /* event <-> epc */
 	struct rcu_head			rcu_head;
kernel/events/core.c +42 -26
index 3eb26c2c6e65..53e2ad73102d 100644
@@ -2275,8 +2275,10 @@ event_sched_out(struct perf_event *event, struct perf_event_context *ctx)
 
 	if (!is_software_event(event))
 		cpc->active_oncpu--;
-	if (event->attr.freq && event->attr.sample_freq)
+	if (event->attr.freq && event->attr.sample_freq) {
 		ctx->nr_freq--;
+		epc->nr_freq--;
+	}
 	if (event->attr.exclusive || !cpc->active_oncpu)
 		cpc->exclusive = 0;
 
@@ -2531,9 +2533,10 @@ event_sched_in(struct perf_event *event, struct perf_event_context *ctx)
 
 	if (!is_software_event(event))
 		cpc->active_oncpu++;
-	if (event->attr.freq && event->attr.sample_freq)
+	if (event->attr.freq && event->attr.sample_freq) {
 		ctx->nr_freq++;
-
+		epc->nr_freq++;
+	}
 	if (event->attr.exclusive)
 		cpc->exclusive = 1;
 
@@ -4096,30 +4099,14 @@ static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bo
 	}
 }
 
-/*
- * combine freq adjustment with unthrottling to avoid two passes over the
- * events. At the same time, make sure, having freq events does not change
- * the rate of unthrottling as that would introduce bias.
- */
-static void
-perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
+static void perf_adjust_freq_unthr_events(struct list_head *event_list)
 {
 	struct perf_event *event;
 	struct hw_perf_event *hwc;
 	u64 now, period = TICK_NSEC;
 	s64 delta;
 
-	/*
-	 * only need to iterate over all events iff:
-	 * - context have events in frequency mode (needs freq adjust)
-	 * - there are events to unthrottle on this cpu
-	 */
-	if (!(ctx->nr_freq || unthrottle))
-		return;
-
-	raw_spin_lock(&ctx->lock);
-
-	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
+	list_for_each_entry(event, event_list, active_list) {
 		if (event->state != PERF_EVENT_STATE_ACTIVE)
 			continue;
 
@@ -4127,8 +4114,6 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
 		if (!event_filter_match(event))
 			continue;
 
-		perf_pmu_disable(event->pmu);
-
 		hwc = &event->hw;
 
 		if (hwc->interrupts == MAX_INTERRUPTS) {
@@ -4138,7 +4123,7 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
 		}
 
 		if (!event->attr.freq || !event->attr.sample_freq)
-			goto next;
+			continue;
 
 		/*
 		 * stop the event and update event->count
@@ -4160,8 +4145,39 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
 			perf_adjust_period(event, period, delta, false);
 
 		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
-	next:
-		perf_pmu_enable(event->pmu);
+	}
+}
+
+/*
+ * combine freq adjustment with unthrottling to avoid two passes over the
+ * events. At the same time, make sure, having freq events does not change
+ * the rate of unthrottling as that would introduce bias.
+ */
+static void
+perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
+{
+	struct perf_event_pmu_context *pmu_ctx;
+
+	/*
+	 * only need to iterate over all events iff:
+	 * - context have events in frequency mode (needs freq adjust)
+	 * - there are events to unthrottle on this cpu
+	 */
+	if (!(ctx->nr_freq || unthrottle))
+		return;
+
+	raw_spin_lock(&ctx->lock);
+
+	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
+		if (!(pmu_ctx->nr_freq || unthrottle))
+			continue;
+		if (pmu_ctx->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT)
+			continue;
+
+		perf_pmu_disable(pmu_ctx->pmu);
+		perf_adjust_freq_unthr_events(&pmu_ctx->pinned_active);
+		perf_adjust_freq_unthr_events(&pmu_ctx->flexible_active);
+		perf_pmu_enable(pmu_ctx->pmu);
 	}
 
 	raw_spin_unlock(&ctx->lock);
Tested-by on v1 1/3
#14 lore raw
In reply to Namhyung Kim
Quoted 9 lines · Namhyung Kim

It was unnecessarily disabling and enabling PMUs for each event. It should be done at PMU level. Add pmu_ctx->nr_freq counter to check it at each PMU. As pmu context has separate active lists for pinned group and flexible group, factor out a new function to do the job.

Another minor optimization is that it can skip PMUs w/ CAP_NO_INTERRUPT even if it needs to unthrottle sampling events.

Quoted 142 lines
---
 include/linux/perf_event.h |  1 +
 kernel/events/core.c       | 68 +++++++++++++++++++++++---------------
 2 files changed, 43 insertions(+), 26 deletions(-)
include/linux/perf_event.h +1 -0
index 0367d748fae0..3eb17dc89f5e 100644
@@ -879,6 +879,7 @@ struct perf_event_pmu_context {
 
 	unsigned int			nr_events;
 	unsigned int			nr_cgroups;
+	unsigned int			nr_freq;
 
 	atomic_t			refcount; /* event <-> epc */
 	struct rcu_head			rcu_head;
kernel/events/core.c +42 -26
index 3eb26c2c6e65..53e2ad73102d 100644
@@ -2275,8 +2275,10 @@ event_sched_out(struct perf_event *event, struct perf_event_context *ctx)
 
 	if (!is_software_event(event))
 		cpc->active_oncpu--;
-	if (event->attr.freq && event->attr.sample_freq)
+	if (event->attr.freq && event->attr.sample_freq) {
 		ctx->nr_freq--;
+		epc->nr_freq--;
+	}
 	if (event->attr.exclusive || !cpc->active_oncpu)
 		cpc->exclusive = 0;
 
@@ -2531,9 +2533,10 @@ event_sched_in(struct perf_event *event, struct perf_event_context *ctx)
 
 	if (!is_software_event(event))
 		cpc->active_oncpu++;
-	if (event->attr.freq && event->attr.sample_freq)
+	if (event->attr.freq && event->attr.sample_freq) {
 		ctx->nr_freq++;
-
+		epc->nr_freq++;
+	}
 	if (event->attr.exclusive)
 		cpc->exclusive = 1;
 
@@ -4096,30 +4099,14 @@ static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bo
 	}
 }
 
-/*
- * combine freq adjustment with unthrottling to avoid two passes over the
- * events. At the same time, make sure, having freq events does not change
- * the rate of unthrottling as that would introduce bias.
- */
-static void
-perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
+static void perf_adjust_freq_unthr_events(struct list_head *event_list)
 {
 	struct perf_event *event;
 	struct hw_perf_event *hwc;
 	u64 now, period = TICK_NSEC;
 	s64 delta;
 
-	/*
-	 * only need to iterate over all events iff:
-	 * - context have events in frequency mode (needs freq adjust)
-	 * - there are events to unthrottle on this cpu
-	 */
-	if (!(ctx->nr_freq || unthrottle))
-		return;
-
-	raw_spin_lock(&ctx->lock);
-
-	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
+	list_for_each_entry(event, event_list, active_list) {
 		if (event->state != PERF_EVENT_STATE_ACTIVE)
 			continue;
 
@@ -4127,8 +4114,6 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
 		if (!event_filter_match(event))
 			continue;
 
-		perf_pmu_disable(event->pmu);
-
 		hwc = &event->hw;
 
 		if (hwc->interrupts == MAX_INTERRUPTS) {
@@ -4138,7 +4123,7 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
 		}
 
 		if (!event->attr.freq || !event->attr.sample_freq)
-			goto next;
+			continue;
 
 		/*
 		 * stop the event and update event->count
@@ -4160,8 +4145,39 @@ perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
 			perf_adjust_period(event, period, delta, false);
 
 		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
-	next:
-		perf_pmu_enable(event->pmu);
+	}
+}
+
+/*
+ * combine freq adjustment with unthrottling to avoid two passes over the
+ * events. At the same time, make sure, having freq events does not change
+ * the rate of unthrottling as that would introduce bias.
+ */
+static void
+perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle)
+{
+	struct perf_event_pmu_context *pmu_ctx;
+
+	/*
+	 * only need to iterate over all events iff:
+	 * - context have events in frequency mode (needs freq adjust)
+	 * - there are events to unthrottle on this cpu
+	 */
+	if (!(ctx->nr_freq || unthrottle))
+		return;
+
+	raw_spin_lock(&ctx->lock);
+
+	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
+		if (!(pmu_ctx->nr_freq || unthrottle))
+			continue;
+		if (pmu_ctx->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT)
+			continue;
+
+		perf_pmu_disable(pmu_ctx->pmu);
+		perf_adjust_freq_unthr_events(&pmu_ctx->pinned_active);
+		perf_adjust_freq_unthr_events(&pmu_ctx->flexible_active);
+		perf_pmu_enable(pmu_ctx->pmu);
 	}
 
 	raw_spin_unlock(&ctx->lock);
Signature · 1 line
2.43.0.rc1.413.gea7ed67945-goog