perf/core: Optimize event reschedule for a PMU
Namhyung Kim
maintainerPATCH 1/1 perf/core: Optimize event reschedule for a PMU
Current ctx_resched() reschedules every events in all PMUs in the context even if it only needs to do it for a single event. This is the case when it opens a new event or enables an existing one. What we want is to reschedule events in the PMU only. Also perf_pmu_resched() currently calls ctx_resched() without PMU information.
Let's add __perf_pmu_resched() to do the work for the given PMU only. The context time should be updated by ctx_sched_{out,in}(EVENT_TIME) outside from it. And change the __pmu_ctx_sched_in() to be symmetrical to the _sched_out() for its arguments so that it can be called easily in the __perf_pmu_resched().
Note that __perf_install_in_context() should call ctx_resched() for the very first event in the context in order to set ctx->is_active. Later events can be handled by __perf_pmu_resched().
Care should be taken when it installs a task event for a PMU and there's no CPU event for the PMU. __perf_pmu_resched() will ask the CPU PMU context to schedule any events in it according to the group info. But as the PMU context was not activated, it didn't set the event context pointer. So I added new NULL checks in the __pmu_ctx_sched_{in,out}.
With this change I can get 4x speed up (but actually it's proportional to the number of uncore PMU events) on a 2-socket Intel EMR machine in opening and closing a perf event for the core PMU in a loop while there are a bunch of uncore PMU events active on the CPU. The test code (stress-pmu) follows below.
Before)
# ./stress-pmu delta: 0.087068 sec (870 usec/op)
After)
# ./stress-pmu delta: 0.021440 sec (214 usec/op)
---
$ cat stress-pmu.c #include <stdio.h> #include <unistd.h> #include <linux/perf_event.h> #include <sys/syscall.h> #include <sys/time.h>
/* from uncore cpumask on EMR */
#define TARGET_CPU 60
#define LOOP 100
#define US2S 1000000
int open_perf_event(int type, int config)
{
struct perf_event_attr attr = {
.type = type,
.config = config,
};
int fd;
fd = syscall(SYS_perf_event_open, &attr, /*pid=*/-1, TARGET_CPU,
/*group_fd=*/-1, /*flags=*/0);
if (fd < 0)
printf("perf_event_open failed (type=%d, config=%d): %m\n", type, config);
return fd;
}
int main(int argc, char *argv[])
{
struct timeval ts1, ts2;
unsigned long long delta;
int target_cpu = TARGET_CPU; /* open random uncore PMU events */
for (int i = 0; i < 100; i++)
open_perf_event(/*type=*/i + 20, /*config=*/0);
gettimeofday(&ts1, NULL); for (int i = 0; i < LOOP; i++) close(open_perf_event(PERF_TYPE_HARDWARE, PERF_COUNT_HW_CPU_CYCLES)); gettimeofday(&ts2, NULL);
delta = ts2.tv_sec * US2S + ts2.tv_usec - (ts1.tv_sec * US2S + ts1.tv_usec); printf("delta: %llu.%06llu sec (%llu usec/op)\n",
delta / US2S, delta % US2S, delta / LOOP);
return 0;
}
--- kernel/events/core.c | 101 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 88 insertions(+), 13 deletions(-)
kernel/events/core.c +88 -13
@@ -709,6 +709,10 @@ static void perf_ctx_enable(struct perf_event_context *ctx, bool cgroup)static void ctx_sched_out (struct perf_event_context *ctx,enum event_type_t event_type);static void ctx_sched_in (struct perf_event_context *ctx,enum event_type_t event_type); +static void __pmu_ctx_sched_out (struct perf_event_pmu_context *pmu_ctx, +enum event_type_t event_type); +static void __pmu_ctx_sched_in (struct perf_event_pmu_context *pmu_ctx, +enum event_type_t event_type);#ifdef CONFIG_CGROUP_PERF
@@ -2683,11 +2687,6 @@ static void perf_event_sched_in(struct perf_cpu_context *cpuctx,* event_type is a bit mask of the types of events involved. For CPU events, * event_type is only either EVENT_PINNED or EVENT_FLEXIBLE. */ -/* -* XXX: ctx_resched() reschedule entire perf_event_context while adding new -* event to the context or enabling existing event in the context. We can -* probably optimize it by rescheduling only affected pmu_ctx. -*/ static void ctx_resched (struct perf_cpu_context *cpuctx,struct perf_event_context *task_ctx,enum event_type_t event_type)
@@ -2728,13 +2727,62 @@ static void ctx_resched(struct perf_cpu_context *cpuctx,perf_ctx_enable (task_ctx, false); } +static void __perf_pmu_resched (struct pmu *pmu, +struct perf_event_context *task_ctx, +enum event_type_t event_type) +{ +bool cpu_event = !!(event_type &EVENT_CPU ); +struct perf_event_pmu_context *epc =NULL ; +struct perf_cpu_pmu_context *cpc =this_cpu_ptr (pmu->cpu_pmu_context); + +/* +* If pinned groups are involved, flexible groups also need to be +* scheduled out. +*/ +if (event_type &EVENT_PINNED ) + event_type |=EVENT_FLEXIBLE ; + + event_type &=EVENT_ALL ; + +perf_pmu_disable (pmu); +if (task_ctx) { +if (WARN_ON_ONCE (!cpc->task_epc || cpc->task_epc->ctx != task_ctx)) + goto out; + + epc = cpc->task_epc; +__pmu_ctx_sched_out (epc, event_type); + } + +/* +* Decide which cpu ctx groups to schedule out based on the types +* of events that caused rescheduling: +* - EVENT_CPU: schedule out corresponding groups; +* - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups; +* - otherwise, do nothing more. +*/ +if (cpu_event) +__pmu_ctx_sched_out (&cpc->epc, event_type); +else if (event_type &EVENT_PINNED ) +__pmu_ctx_sched_out (&cpc->epc,EVENT_FLEXIBLE ); + +__pmu_ctx_sched_in (&cpc->epc,EVENT_PINNED ); +if (task_ctx) +__pmu_ctx_sched_in (epc,EVENT_PINNED ); +__pmu_ctx_sched_in (&cpc->epc,EVENT_FLEXIBLE ); +if (task_ctx) +__pmu_ctx_sched_in (epc,EVENT_FLEXIBLE ); + +out: +perf_pmu_enable (pmu); +} +void perf_pmu_resched (struct pmu *pmu) {struct perf_cpu_context *cpuctx =this_cpu_ptr (&perf_cpu_context);struct perf_event_context *task_ctx = cpuctx->task_ctx;perf_ctx_lock (cpuctx, task_ctx); -ctx_resched (cpuctx, task_ctx,EVENT_ALL |EVENT_CPU ); +__perf_pmu_resched (pmu, task_ctx,EVENT_ALL |EVENT_CPU );perf_ctx_unlock (cpuctx, task_ctx); }
@@ -2792,7 +2840,14 @@ static int __perf_install_in_context(void *info)if (reprogram) {ctx_sched_out (ctx,EVENT_TIME );add_event_to_ctx (event, ctx); -ctx_resched (cpuctx, task_ctx,get_event_type (event)); +if (ctx->nr_events ==1 ) { +/* The first event needs to set ctx->is_active. */ +ctx_resched (cpuctx, task_ctx,get_event_type (event)); + }else { +__perf_pmu_resched (event->pmu_ctx->pmu, task_ctx, +get_event_type (event)); +ctx_sched_in (ctx,EVENT_TIME ); + } }else {add_event_to_ctx (event, ctx); }
@@ -2962,7 +3017,8 @@ static void __perf_event_enable(struct perf_event *event,if (ctx->task)WARN_ON_ONCE (task_ctx != ctx); -ctx_resched (cpuctx, task_ctx,get_event_type (event)); +__perf_pmu_resched (event->pmu_ctx->pmu, task_ctx,get_event_type (event)); +ctx_sched_in (ctx,EVENT_TIME ); }/*
@@ -3230,6 +3286,13 @@ static void __pmu_ctx_sched_out(struct perf_event_pmu_context *pmu_ctx,struct perf_event *event, *tmp;struct pmu *pmu = pmu_ctx->pmu; +/* +* CPU's pmu_ctx might not be active when __perf_pmu_resched() is called +* for task events and there's no cpu events. +*/ +if (ctx ==NULL ) +return ; +if (ctx->task && !ctx->is_active) {struct perf_cpu_pmu_context *cpc;
@@ -3872,10 +3935,22 @@ static void ctx_groups_sched_in(struct perf_event_context *ctx, } } -static void __pmu_ctx_sched_in (struct perf_event_context *ctx, -struct pmu *pmu) +static void __pmu_ctx_sched_in (struct perf_event_pmu_context *pmu_ctx, +enum event_type_t event_type) { -pmu_groups_sched_in (ctx, &ctx->flexible_groups, pmu); +struct perf_event_context *ctx = pmu_ctx->ctx; + +/* +* CPU's pmu_ctx might not be active when __perf_pmu_resched() is called +* for task events and there's no cpu events. +*/ +if (ctx ==NULL ) +return ; + +if (event_type &EVENT_PINNED ) +pmu_groups_sched_in (ctx, &ctx->pinned_groups, pmu_ctx->pmu); +if (event_type &EVENT_FLEXIBLE ) +pmu_groups_sched_in (ctx, &ctx->flexible_groups, pmu_ctx->pmu); } static void
@@ -4309,14 +4384,14 @@ static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc)update_context_time (&cpuctx->ctx);__pmu_ctx_sched_out (cpu_epc,EVENT_FLEXIBLE );rotate_ctx (&cpuctx->ctx, cpu_event); -__pmu_ctx_sched_in (&cpuctx->ctx, pmu); +__pmu_ctx_sched_in (cpu_epc,EVENT_FLEXIBLE ); }if (task_event)rotate_ctx (task_epc->ctx, task_event);if (task_event || (task_epc && cpu_event)) -__pmu_ctx_sched_in (task_epc->ctx, pmu); +__pmu_ctx_sched_in (task_epc,EVENT_FLEXIBLE );perf_pmu_enable (pmu);perf_ctx_unlock (cpuctx, cpuctx->task_ctx);
Signature · 1 line
2.46.0.rc1.232.g9752f9e123-goog
Peter Zijlstra
maintainerQuoted 42 lines · Namhyung Kim
kernel/events/core.c +38
-0@@ -2728,13 +2727,62 @@ static void ctx_resched(struct perf_cpu_context *cpuctx,perf_ctx_enable (task_ctx, false); } +static void __perf_pmu_resched (struct pmu *pmu, +struct perf_event_context *task_ctx, +enum event_type_t event_type) +{ +bool cpu_event = !!(event_type &EVENT_CPU ); +struct perf_event_pmu_context *epc =NULL ; +struct perf_cpu_pmu_context *cpc =this_cpu_ptr (pmu->cpu_pmu_context); + +/* +* If pinned groups are involved, flexible groups also need to be +* scheduled out. +*/ +if (event_type &EVENT_PINNED ) + event_type |=EVENT_FLEXIBLE ; + + event_type &=EVENT_ALL ; + +perf_pmu_disable (pmu); +if (task_ctx) { +if (WARN_ON_ONCE (!cpc->task_epc || cpc->task_epc->ctx != task_ctx)) + goto out; + + epc = cpc->task_epc; +__pmu_ctx_sched_out (epc, event_type); + } + +/* +* Decide which cpu ctx groups to schedule out based on the types +* of events that caused rescheduling: +* - EVENT_CPU: schedule out corresponding groups; +* - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups; +* - otherwise, do nothing more. +*/ +if (cpu_event) +__pmu_ctx_sched_out (&cpc->epc, event_type); +else if (event_type &EVENT_PINNED ) +__pmu_ctx_sched_out (&cpc->epc,EVENT_FLEXIBLE ); +
kernel/events/core.c static void ctx_resched(struct perf_cpu_context *cpuctx, · line 2777
Quoted 10 lines
kernel/events/core.c +10
-0+__pmu_ctx_sched_in (&cpc->epc,EVENT_PINNED ); +if (task_ctx) +__pmu_ctx_sched_in (epc,EVENT_PINNED ); +__pmu_ctx_sched_in (&cpc->epc,EVENT_FLEXIBLE ); +if (task_ctx) +__pmu_ctx_sched_in (epc,EVENT_FLEXIBLE ); + +out: +perf_pmu_enable (pmu); +}
I so dislike duplication...
So lets see, ctx_resched() has pmu_ctx iterations in:
perf_ctx_{en,dis}able()
ctx_sched_{in,out}()
Can't we punch a 'struct pmu *pmu' argument through those callchains and short-circuit the iteration when !NULL?
The alternative would be to lift the iteration I suppose.
Namhyung Kim
maintainerHi Peter,
Quoted 57 lines · Peter Zijlstra
Quoted 42 lines · Namhyung Kim
kernel/events/core.c +38
-0@@ -2728,13 +2727,62 @@ static void ctx_resched(struct perf_cpu_context *cpuctx,perf_ctx_enable (task_ctx, false); } +static void __perf_pmu_resched (struct pmu *pmu, +struct perf_event_context *task_ctx, +enum event_type_t event_type) +{ +bool cpu_event = !!(event_type &EVENT_CPU ); +struct perf_event_pmu_context *epc =NULL ; +struct perf_cpu_pmu_context *cpc =this_cpu_ptr (pmu->cpu_pmu_context); + +/* +* If pinned groups are involved, flexible groups also need to be +* scheduled out. +*/ +if (event_type &EVENT_PINNED ) + event_type |=EVENT_FLEXIBLE ; + + event_type &=EVENT_ALL ; + +perf_pmu_disable (pmu); +if (task_ctx) { +if (WARN_ON_ONCE (!cpc->task_epc || cpc->task_epc->ctx != task_ctx)) + goto out; + + epc = cpc->task_epc; +__pmu_ctx_sched_out (epc, event_type); + } + +/* +* Decide which cpu ctx groups to schedule out based on the types +* of events that caused rescheduling: +* - EVENT_CPU: schedule out corresponding groups; +* - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups; +* - otherwise, do nothing more. +*/ +if (cpu_event) +__pmu_ctx_sched_out (&cpc->epc, event_type); +else if (event_type &EVENT_PINNED ) +__pmu_ctx_sched_out (&cpc->epc,EVENT_FLEXIBLE ); +kernel/events/core.c static void ctx_resched(struct perf_cpu_context *cpuctx, · line 2777
Quoted 10 lines
kernel/events/core.c +10
-0+__pmu_ctx_sched_in (&cpc->epc,EVENT_PINNED ); +if (task_ctx) +__pmu_ctx_sched_in (epc,EVENT_PINNED ); +__pmu_ctx_sched_in (&cpc->epc,EVENT_FLEXIBLE ); +if (task_ctx) +__pmu_ctx_sched_in (epc,EVENT_FLEXIBLE ); + +out: +perf_pmu_enable (pmu); +}I so dislike duplication...
Quoted 7 lines
So lets see,
ctx_resched()haspmu_ctxiterations in:perf_ctx_{en,dis}able()
ctx_sched_{in,out}()Can't we punch a 'struct pmu *pmu' argument through those callchains and short-circuit the iteration when !NULL?
Sure, will do that in v2.
Thanks,
Namhyung
Quoted 2 lines
The alternative would be to lift the iteration I suppose.
Quoted 30 lines · Namhyung Kim
Current
ctx_resched()reschedules every events in all PMUs in the context even if it only needs to do it for a single event. This is the case when it opens a new event or enables an existing one. What we want is to reschedule events in the PMU only. Alsoperf_pmu_resched()currently callsctx_resched()without PMU information.Let's add
__perf_pmu_resched()to do the work for the given PMU only. The context time should be updated by ctx_sched_{out,in}(EVENT_TIME) outside from it. And change the__pmu_ctx_sched_in()to be symmetrical to the_sched_out()for its arguments so that it can be called easily in the__perf_pmu_resched().Note that
__perf_install_in_context()should callctx_resched()for the very first event in the context in order to setctx->is_active. Later events can be handled by__perf_pmu_resched().Care should be taken when it installs a task event for a PMU and there's no CPU event for the PMU.
__perf_pmu_resched()will ask the CPU PMU context to schedule any events in it according to the group info. But as the PMU context was not activated, it didn't set the event context pointer. So I added new NULL checks in the __pmu_ctx_sched_{in,out}.With this change I can get 4x speed up (but actually it's proportional to the number of uncore PMU events) on a 2-socket Intel EMR machine in opening and closing a perf event for the core PMU in a loop while there are a bunch of uncore PMU events active on the CPU. The test code (stress-pmu) follows below.
Quoted 3 lines
Before)
# ./stress-pmu delta: 0.087068 sec (870 usec/op)
Hi Namhyung,
I wonder how I could test the performance boost on the virtualized environment. So, I assume this will have a better performance by reducing the number of wrmsrs to event selectors and counters?
I wonder if I need to run multiple instances of stress-pmu to increase the number of PMU context switches?
Thanks.
-Mingwei
Quoted 236 lines
After)
# ./stress-pmu delta: 0.021440 sec (214 usec/op)Signed-off-by Namhyung Kim <[email protected]>---
$ cat stress-pmu.c #include <stdio.h> #include <unistd.h> #include <linux/perf_event.h> #include <sys/syscall.h> #include <sys/time.h>/* from uncore cpumask on EMR */
#defineTARGET_CPU60#define LOOP 100 #define US2S 1000000 int open_perf_event(int type, int config) { struct perf_event_attr attr = { .type = type, .config = config, }; int fd;fd = syscall(SYS_perf_event_open, &attr, /*pid=*/-1, TARGET_CPU, /*group_fd=*/-1, /*flags=*/0); if (fd < 0) printf("perf_event_open failed (type=%d, config=%d): %m\n", type, config); return fd;}
int main(int argc, char *argv[]) { struct timeval ts1, ts2; unsigned long long delta; int target_cpu = TARGET_CPU;/* open random uncore PMU events */ for (int i = 0; i < 100; i++) open_perf_event(/*type=*/i + 20, /*config=*/0); gettimeofday(&ts1, NULL); for (int i = 0; i < LOOP; i++) close(open_perf_event(PERF_TYPE_HARDWARE, PERF_COUNT_HW_CPU_CYCLES)); gettimeofday(&ts2, NULL); delta = ts2.tv_sec * US2S + ts2.tv_usec - (ts1.tv_sec * US2S + ts1.tv_usec); printf("delta: %llu.%06llu sec (%llu usec/op)\n", delta / US2S, delta % US2S, delta / LOOP); return 0; } --- kernel/events/core.c | 101 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 88 insertions(+), 13 deletions(-)kernel/events/core.c +88
-14@@ -709,6 +709,10 @@ static void perf_ctx_enable(struct perf_event_context *ctx, bool cgroup)static void ctx_sched_out (struct perf_event_context *ctx,enum event_type_t event_type);static void ctx_sched_in (struct perf_event_context *ctx,enum event_type_t event_type); +static void __pmu_ctx_sched_out (struct perf_event_pmu_context *pmu_ctx, +enum event_type_t event_type); +static void __pmu_ctx_sched_in (struct perf_event_pmu_context *pmu_ctx, +enum event_type_t event_type);#ifdef CONFIG_CGROUP_PERF @@ -2683,11 +2687,6 @@ static void perf_event_sched_in(struct perf_cpu_context *cpuctx,* event_type is a bit mask of the types of events involved. For CPU events, * event_type is only either EVENT_PINNED or EVENT_FLEXIBLE. */ -/* -* XXX: ctx_resched() reschedule entire perf_event_context while adding new -* event to the context or enabling existing event in the context. We can -* probably optimize it by rescheduling only affected pmu_ctx. -*/ static void ctx_resched (struct perf_cpu_context *cpuctx,struct perf_event_context *task_ctx,enum event_type_t event_type)@@ -2728,13 +2727,62 @@ static void ctx_resched(struct perf_cpu_context *cpuctx,perf_ctx_enable (task_ctx, false); } +static void __perf_pmu_resched (struct pmu *pmu, +struct perf_event_context *task_ctx, +enum event_type_t event_type) +{ +bool cpu_event = !!(event_type &EVENT_CPU ); +struct perf_event_pmu_context *epc =NULL ; +struct perf_cpu_pmu_context *cpc =this_cpu_ptr (pmu->cpu_pmu_context); + +/* +* If pinned groups are involved, flexible groups also need to be +* scheduled out. +*/ +if (event_type &EVENT_PINNED ) + event_type |=EVENT_FLEXIBLE ; + + event_type &=EVENT_ALL ; + +perf_pmu_disable (pmu); +if (task_ctx) { +if (WARN_ON_ONCE (!cpc->task_epc || cpc->task_epc->ctx != task_ctx)) + goto out; + + epc = cpc->task_epc; +__pmu_ctx_sched_out (epc, event_type); + } + +/* +* Decide which cpu ctx groups to schedule out based on the types +* of events that caused rescheduling: +* - EVENT_CPU: schedule out corresponding groups; +* - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups; +* - otherwise, do nothing more. +*/ +if (cpu_event) +__pmu_ctx_sched_out (&cpc->epc, event_type); +else if (event_type &EVENT_PINNED ) +__pmu_ctx_sched_out (&cpc->epc,EVENT_FLEXIBLE ); + +__pmu_ctx_sched_in (&cpc->epc,EVENT_PINNED ); +if (task_ctx) +__pmu_ctx_sched_in (epc,EVENT_PINNED ); +__pmu_ctx_sched_in (&cpc->epc,EVENT_FLEXIBLE ); +if (task_ctx) +__pmu_ctx_sched_in (epc,EVENT_FLEXIBLE ); + +out: +perf_pmu_enable (pmu); +} +void perf_pmu_resched (struct pmu *pmu) {struct perf_cpu_context *cpuctx =this_cpu_ptr (&perf_cpu_context);struct perf_event_context *task_ctx = cpuctx->task_ctx;perf_ctx_lock (cpuctx, task_ctx); -ctx_resched (cpuctx, task_ctx,EVENT_ALL |EVENT_CPU ); +__perf_pmu_resched (pmu, task_ctx,EVENT_ALL |EVENT_CPU );perf_ctx_unlock (cpuctx, task_ctx); }@@ -2792,7 +2840,14 @@ static int __perf_install_in_context(void *info)if (reprogram) {ctx_sched_out (ctx,EVENT_TIME );add_event_to_ctx (event, ctx); -ctx_resched (cpuctx, task_ctx,get_event_type (event)); +if (ctx->nr_events ==1 ) { +/* The first event needs to set ctx->is_active. */ +ctx_resched (cpuctx, task_ctx,get_event_type (event)); + }else { +__perf_pmu_resched (event->pmu_ctx->pmu, task_ctx, +get_event_type (event)); +ctx_sched_in (ctx,EVENT_TIME ); + } }else {add_event_to_ctx (event, ctx); }@@ -2962,7 +3017,8 @@ static void __perf_event_enable(struct perf_event *event,if (ctx->task)WARN_ON_ONCE (task_ctx != ctx); -ctx_resched (cpuctx, task_ctx,get_event_type (event)); +__perf_pmu_resched (event->pmu_ctx->pmu, task_ctx,get_event_type (event)); +ctx_sched_in (ctx,EVENT_TIME ); }/* @@ -3230,6 +3286,13 @@ static void __pmu_ctx_sched_out(struct perf_event_pmu_context *pmu_ctx,struct perf_event *event, *tmp;struct pmu *pmu = pmu_ctx->pmu; +/* +* CPU's pmu_ctx might not be active when __perf_pmu_resched() is called +* for task events and there's no cpu events. +*/ +if (ctx ==NULL ) +return ; +if (ctx->task && !ctx->is_active) {struct perf_cpu_pmu_context *cpc;@@ -3872,10 +3935,22 @@ static void ctx_groups_sched_in(struct perf_event_context *ctx, } } -static void __pmu_ctx_sched_in (struct perf_event_context *ctx, -struct pmu *pmu) +static void __pmu_ctx_sched_in (struct perf_event_pmu_context *pmu_ctx, +enum event_type_t event_type) { -pmu_groups_sched_in (ctx, &ctx->flexible_groups, pmu); +struct perf_event_context *ctx = pmu_ctx->ctx; + +/* +* CPU's pmu_ctx might not be active when __perf_pmu_resched() is called +* for task events and there's no cpu events. +*/ +if (ctx ==NULL ) +return ; + +if (event_type &EVENT_PINNED ) +pmu_groups_sched_in (ctx, &ctx->pinned_groups, pmu_ctx->pmu); +if (event_type &EVENT_FLEXIBLE ) +pmu_groups_sched_in (ctx, &ctx->flexible_groups, pmu_ctx->pmu); } static void@@ -4309,14 +4384,14 @@ static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc)update_context_time (&cpuctx->ctx);__pmu_ctx_sched_out (cpu_epc,EVENT_FLEXIBLE );rotate_ctx (&cpuctx->ctx, cpu_event); -__pmu_ctx_sched_in (&cpuctx->ctx, pmu); +__pmu_ctx_sched_in (cpu_epc,EVENT_FLEXIBLE ); }if (task_event)rotate_ctx (task_epc->ctx, task_event);if (task_event || (task_epc && cpu_event)) -__pmu_ctx_sched_in (task_epc->ctx, pmu); +__pmu_ctx_sched_in (task_epc,EVENT_FLEXIBLE );perf_pmu_enable (pmu);perf_ctx_unlock (cpuctx, cpuctx->task_ctx); --2.46.0.rc1.232.g9752f9e123-goog
Namhyung Kim
maintainerHi Mingwei,
Quoted 36 lines · Mingwei Zhang
Quoted 30 lines · Namhyung Kim
Current
ctx_resched()reschedules every events in all PMUs in the context even if it only needs to do it for a single event. This is the case when it opens a new event or enables an existing one. What we want is to reschedule events in the PMU only. Alsoperf_pmu_resched()currently callsctx_resched()without PMU information.Let's add
__perf_pmu_resched()to do the work for the given PMU only. The context time should be updated by ctx_sched_{out,in}(EVENT_TIME) outside from it. And change the__pmu_ctx_sched_in()to be symmetrical to the_sched_out()for its arguments so that it can be called easily in the__perf_pmu_resched().Note that
__perf_install_in_context()should callctx_resched()for the very first event in the context in order to setctx->is_active. Later events can be handled by__perf_pmu_resched().Care should be taken when it installs a task event for a PMU and there's no CPU event for the PMU.
__perf_pmu_resched()will ask the CPU PMU context to schedule any events in it according to the group info. But as the PMU context was not activated, it didn't set the event context pointer. So I added new NULL checks in the __pmu_ctx_sched_{in,out}.With this change I can get 4x speed up (but actually it's proportional to the number of uncore PMU events) on a 2-socket Intel EMR machine in opening and closing a perf event for the core PMU in a loop while there are a bunch of uncore PMU events active on the CPU. The test code (stress-pmu) follows below.
Quoted 3 lines
Before)
# ./stress-pmu delta: 0.087068 sec (870 usec/op)
Quoted 5 lines
Hi Namhyung,
I wonder how I could test the performance boost on the virtualized environment. So, I assume this will have a better performance by reducing the number of wrmsrs to event selectors and counters?
Right.
Quoted 3 lines
I wonder if I need to run multiple instances of stress-pmu to increase the number of PMU context switches?
Yep, I think it'd work. Basically anything that opens more events in different PMUs. But make sure the vcpu thread is running on the affected CPU (60 in my test).
Thanks,
Namhyung