sched/core: For a new metric, add vcpu->nonaffine_time
PATCH 1/2 sched/core: For a new metric, add vcpu->nonaffine_time
To monitor the effectiveness of vCPU soft-affinity on NUMA hosts, we'd like to create a vCPU metric that accumulates the amount of vCPU time running outside of the soft affinity mask of the sched-unit:
- Add a new time counter, nonaffine_time to struct vcpu.
- Accumulate the nonaffine_time on vcpu_runstate_change():
Account the time spent in the RUNSTATE_running state outside of unit->cpu_soft_affinity: It is always initialized and defaults to cpumask_all (bits for all NR_CPUS set), so we only accumulate nonaffine time when the vCPU runs on an unset CPU (non-affine).
In the next patch, this field can be used to retrieve the accumulated nonaffine running time e.g. using vcpu_runstate_get().
--- xen/common/sched/core.c | 20 ++++++++++++++++++++ xen/include/xen/sched.h | 11 +++++++++++ 2 files changed, 31 insertions(+)
xen/common/sched/core.c +20 -0
@@ -260,6 +260,23 @@ static inline void vcpu_urgent_count_update(struct vcpu *v) } } +/* +* For accounting non-affine running time of a vCPU, return true if +* the vCPU is running in RUNSTATE_running state while not on a CPU +* in unit->cpu_soft_affinity. +*/ +static inline bool nonaffine (const struct vcpu *v, +const struct sched_unit *unit) +{ +/* +* unit->cpu_soft_affinity is always initialized and defaults to +* cpumask_all (bits for all NR_CPUS set), so we only accumulate +* nonaffine time when the vCPU runs on an unset CPU (non-affine). +*/ +return v->runstate.state == RUNSTATE_running && + !cpumask_test_cpu (v->processor, unit->cpu_soft_affinity); +} +static inline void vcpu_runstate_change (struct vcpu *v,int new_state,s_time_t new_entry_time) {
@@ -285,6 +302,9 @@ static inline void vcpu_runstate_change( { v->runstate.time[v->runstate.state] += delta; v->runstate.state_entry_time = new_entry_time; + +if (nonaffine (v, unit) )/* When running nonaffine, add delta */ + v->nonaffine_time += delta; } v->runstate.state = new_state;
xen/include/xen/sched.h +11 -0
@@ -198,7 +198,18 @@ struct vcpustruct sched_unit *sched_unit; +/* +* The struct vcpu_runstate_info contains the vCPU time spent +* in each runstate and the entry time of the current runstate. +* +* Note: This field is used for the guest runstate shared memory area. +* Therefore, it is part of the frozen guest API and cannot be changed. +*/ struct vcpu_runstate_info runstate; + +/* vCPU time running outside the scheduling unit's soft_affinity mask */ +uint64_t nonaffine_time; +#ifndef CONFIG_COMPAT # define runstate_guest (v) ((v)->runstate_guest)XEN_GUEST_HANDLE (vcpu_runstate_info_t) runstate_guest;/* guest address */
Signature · 1 line
2.43.0
PATCH 2/2 sched/core: Update vcpu_runstate_get() to return nonaffine time
Update vcpu_runstate_get() to return a snapshot of the accumulated non-affine vCPU running time at the current time of this call.
We cannot change the struct vcpu_runstate_info: It is part of the Guest shared memory area that is part of the frozen VM ABI.
Instead return the new value: This way we do not have to change all old callers to pass a NULL in place of it, and we also we don't want an internal shadow struct that we memcpy from with sizeof(). To be open open to return data in the future, return a struct with the new field.
--- xen/common/sched/core.c | 26 ++++++++++++++++++++++++-- xen/include/public/vcpu.h | 10 ++++++++++ xen/include/xen/sched.h | 4 ++-- 3 files changed, 36 insertions(+), 4 deletions(-)
xen/common/sched/core.c +24 -2
@@ -324,12 +324,25 @@ void sched_guest_idle(void (*idle) (void), unsigned int cpu)atomic_dec (&per_cpu (sched_urgent_count, cpu)); } -void vcpu_runstate_get (const struct vcpu *v, -struct vcpu_runstate_info *runstate) +/** +* vcpu_runstate_get(): Return vCPU time spent in different runstates +* +* @param v: vCPU to get runstate times (since vCPU start) +* @param runstate: Return time spent in each runstate. +* This structure is part of the runstate memory areas +* shared with the domains which is part of the ABI +* with domains that is frozen and cannot be changed. +* To return additional values, use e.g. the return +* value(no need to change all callers) of this function. +* @returns struct with non-affine running time since vcpu creation +*/ +vcpu_runstate_extra_t vcpu_runstate_get (const struct vcpu *v, +struct vcpu_runstate_info *runstate) {spinlock_t *lock;s_time_t delta;struct sched_unit *unit; +vcpu_runstate_extra_t ret;rcu_read_lock (&sched_res_rculock);
@@ -343,14 +356,23 @@ void vcpu_runstate_get(const struct vcpu *v, : v->sched_unit; lock =likely (v == current) ?NULL :unit_schedule_lock_irq (unit);memcpy (runstate, &v->runstate,sizeof (*runstate)); + ret.nonaffine_time = v->nonaffine_time;/* accumulated nonaffine time */ + delta =NOW () - runstate->state_entry_time;if ( delta >0 ) + { runstate->time[runstate->state] += delta; +if (nonaffine (v, unit) )/* When running nonaffine, add the delta */ + ret.nonaffine_time += delta; + } +if (unlikely (lock !=NULL ) )unit_schedule_unlock_irq (lock, unit);rcu_read_unlock (&sched_res_rculock); + +return ret; }uint64_t get_cpu_idle_time (unsigned int cpu)
xen/include/public/vcpu.h +10 -0
@@ -79,8 +79,18 @@ struct vcpu_runstate_info {uint64_t time[4 ]; };typedef struct vcpu_runstate_info vcpu_runstate_info_t ; +/* vcpu_runstate_info_t is in the Guest shared memory area (frozen ABI) */ DEFINE_XEN_GUEST_HANDLE (vcpu_runstate_info_t); +/* +* Extra information returned from vcpu_runstate_get that is not part +* of the Guest shared memory area (not part of the frozen Guest ABI) +*/ +struct vcpu_runstate_extra { +uint64_t nonaffine_time;/* Time running outside soft_affinity mask */ +}; +typedef struct vcpu_runstate_extra vcpu_runstate_extra_t ; +/* VCPU is currently running on a physical CPU. */ #define RUNSTATE_running 0
xen/include/xen/sched.h +2 -2
@@ -1110,8 +1110,8 @@ int vcpu_set_hard_affinity(struct vcpu *v, const cpumask_t *affinity);int vcpu_affinity_domctl (struct domain *d,uint32_t cmd,struct xen_domctl_vcpuaffinity *vcpuaff); -void vcpu_runstate_get (const struct vcpu *v, -struct vcpu_runstate_info *runstate); +vcpu_runstate_extra_t vcpu_runstate_get (const struct vcpu *v, +struct vcpu_runstate_info *runstate);uint64_t get_cpu_idle_time (unsigned int cpu);void sched_guest_idle (void (*idle) (void ),unsigned int cpu);void scheduler_enable (void );
Signature · 1 line
2.43.0
Re: [PATCH 2/2] sched/core: Update vcpu_runstate_get() to return nonaffine time
xen/include/public/vcpu.h struct vcpu_runstate_info { · line 83
Quoted 8 lines · Bernhard Kaindl
xen/include/public/vcpu.h +1
-0@@ -79,8 +79,18 @@ struct vcpu_runstate_info {uint64_t time[4 ]; };typedef struct vcpu_runstate_info vcpu_runstate_info_t ; +/* vcpu_runstate_info_t is in the Guest shared memory area (frozen ABI) */ DEFINE_XEN_GUEST_HANDLE (vcpu_runstate_info_t);
Personally I'm opposed to this kind of comment.
xen/include/public/vcpu.h struct vcpu_runstate_info { · line 92
Quoted 8 lines
xen/include/public/vcpu.h +8
-0+/* +* Extra information returned from vcpu_runstate_get that is not part +* of the Guest shared memory area (not part of the frozen Guest ABI) +*/ +struct vcpu_runstate_extra { +uint64_t nonaffine_time;/* Time running outside soft_affinity mask */ +}; +typedef struct vcpu_runstate_extra vcpu_runstate_extra_t ;
And the next time we need some further piece of statistics, we'll have to add yet another new sub-hypercall? Or wait, there is no new sub- hypercall here. How's this piece of data going to make it out to guest space then?
xen/include/xen/sched.h int vcpu_set_hard_affinity(struct vcpu *v, const cpumask_t *affinity); · line 1114
Quoted 10 lines
xen/include/xen/sched.h +2
-2@@ -1110,8 +1110,8 @@ int vcpu_set_hard_affinity(struct vcpu *v, const cpumask_t *affinity);int vcpu_affinity_domctl (struct domain *d,uint32_t cmd,struct xen_domctl_vcpuaffinity *vcpuaff); -void vcpu_runstate_get (const struct vcpu *v, -struct vcpu_runstate_info *runstate); +vcpu_runstate_extra_t vcpu_runstate_get (const struct vcpu *v, +struct vcpu_runstate_info *runstate);
As long as it's only a single 64-bit field in the struct, returning this by value may be okay(ish). Yet I'd still recommend against doing so.
Also, having reached the end of the patch: Where's the caller making use of this new return value?
Jan
Re: [PATCH 2/2] sched/core: Update vcpu_runstate_get() to return nonaffine time
In reply to Bernhard KaindlQuoted 10 lines · Bernhard Kaindl
Update
vcpu_runstate_get()to return a snapshot of the accumulated non-affine vCPU running time at the current time of this call.We cannot change the struct
vcpu_runstate_info: It is part of the Guest shared memory area that is part of the frozen VM ABI.Instead return the new value: This way we do not have to change all old callers to pass a NULL in place of it, and we also we don't want an internal shadow struct that we memcpy from with
sizeof(). To be open open to return data in the future, return a struct with the new field.
Double "open".
It is not clear to me how the non-affine vCPU running time returned by vcpu_runstate_get() will be used afterwards, as you don't add any users looking at the return value. I can hardly review the patch without this information. Right now I'd NACK this series, as it seems to add dead code only.
IOW: please include a/some patch(es) making use of that code.
Quoted 22 lines
Signed-off-by Bernhard Kaindl <[email protected]>--- xen/common/sched/core.c | 26 ++++++++++++++++++++++++-- xen/include/public/vcpu.h | 10 ++++++++++ xen/include/xen/sched.h | 4 ++-- 3 files changed, 36 insertions(+), 4 deletions(-)xen/common/sched/core.c +4
-2@@ -324,12 +324,25 @@ void sched_guest_idle(void (*idle) (void), unsigned int cpu)atomic_dec (&per_cpu (sched_urgent_count, cpu)); } -void vcpu_runstate_get (const struct vcpu *v, -struct vcpu_runstate_info *runstate) +/** +* vcpu_runstate_get(): Return vCPU time spent in different runstates +* +* @param v: vCPU to get runstate times (since vCPU start)
xen/common/sched/core.c void sched_guest_idle(void (*idle) (void), unsigned int cpu) · line 340
Quoted 10 lines
xen/common/sched/core.c +10
-0+* @param runstate: Return time spent in each runstate. +* This structure is part of the runstate memory areas +* shared with the domains which is part of the ABI +* with domains that is frozen and cannot be changed. +* To return additional values, use e.g. the return +* value(no need to change all callers) of this function. +* @returns struct with non-affine running time since vcpu creation +*/ +vcpu_runstate_extra_t vcpu_runstate_get (const struct vcpu *v, +struct vcpu_runstate_info *runstate)
Returning a struct by value is limiting the possibility to expand this struct later.
I don't think it is a good idea to do it this way.
Quoted 42 lines
xen/common/sched/core.c +10
-0{spinlock_t *lock;s_time_t delta;struct sched_unit *unit; +vcpu_runstate_extra_t ret;rcu_read_lock (&sched_res_rculock);@@ -343,14 +356,23 @@ void vcpu_runstate_get(const struct vcpu *v, : v->sched_unit; lock =likely (v == current) ?NULL :unit_schedule_lock_irq (unit);memcpy (runstate, &v->runstate,sizeof (*runstate)); + ret.nonaffine_time = v->nonaffine_time;/* accumulated nonaffine time */ + delta =NOW () - runstate->state_entry_time;if ( delta >0 ) + { runstate->time[runstate->state] += delta; +if (nonaffine (v, unit) )/* When running nonaffine, add the delta */ + ret.nonaffine_time += delta; + } +if (unlikely (lock !=NULL ) )unit_schedule_unlock_irq (lock, unit);rcu_read_unlock (&sched_res_rculock); + +return ret; }uint64_t get_cpu_idle_time (unsigned int cpu)xen/include/public/vcpu.h +1
-0@@ -79,8 +79,18 @@ struct vcpu_runstate_info {uint64_t time[4 ]; };typedef struct vcpu_runstate_info vcpu_runstate_info_t ; +/* vcpu_runstate_info_t is in the Guest shared memory area (frozen ABI) */ DEFINE_XEN_GUEST_HANDLE (vcpu_runstate_info_t);
xen/include/public/vcpu.h struct vcpu_runstate_info { · line 93
Quoted 10 lines
xen/include/public/vcpu.h +9
-0+/* +* Extra information returned from vcpu_runstate_get that is not part +* of the Guest shared memory area (not part of the frozen Guest ABI) +*/ +struct vcpu_runstate_extra { +uint64_t nonaffine_time;/* Time running outside soft_affinity mask */ +}; +typedef struct vcpu_runstate_extra vcpu_runstate_extra_t ; +
Is it really needed to add this to the public header? This way you are just adding another stable interface which can't be expanded easily.
Quoted 8 lines
xen/include/public/vcpu.h +0
-0/* VCPU is currently running on a physical CPU. */ #define RUNSTATE_running 0xen/include/xen/sched.h +0
-0
xen/include/xen/sched.h int vcpu_set_hard_affinity(struct vcpu *v, const cpumask_t *affinity); · line 1117
Quoted 10 lines
xen/include/xen/sched.h +2
-2@@ -1110,8 +1110,8 @@ int vcpu_set_hard_affinity(struct vcpu *v, const cpumask_t *affinity);int vcpu_affinity_domctl (struct domain *d,uint32_t cmd,struct xen_domctl_vcpuaffinity *vcpuaff); -void vcpu_runstate_get (const struct vcpu *v, -struct vcpu_runstate_info *runstate); +vcpu_runstate_extra_t vcpu_runstate_get (const struct vcpu *v, +struct vcpu_runstate_info *runstate);uint64_t get_cpu_idle_time (unsigned int cpu);void sched_guest_idle (void (*idle) (void ),unsigned int cpu);void scheduler_enable (void );
Juergen
Quoted 6 lines · Bernhard Kaindl
To monitor the effectiveness of vCPU soft-affinity on NUMA hosts, we'd like to create a vCPU metric that accumulates the amount of vCPU time running outside of the soft affinity mask of the sched-unit:
- Add a new time counter,
nonaffine_timeto struct vcpu.
Quoted 8 lines
- Accumulate the
nonaffine_timeonvcpu_runstate_change():
Account the time spent in theRUNSTATE_runningstate outside ofunit->cpu_soft_affinity: It is always initialized and defaults tocpumask_all(bits for allNR_CPUSset), so we only accumulate nonaffine time when the vCPU runs on an unset CPU (non-affine).In the next patch, this field can be used to retrieve the accumulated nonaffine running time e.g. using
vcpu_runstate_get().
Please avoid phrases like "in the next patch" in commit messages. There is no guarantee a series will be committed in one go.
I'd just drop this last sentence.
Quoted 12 lines
Signed-off-by Bernhard Kaindl <[email protected]>--- xen/common/sched/core.c | 20 ++++++++++++++++++++ xen/include/xen/sched.h | 11 +++++++++++ 2 files changed, 31 insertions(+)xen/common/sched/core.c +0
-0
xen/common/sched/core.c static inline void vcpu_urgent_count_update(struct vcpu *v) · line 266
Quoted 7 lines
xen/common/sched/core.c +4
-0@@ -260,6 +260,23 @@ static inline void vcpu_urgent_count_update(struct vcpu *v) } } +/* +* For accounting non-affine running time of a vCPU, return true if +* the vCPU is running in RUNSTATE_running state while not on a CPU +* in unit->cpu_soft_affinity.
"the vCPU is running in RUNSTATE_running state" is a weird statement. When running it will always be in the RUNSTATE_running state. I'd write "the vCPU is in RUNSTATE_running state".
Quoted 17 lines
xen/common/sched/core.c +13
-0+*/ +static inline bool nonaffine (const struct vcpu *v, +const struct sched_unit *unit) +{ +/* +* unit->cpu_soft_affinity is always initialized and defaults to +* cpumask_all (bits for all NR_CPUS set), so we only accumulate +* nonaffine time when the vCPU runs on an unset CPU (non-affine). +*/ +return v->runstate.state == RUNSTATE_running && + !cpumask_test_cpu (v->processor, unit->cpu_soft_affinity); +} +static inline void vcpu_runstate_change (struct vcpu *v,int new_state,s_time_t new_entry_time) {
xen/common/sched/core.c static inline void vcpu_runstate_change( · line 308
Quoted 7 lines
xen/common/sched/core.c +3
-0@@ -285,6 +302,9 @@ static inline void vcpu_runstate_change( { v->runstate.time[v->runstate.state] += delta; v->runstate.state_entry_time = new_entry_time; + +if (nonaffine (v, unit) )/* When running nonaffine, add delta */ + v->nonaffine_time += delta; }
Is this really correct? Imagine a vcpu running for very long time on a physical cpu without losing it (RUNSTATE_running for minutes, hours or even days). Now someone is changing the soft-affinity of the vcpu and as a result it will be moved to another physical cpu. You will add all the long time the vcpu was running to v->nonaffine_time in spite of the affinity change having happened only nanoseconds before.
Quoted 6 lines
xen/common/sched/core.c +0
-0v->runstate.state = new_state;xen/include/xen/sched.h +0
-0
xen/include/xen/sched.h struct vcpu · line 207
Quoted 10 lines
xen/include/xen/sched.h +7
-0@@ -198,7 +198,18 @@ struct vcpustruct sched_unit *sched_unit; +/* +* The struct vcpu_runstate_info contains the vCPU time spent +* in each runstate and the entry time of the current runstate. +* +* Note: This field is used for the guest runstate shared memory area. +* Therefore, it is part of the frozen guest API and cannot be changed. +*/
s/frozen/public/
In the end I'm not really sure this comment is adding much value. But maybe I'm biased as I've worked with this code a lot.
xen/include/xen/sched.h struct vcpu · line 215
Quoted 8 lines
xen/include/xen/sched.h +4
-0struct vcpu_runstate_info runstate; + +/* vCPU time running outside the scheduling unit's soft_affinity mask */ +uint64_t nonaffine_time; +#ifndef CONFIG_COMPAT # define runstate_guest (v) ((v)->runstate_guest)XEN_GUEST_HANDLE (vcpu_runstate_info_t) runstate_guest;/* guest address */
Juergen