Skip to contentSkip to conversation

xen: add steal_clock support on x86

PATCH In review Juergen Gross 5 participants 33 messages May 18, 2016 – May 19, 2016

Subsystems (5)
lore
Share & tools
lore

Juergen Gross

author maintainer
#0 lore raw

PATCH 1/1 xen: add steal_clock support on x86

With CONFIG_PARAVIRT_TIME_ACCOUNTING the kernel is capable to account for time a thread wasn't able to run due to hypervisor scheduling. Add support in Xen arch independent time handling for this feature by moving it out of the arm arch into drivers/xen.

---
 arch/arm/xen/enlighten.c | 17 ++---------------
 arch/x86/xen/time.c      |  2 ++
 drivers/xen/time.c       | 19 +++++++++++++++++++
 include/xen/xen-ops.h    |  1 +
 4 files changed, 24 insertions(+), 15 deletions(-)
arch/arm/xen/enlighten.c +2 -15
index 75cd734..9163b94 100644
@@ -84,19 +84,6 @@ int xen_unmap_domain_gfn_range(struct vm_area_struct *vma,
 }
 EXPORT_SYMBOL_GPL(xen_unmap_domain_gfn_range);
 
-static unsigned long long xen_stolen_accounting(int cpu)
-{
-	struct vcpu_runstate_info state;
-
-	BUG_ON(cpu != smp_processor_id());
-
-	xen_get_runstate_snapshot(&state);
-
-	WARN_ON(state.state != RUNSTATE_running);
-
-	return state.time[RUNSTATE_runnable] + state.time[RUNSTATE_offline];
-}
-
 static void xen_read_wallclock(struct timespec64 *ts)
 {
 	u32 version;
@@ -355,8 +342,8 @@ static int __init xen_guest_init(void)
 
 	register_cpu_notifier(&xen_cpu_notifier);
 
-	pv_time_ops.steal_clock = xen_stolen_accounting;
-	static_key_slow_inc(&paravirt_steal_enabled);
+	xen_time_setup_guest();
+
 	if (xen_initial_domain())
 		pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
 
arch/x86/xen/time.c +2 -0
index a0a4e55..f478169 100644
@@ -431,6 +431,8 @@ static void __init xen_time_init(void)
 	xen_setup_timer(cpu);
 	xen_setup_cpu_clockevents();
 
+	xen_time_setup_guest();
+
 	if (xen_initial_domain())
 		pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
 }
drivers/xen/time.c +19 -0
index 7107842..6648a78 100644
@@ -75,6 +75,15 @@ bool xen_vcpu_stolen(int vcpu)
 	return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable;
 }
 
+static u64 xen_steal_clock(int cpu)
+{
+	struct vcpu_runstate_info state;
+
+	BUG_ON(cpu != smp_processor_id());
+	xen_get_runstate_snapshot(&state);
+	return state.time[RUNSTATE_runnable] + state.time[RUNSTATE_offline];
+}
+
 void xen_setup_runstate_info(int cpu)
 {
 	struct vcpu_register_runstate_memory_area area;
@@ -86,3 +95,13 @@ void xen_setup_runstate_info(int cpu)
 		BUG();
 }
 
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}
include/xen/xen-ops.h +1 -0
index 86abe07..5ce51c2 100644
@@ -21,6 +21,7 @@ void xen_resume_notifier_unregister(struct notifier_block *nb);
 
 bool xen_vcpu_stolen(int vcpu);
 void xen_setup_runstate_info(int cpu);
+void __init xen_time_setup_guest(void);
 void xen_get_runstate_snapshot(struct vcpu_runstate_info *res);
 
 int xen_setup_shutdown_event(void);
Signature · 1 line
2.6.6
#1 lore raw

drivers/xen/time.c void xen_setup_runstate_info(int cpu) · line 107

Quoted 12 lines · Juergen Gross

}

drivers/xen/time.c +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

-boris

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#2 lore raw
In reply to Juergen Gross

drivers/xen/time.c void xen_setup_runstate_info(int cpu) · line 107

Quoted 12 lines · Juergen Gross

}

drivers/xen/time.c +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

-boris

author maintainer
#3 lore raw
Quoted 17 lines · Boris Ostrovsky

drivers/xen/time.c void xen_setup_runstate_info(int cpu) · line 107

Quoted 12 lines · Juergen Gross

}

drivers/xen/time.c +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too? It is a Xen-specific hack, so I guess nobody will cry. Maybe it would be a good idea to select CONFIG_PARAVIRT_TIME_ACCOUNTING for XEN then?

Juergen

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

author maintainer
#4 lore raw
In reply to Boris Ostrovsky
Quoted 17 lines · Boris Ostrovsky

drivers/xen/time.c void xen_setup_runstate_info(int cpu) · line 107

Quoted 12 lines · Juergen Gross

}

drivers/xen/time.c +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too? It is a Xen-specific hack, so I guess nobody will cry. Maybe it would be a good idea to select CONFIG_PARAVIRT_TIME_ACCOUNTING for XEN then?

Juergen

#5 lore raw
Quoted 17 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

drivers/xen/time.c void xen_setup_runstate_info(int cpu) · line 107

Quoted 12 lines · Juergen Gross

}

drivers/xen/time.c +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Quoted 3 lines

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

-boris

Quoted 7 lines

It is a
Xen-specific hack, so I guess nobody will cry. Maybe it would be a good idea to select CONFIG_PARAVIRT_TIME_ACCOUNTING for XEN then?

Juergen

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#6 lore raw
In reply to Juergen Gross
Quoted 17 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

drivers/xen/time.c void xen_setup_runstate_info(int cpu) · line 107

Quoted 12 lines · Juergen Gross

}

drivers/xen/time.c +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Quoted 3 lines

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

-boris

Quoted 7 lines

It is a
Xen-specific hack, so I guess nobody will cry. Maybe it would be a good idea to select CONFIG_PARAVIRT_TIME_ACCOUNTING for XEN then?

Juergen

author maintainer
#7 lore raw
Quoted 18 lines · Boris Ostrovsky
Quoted 17 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

drivers/xen/time.c void xen_setup_runstate_info(int cpu) · line 107

Quoted 12 lines · Juergen Gross

}

drivers/xen/time.c +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Quoted 5 lines
Quoted 3 lines

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

Quoted 2 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

Juergen

#8 lore raw
Quoted 27 lines · Juergen Gross
Quoted 18 lines · Boris Ostrovsky
Quoted 17 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

drivers/xen/time.c void xen_setup_runstate_info(int cpu) · line 107

Quoted 12 lines · Juergen Gross

}

drivers/xen/time.c +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Quoted 5 lines
Quoted 3 lines

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

Quoted 7 lines
Quoted 2 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

I agree.

Although I don't think selecting PARAVIRT_TIME_ACC' is necessary -- I don't think it's essential (or is it?).

David

author maintainer
#9 lore raw
Quoted 36 lines · David Vrabel
Quoted 27 lines · Juergen Gross
Quoted 18 lines · Boris Ostrovsky
Quoted 17 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

drivers/xen/time.c void xen_setup_runstate_info(int cpu) · line 107

Quoted 12 lines · Juergen Gross

}

drivers/xen/time.c +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Quoted 5 lines
Quoted 3 lines

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

Quoted 7 lines
Quoted 2 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

Quoted 4 lines

I agree.

Although I don't think selecting PARAVIRT_TIME_ACC' is necessary -- I don't think it's essential (or is it?).

Not doing so will change behavior in case I rip out do_stolen_accounting(). What about "default y if XEN" ?

Juergen

#10 lore raw
Quoted 37 lines · Juergen Gross
Quoted 36 lines · David Vrabel
Quoted 34 lines · Juergen Gross
Quoted 18 lines · Boris Ostrovsky
Quoted 17 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

Review

Quoted 12 lines · Juergen Gross

}

(quoted diff) +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Quoted 5 lines
Quoted 3 lines

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

Quoted 2 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

Quoted 7 lines
Quoted 4 lines

I agree.

Although I don't think selecting PARAVIRT_TIME_ACC' is necessary -- I don't think it's essential (or is it?).

Not doing so will change behavior in case I rip out do_stolen_accounting(). What about "default y if XEN" ?

Ok.

David

#11 lore raw
In reply to Juergen Gross
Quoted 37 lines · Juergen Gross
Quoted 36 lines · David Vrabel
Quoted 34 lines · Juergen Gross
Quoted 18 lines · Boris Ostrovsky
Quoted 17 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

Review

Quoted 12 lines · Juergen Gross

}

(quoted diff) +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Quoted 5 lines
Quoted 3 lines

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

Quoted 2 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

Quoted 7 lines
Quoted 4 lines

I agree.

Although I don't think selecting PARAVIRT_TIME_ACC' is necessary -- I don't think it's essential (or is it?).

Not doing so will change behavior in case I rip out do_stolen_accounting(). What about "default y if XEN" ?

Ok.

David

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

author maintainer
#12 lore raw
In reply to David Vrabel
Quoted 36 lines · David Vrabel
Quoted 27 lines · Juergen Gross
Quoted 18 lines · Boris Ostrovsky
Quoted 17 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

drivers/xen/time.c void xen_setup_runstate_info(int cpu) · line 107

Quoted 12 lines · Juergen Gross

}

drivers/xen/time.c +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Quoted 5 lines
Quoted 3 lines

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

Quoted 7 lines
Quoted 2 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

Quoted 4 lines

I agree.

Although I don't think selecting PARAVIRT_TIME_ACC' is necessary -- I don't think it's essential (or is it?).

Not doing so will change behavior in case I rip out do_stolen_accounting(). What about "default y if XEN" ?

Juergen

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#13 lore raw
In reply to David Vrabel
Quoted 20 lines · David Vrabel
Quoted 19 lines · Juergen Gross
Quoted 18 lines · Boris Ostrovsky
Quoted 17 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

drivers/xen/time.c void xen_setup_runstate_info(int cpu) · line 107

Quoted 12 lines · Juergen Gross

}

drivers/xen/time.c +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Quoted 5 lines
Quoted 5 lines
Quoted 4 lines
Quoted 3 lines

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

I looked at KVM code (PARAVIRT_TIME_ACCOUNTING is not selected there neither) and in their case that's presumably because stealing accounting is a CPUID bit, i.e. it might not be supported. In Xen case we always have this interface.

Quoted 11 lines
Quoted 7 lines
Quoted 3 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

I agree.

Although I don't think selecting PARAVIRT_TIME_ACC' is necessary -- I don't think it's essential (or is it?).

Looks like it's useful only if paravirt_steal_rq_enabled, which we don't support yet.

-boris

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#14 lore raw
In reply to David Vrabel
Quoted 20 lines · David Vrabel
Quoted 19 lines · Juergen Gross
Quoted 18 lines · Boris Ostrovsky
Quoted 17 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

drivers/xen/time.c void xen_setup_runstate_info(int cpu) · line 107

Quoted 12 lines · Juergen Gross

}

drivers/xen/time.c +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Quoted 5 lines
Quoted 5 lines
Quoted 4 lines
Quoted 3 lines

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

I looked at KVM code (PARAVIRT_TIME_ACCOUNTING is not selected there neither) and in their case that's presumably because stealing accounting is a CPUID bit, i.e. it might not be supported. In Xen case we always have this interface.

Quoted 11 lines
Quoted 7 lines
Quoted 3 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

I agree.

Although I don't think selecting PARAVIRT_TIME_ACC' is necessary -- I don't think it's essential (or is it?).

Looks like it's useful only if paravirt_steal_rq_enabled, which we don't support yet.

-boris

author maintainer
#15 lore raw
Quoted 24 lines · Boris Ostrovsky
Quoted 23 lines · David Vrabel
Quoted 22 lines · Juergen Gross
Quoted 21 lines · Boris Ostrovsky
Quoted 20 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

Review

Quoted 12 lines · Juergen Gross

}

(quoted diff) +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too?

Quoted 8 lines
Quoted 2 lines
Quoted 2 lines
Quoted 1 line

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

I looked at KVM code (PARAVIRT_TIME_ACCOUNTING is not selected there neither) and in their case that's presumably because stealing accounting is a CPUID bit, i.e. it might not be supported. In Xen case we always have this interface.

So they added it later and the default is to keep the old behavior.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

Quoted 7 lines
Quoted 4 lines

I agree.

Although I don't think selecting PARAVIRT_TIME_ACC' is necessary -- I don't think it's essential (or is it?).

Looks like it's useful only if paravirt_steal_rq_enabled, which we don't support yet.

I think the patch is still useful. It is reducing code size and it is removing arch-specific Xen-hack(s). With the patch Xen's solution for arm and x86 is common and the same as for KVM. Adding paravirt_steal_rq_enabled later will be much easier as only one function needs substantial modification.

Juergen

#16 lore raw
Quoted 44 lines · Juergen Gross
Quoted 23 lines · Boris Ostrovsky
Quoted 22 lines · David Vrabel
Quoted 21 lines · Juergen Gross
Quoted 20 lines · Boris Ostrovsky
Quoted 19 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

Review

Quoted 12 lines · Juergen Gross

}

(quoted diff) +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

Quoted 8 lines
Quoted 3 lines
Quoted 3 lines
Quoted 2 lines
Quoted 1 line

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

I looked at KVM code (PARAVIRT_TIME_ACCOUNTING is not selected there neither) and in their case that's presumably because stealing accounting is a CPUID bit, i.e. it might not be supported. In Xen case we always have this interface.

So they added it later and the default is to keep the old behavior.

Quoted 10 lines
Quoted 10 lines
Quoted 6 lines
Quoted 2 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

I agree.

Although I don't think selecting PARAVIRT_TIME_ACC' is necessary -- I don't think it's essential (or is it?).

Quoted 7 lines
Quoted 2 lines

Looks like it's useful only if paravirt_steal_rq_enabled, which we don't support yet.

I think the patch is still useful. It is reducing code size and it is removing arch-specific Xen-hack(s). With the patch Xen's solution for arm and x86 is common and the same as for KVM. Adding paravirt_steal_rq_enabled later will be much easier as only one function needs substantial modification.

I am not arguing against having a patch that will remove do_stolen_accounting(). I was responding to David's statement about whether we need to select CONFIG_PARAVIRT_TIME_ACCOUNTING, and I am not sure this is necessary since steal_account_process_tick() (that will take case of things that do_stolen_accounting() currently does) doesn't need it.

(And if it is indeed needed --- can we have Xen's Kconfig select it instead of "default y if XEN" ?)

-boris

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#17 lore raw
In reply to Juergen Gross
Quoted 44 lines · Juergen Gross
Quoted 23 lines · Boris Ostrovsky
Quoted 22 lines · David Vrabel
Quoted 21 lines · Juergen Gross
Quoted 20 lines · Boris Ostrovsky
Quoted 19 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

Review

Quoted 12 lines · Juergen Gross

}

(quoted diff) +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

Quoted 8 lines
Quoted 3 lines
Quoted 3 lines
Quoted 2 lines
Quoted 1 line

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

I looked at KVM code (PARAVIRT_TIME_ACCOUNTING is not selected there neither) and in their case that's presumably because stealing accounting is a CPUID bit, i.e. it might not be supported. In Xen case we always have this interface.

So they added it later and the default is to keep the old behavior.

Quoted 10 lines
Quoted 10 lines
Quoted 6 lines
Quoted 2 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

I agree.

Although I don't think selecting PARAVIRT_TIME_ACC' is necessary -- I don't think it's essential (or is it?).

Quoted 7 lines
Quoted 2 lines

Looks like it's useful only if paravirt_steal_rq_enabled, which we don't support yet.

I think the patch is still useful. It is reducing code size and it is removing arch-specific Xen-hack(s). With the patch Xen's solution for arm and x86 is common and the same as for KVM. Adding paravirt_steal_rq_enabled later will be much easier as only one function needs substantial modification.

I am not arguing against having a patch that will remove do_stolen_accounting(). I was responding to David's statement about whether we need to select CONFIG_PARAVIRT_TIME_ACCOUNTING, and I am not sure this is necessary since steal_account_process_tick() (that will take case of things that do_stolen_accounting() currently does) doesn't need it.

(And if it is indeed needed --- can we have Xen's Kconfig select it instead of "default y if XEN" ?)

-boris

author maintainer
#18 lore raw
Quoted 53 lines · Boris Ostrovsky
Quoted 44 lines · Juergen Gross
Quoted 23 lines · Boris Ostrovsky
Quoted 22 lines · David Vrabel
Quoted 21 lines · Juergen Gross
Quoted 20 lines · Boris Ostrovsky
Quoted 19 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

Review

Quoted 12 lines · Juergen Gross

}

(quoted diff) +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

Quoted 8 lines
Quoted 3 lines
Quoted 3 lines
Quoted 2 lines
Quoted 1 line

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

I looked at KVM code (PARAVIRT_TIME_ACCOUNTING is not selected there neither) and in their case that's presumably because stealing accounting is a CPUID bit, i.e. it might not be supported. In Xen case we always have this interface.

So they added it later and the default is to keep the old behavior.

Quoted 10 lines
Quoted 10 lines
Quoted 6 lines
Quoted 2 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

I agree.

Although I don't think selecting PARAVIRT_TIME_ACC' is necessary -- I don't think it's essential (or is it?).

Quoted 7 lines
Quoted 2 lines

Looks like it's useful only if paravirt_steal_rq_enabled, which we don't support yet.

I think the patch is still useful. It is reducing code size and it is removing arch-specific Xen-hack(s). With the patch Xen's solution for arm and x86 is common and the same as for KVM. Adding paravirt_steal_rq_enabled later will be much easier as only one function needs substantial modification.

Quoted 6 lines

I am not arguing against having a patch that will remove do_stolen_accounting(). I was responding to David's statement about whether we need to select CONFIG_PARAVIRT_TIME_ACCOUNTING, and I am not sure this is necessary since steal_account_process_tick() (that will take case of things that do_stolen_accounting() currently does) doesn't need it.

Aah, okay. That's a good reason to not add the Kconfig stuff.

Quoted 2 lines

(And if it is indeed needed --- can we have Xen's Kconfig select it instead of "default y if XEN" ?)

I've verified that CONFIG_PARAVIRT_TIME_ACCOUNTING is _not_ needed. I've removed it from .config and used my patch with do_stolen_accounting() removed. In an overcommitted guest (4 vcpus on 2 physical cpus) running a parallel make top showed near 50% stolen time.

Juergen

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

author maintainer
#19 lore raw
In reply to Boris Ostrovsky
Quoted 53 lines · Boris Ostrovsky
Quoted 44 lines · Juergen Gross
Quoted 23 lines · Boris Ostrovsky
Quoted 22 lines · David Vrabel
Quoted 21 lines · Juergen Gross
Quoted 20 lines · Boris Ostrovsky
Quoted 19 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

Review

Quoted 12 lines · Juergen Gross

}

(quoted diff) +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

Quoted 8 lines
Quoted 3 lines
Quoted 3 lines
Quoted 2 lines
Quoted 1 line

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

I looked at KVM code (PARAVIRT_TIME_ACCOUNTING is not selected there neither) and in their case that's presumably because stealing accounting is a CPUID bit, i.e. it might not be supported. In Xen case we always have this interface.

So they added it later and the default is to keep the old behavior.

Quoted 10 lines
Quoted 10 lines
Quoted 6 lines
Quoted 2 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

I agree.

Although I don't think selecting PARAVIRT_TIME_ACC' is necessary -- I don't think it's essential (or is it?).

Quoted 7 lines
Quoted 2 lines

Looks like it's useful only if paravirt_steal_rq_enabled, which we don't support yet.

I think the patch is still useful. It is reducing code size and it is removing arch-specific Xen-hack(s). With the patch Xen's solution for arm and x86 is common and the same as for KVM. Adding paravirt_steal_rq_enabled later will be much easier as only one function needs substantial modification.

Quoted 6 lines

I am not arguing against having a patch that will remove do_stolen_accounting(). I was responding to David's statement about whether we need to select CONFIG_PARAVIRT_TIME_ACCOUNTING, and I am not sure this is necessary since steal_account_process_tick() (that will take case of things that do_stolen_accounting() currently does) doesn't need it.

Aah, okay. That's a good reason to not add the Kconfig stuff.

Quoted 2 lines

(And if it is indeed needed --- can we have Xen's Kconfig select it instead of "default y if XEN" ?)

I've verified that CONFIG_PARAVIRT_TIME_ACCOUNTING is _not_ needed. I've removed it from .config and used my patch with do_stolen_accounting() removed. In an overcommitted guest (4 vcpus on 2 physical cpus) running a parallel make top showed near 50% stolen time.

Juergen

author maintainer
#20 lore raw
In reply to Boris Ostrovsky
Quoted 24 lines · Boris Ostrovsky
Quoted 23 lines · David Vrabel
Quoted 22 lines · Juergen Gross
Quoted 21 lines · Boris Ostrovsky
Quoted 20 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

Review

Quoted 12 lines · Juergen Gross

}

(quoted diff) +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too?

Quoted 8 lines
Quoted 2 lines
Quoted 2 lines
Quoted 1 line

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

I looked at KVM code (PARAVIRT_TIME_ACCOUNTING is not selected there neither) and in their case that's presumably because stealing accounting is a CPUID bit, i.e. it might not be supported. In Xen case we always have this interface.

So they added it later and the default is to keep the old behavior.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

Quoted 7 lines
Quoted 4 lines

I agree.

Although I don't think selecting PARAVIRT_TIME_ACC' is necessary -- I don't think it's essential (or is it?).

Looks like it's useful only if paravirt_steal_rq_enabled, which we don't support yet.

I think the patch is still useful. It is reducing code size and it is removing arch-specific Xen-hack(s). With the patch Xen's solution for arm and x86 is common and the same as for KVM. Adding paravirt_steal_rq_enabled later will be much easier as only one function needs substantial modification.

Juergen

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#21 lore raw
In reply to Juergen Gross
Quoted 27 lines · Juergen Gross
Quoted 18 lines · Boris Ostrovsky
Quoted 17 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

drivers/xen/time.c void xen_setup_runstate_info(int cpu) · line 107

Quoted 12 lines · Juergen Gross

}

drivers/xen/time.c +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Quoted 5 lines
Quoted 3 lines

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

Quoted 7 lines
Quoted 2 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

I agree.

Although I don't think selecting PARAVIRT_TIME_ACC' is necessary -- I don't think it's essential (or is it?).

David

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

author maintainer
#22 lore raw
In reply to Boris Ostrovsky
Quoted 18 lines · Boris Ostrovsky
Quoted 17 lines · Juergen Gross
Quoted 16 lines · Boris Ostrovsky

drivers/xen/time.c void xen_setup_runstate_info(int cpu) · line 107

Quoted 12 lines · Juergen Gross

}

drivers/xen/time.c +10 -0
+void __init xen_time_setup_guest(void)
+{
+	pv_time_ops.steal_clock = xen_steal_clock;
+
+	static_key_slow_inc(&paravirt_steal_enabled);
+	/*
+	 * We can't set paravirt_steal_rq_enabled as this would require the
+	 * capability to read another cpu's runstate info.
+	 */
+}

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Quoted 5 lines
Quoted 3 lines

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too?

I don't think PARAVIRT_TIME_ACCOUNTING is always selected for Xen. If

This is easy to accomplish. :-)

Quoted 2 lines

that's indeed the case then we should ifndef do_stolen_accounting(). Or maybe check for paravirt_steal_enabled.

Is this really a sensible thing to do? There is a mechanism used by KVM to do the stolen accounting. I think we should use it instead of having a second implementation doing the same thing in case the generic one isn't enabled.

Juergen

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#23 lore raw
In reply to Juergen Gross
Quoted 11 lines · Juergen Gross
Quoted 4 lines · Boris Ostrovsky


Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too? It is a Xen-specific hack, so I guess nobody will cry. Maybe it would be a good idea to select CONFIG_PARAVIRT_TIME_ACCOUNTING for XEN then?

So, config options aside, if I understand this correctly, it looks like we were actually already doing steal time accounting, although in a non-standard way.

And yet, people seem to have issues relating to lack of (proper?) steal time accounting (Cc-ing Tony).

I guess this means that, either:
 - the issue being reported is actually not caused by the lack of steal time accounting,
 - our current (Xen specific) steal time accounting solution is flawed,
 - the issue is caused by the lack of the bit of steal time accounting that we do not support yet,
 - other ideas? Tony?

Dario

Signature · 4 lines
<<This happens because I choose it to happen!>> (Raistlin Majere)
-----------------------------------------------------------------
Dario Faggioli, Ph.D, http://about.me/dario.faggioli
Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#24 lore raw
In reply to Juergen Gross
Quoted 11 lines · Juergen Gross
Quoted 4 lines · Boris Ostrovsky


Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too? It is a Xen-specific hack, so I guess nobody will cry. Maybe it would be a good idea to select CONFIG_PARAVIRT_TIME_ACCOUNTING for XEN then?

So, config options aside, if I understand this correctly, it looks like we were actually already doing steal time accounting, although in a non-standard way.

And yet, people seem to have issues relating to lack of (proper?) steal time accounting (Cc-ing Tony).

I guess this means that, either:
 - the issue being reported is actually not caused by the lack of steal time accounting,
 - our current (Xen specific) steal time accounting solution is flawed,
 - the issue is caused by the lack of the bit of steal time accounting that we do not support yet,
 - other ideas? Tony?

Dario

Signature · 4 lines
<<This happens because I choose it to happen!>> (Raistlin Majere)
-----------------------------------------------------------------
Dario Faggioli, Ph.D, http://about.me/dario.faggioli
Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)
#25 lore raw
Quoted 19 lines · Dario Faggioli
Quoted 11 lines · Juergen Gross
Quoted 4 lines · Boris Ostrovsky

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too? It is a Xen-specific hack, so I guess nobody will cry. Maybe it would be a good idea to select CONFIG_PARAVIRT_TIME_ACCOUNTING for XEN then?

So, config options aside, if I understand this correctly, it looks like we were actually already doing steal time accounting, although in a non-standard way.

And yet, people seem to have issues relating to lack of (proper?) steal time accounting (Cc-ing Tony).

Quoted 6 lines

I guess this means that, either:
- the issue being reported is actually not caused by the lack of steal time accounting,
- our current (Xen specific) steal time accounting solution is flawed,
- the issue is caused by the lack of the bit of steal time accounting that we do not support yet,

I believe it's this one.

Tony narrowed the problem down to update_curr() where vruntime is calculated, based on runqueue's clock_task value. That value is computed in update_rq_clock_task(), which needs paravirt_steal_rq_enabled.

-boris

Quoted 3 lines

- other ideas? Tony?

Dario

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#26 lore raw
In reply to Dario Faggioli
Quoted 19 lines · Dario Faggioli
Quoted 11 lines · Juergen Gross
Quoted 4 lines · Boris Ostrovsky

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too? It is a Xen-specific hack, so I guess nobody will cry. Maybe it would be a good idea to select CONFIG_PARAVIRT_TIME_ACCOUNTING for XEN then?

So, config options aside, if I understand this correctly, it looks like we were actually already doing steal time accounting, although in a non-standard way.

And yet, people seem to have issues relating to lack of (proper?) steal time accounting (Cc-ing Tony).

Quoted 6 lines

I guess this means that, either:
- the issue being reported is actually not caused by the lack of steal time accounting,
- our current (Xen specific) steal time accounting solution is flawed,
- the issue is caused by the lack of the bit of steal time accounting that we do not support yet,

I believe it's this one.

Tony narrowed the problem down to update_curr() where vruntime is calculated, based on runqueue's clock_task value. That value is computed in update_rq_clock_task(), which needs paravirt_steal_rq_enabled.

-boris

Quoted 3 lines

- other ideas? Tony?

Dario

#27 lore raw
Quoted 27 lines · Boris Ostrovsky
Quoted 19 lines · Dario Faggioli
Quoted 11 lines · Juergen Gross
Quoted 4 lines · Boris Ostrovsky

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too? It is a Xen-specific hack, so I guess nobody will cry. Maybe it would be a good idea to select CONFIG_PARAVIRT_TIME_ACCOUNTING for XEN then?

So, config options aside, if I understand this correctly, it looks like we were actually already doing steal time accounting, although in a non-standard way.

And yet, people seem to have issues relating to lack of (proper?) steal time accounting (Cc-ing Tony).

Quoted 6 lines

I guess this means that, either:
- the issue being reported is actually not caused by the lack of steal time accounting,
- our current (Xen specific) steal time accounting solution is flawed,
- the issue is caused by the lack of the bit of steal time accounting that we do not support yet,

Quoted 6 lines

I believe it's this one.

Tony narrowed the problem down to update_curr() where vruntime is calculated, based on runqueue's clock_task value. That value is computed in update_rq_clock_task(), which needs paravirt_steal_rq_enabled.

Hi Boris,

You are right.

The real problem is steal_clock in pv_time_ops is implemented in KVM but not in Xen.

arch/x86/include/asm/paravirt_types.h

struct pv_time_ops {
        unsigned long long (*sched_clock)(void);
        unsigned long long (*steal_clock)(int cpu);
        unsigned long (*get_tsc_khz)(void);
};

(1) KVM implemented both sched_clock and steal_clock.

arch/x86/kernel/kvmclock.c
pv_time_ops.sched_clock = kvm_clock_read;

arch/x86/kernel/kvm.c
pv_time_ops.steal_clock = kvm_steal_clock;

(2) However, Xen just implemented sched_clock while the steal_clock is still native_steal_clock(). The function native_steal_clock() just simply return 0.

arch/x86/xen/time.c
.sched_clock = xen_clocksource_read;

arch/x86/kernel/paravirt.c
static u64 native_steal_clock(int cpu)

{
         return 0;
}

Therefore, even though update_rq_clock_task() calculates the value and paravirt_steal_rq_enabled option is enabled, the steal value just returns 0. This will cause the problem which I mentioned.

update_rq_clock_task
--> paravirt_steal_clock
--> pv_time_ops.steal_clock
--> native_steal_clock (if in Xen)
--> 0

The fundamental solution is to implement a steal_clock in Xen(learn from KVM implementation) instead of using the native one.

Tony

Quoted 7 lines

-boris

Quoted 3 lines

- other ideas? Tony?

Dario

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#28 lore raw
In reply to Boris Ostrovsky
Quoted 27 lines · Boris Ostrovsky
Quoted 19 lines · Dario Faggioli
Quoted 11 lines · Juergen Gross
Quoted 4 lines · Boris Ostrovsky

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too? It is a Xen-specific hack, so I guess nobody will cry. Maybe it would be a good idea to select CONFIG_PARAVIRT_TIME_ACCOUNTING for XEN then?

So, config options aside, if I understand this correctly, it looks like we were actually already doing steal time accounting, although in a non-standard way.

And yet, people seem to have issues relating to lack of (proper?) steal time accounting (Cc-ing Tony).

Quoted 6 lines

I guess this means that, either:
- the issue being reported is actually not caused by the lack of steal time accounting,
- our current (Xen specific) steal time accounting solution is flawed,
- the issue is caused by the lack of the bit of steal time accounting that we do not support yet,

Quoted 6 lines

I believe it's this one.

Tony narrowed the problem down to update_curr() where vruntime is calculated, based on runqueue's clock_task value. That value is computed in update_rq_clock_task(), which needs paravirt_steal_rq_enabled.

Hi Boris,

You are right.

The real problem is steal_clock in pv_time_ops is implemented in KVM but not in Xen.

arch/x86/include/asm/paravirt_types.h

struct pv_time_ops {
        unsigned long long (*sched_clock)(void);
        unsigned long long (*steal_clock)(int cpu);
        unsigned long (*get_tsc_khz)(void);
};

(1) KVM implemented both sched_clock and steal_clock.

arch/x86/kernel/kvmclock.c
pv_time_ops.sched_clock = kvm_clock_read;

arch/x86/kernel/kvm.c
pv_time_ops.steal_clock = kvm_steal_clock;

(2) However, Xen just implemented sched_clock while the steal_clock is still native_steal_clock(). The function native_steal_clock() just simply return 0.

arch/x86/xen/time.c
.sched_clock = xen_clocksource_read;

arch/x86/kernel/paravirt.c
static u64 native_steal_clock(int cpu)

{
         return 0;
}

Therefore, even though update_rq_clock_task() calculates the value and paravirt_steal_rq_enabled option is enabled, the steal value just returns 0. This will cause the problem which I mentioned.

update_rq_clock_task
--> paravirt_steal_clock
--> pv_time_ops.steal_clock
--> native_steal_clock (if in Xen)
--> 0

The fundamental solution is to implement a steal_clock in Xen(learn from KVM implementation) instead of using the native one.

Tony

Quoted 7 lines

-boris

Quoted 3 lines

- other ideas? Tony?

Dario

#29 lore raw
Quoted 84 lines · Tony S
Quoted 27 lines · Boris Ostrovsky
Quoted 19 lines · Dario Faggioli
Quoted 11 lines · Juergen Gross
Quoted 4 lines · Boris Ostrovsky

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too? It is a Xen-specific hack, so I guess nobody will cry. Maybe it would be a good idea to select CONFIG_PARAVIRT_TIME_ACCOUNTING for XEN then?

So, config options aside, if I understand this correctly, it looks like we were actually already doing steal time accounting, although in a non-standard way.

And yet, people seem to have issues relating to lack of (proper?) steal time accounting (Cc-ing Tony).

Quoted 6 lines

I guess this means that, either:
- the issue being reported is actually not caused by the lack of steal time accounting,
- our current (Xen specific) steal time accounting solution is flawed,
- the issue is caused by the lack of the bit of steal time accounting that we do not support yet,

Quoted 6 lines

I believe it's this one.

Tony narrowed the problem down to update_curr() where vruntime is calculated, based on runqueue's clock_task value. That value is computed in update_rq_clock_task(), which needs paravirt_steal_rq_enabled.

Hi Boris,

You are right.

The real problem is steal_clock in pv_time_ops is implemented in KVM but not in Xen.

arch/x86/include/asm/paravirt_types.h

struct pv_time_ops {
        unsigned long long (*sched_clock)(void);
        unsigned long long (*steal_clock)(int cpu);
        unsigned long (*get_tsc_khz)(void);
};

(1) KVM implemented both sched_clock and steal_clock.

arch/x86/kernel/kvmclock.c
pv_time_ops.sched_clock = kvm_clock_read;

arch/x86/kernel/kvm.c
pv_time_ops.steal_clock = kvm_steal_clock;

(2) However, Xen just implemented sched_clock while the steal_clock is still native_steal_clock(). The function native_steal_clock() just simply return 0.

arch/x86/xen/time.c
.sched_clock = xen_clocksource_read;

arch/x86/kernel/paravirt.c
static u64 native_steal_clock(int cpu)

{
         return 0;
}

Therefore, even though update_rq_clock_task() calculates the value and paravirt_steal_rq_enabled option is enabled, the steal value just returns 0. This will cause the problem which I mentioned.

update_rq_clock_task
--> paravirt_steal_clock
--> pv_time_ops.steal_clock
--> native_steal_clock (if in Xen)
--> 0

Quoted 5 lines

The fundamental solution is to implement a steal_clock in Xen(learn from KVM implementation) instead of using the native one.

Tony

Also, I tried the latest long term version of Linux 4.4, this issue still exists there. Hoping the next version can add this patch.

Tony

Quoted 7 lines
Quoted 7 lines

-boris

Quoted 3 lines

- other ideas? Tony?

Dario

#30 lore raw
Quoted 24 lines · Tony S
Quoted 22 lines · Tony S
Quoted 5 lines · Boris Ostrovsky

Tony narrowed the problem down to update_curr() where vruntime is calculated, based on runqueue's clock_task value. That value is computed
in update_rq_clock_task(), which needs paravirt_steal_rq_enabled.

Hi Boris,

You are right.

The real problem is steal_clock in pv_time_ops is implemented in KVM
but not in Xen.

[...]

Therefore, even though update_rq_clock_task() calculates the value and
paravirt_steal_rq_enabled option is enabled, the steal value just returns 0. This will cause the problem which I mentioned.

Quoted 6 lines
Quoted 6 lines

update_rq_clock_task
--> paravirt_steal_clock
--> pv_time_ops.steal_clock
--> native_steal_clock (if in Xen)
--> 0

Ok, thanks again for confirming this.

Quoted 3 lines

Also, I tried the latest long term version of Linux 4.4, this issue still exists there. Hoping the next version can add this patch.

Yes, as Juergen said here:

 http://lists.xen.org/archives/html/xen-devel/2016-05/msg01712.html

he has in his plans to implement the full mechanism. It will just take a bit longer, due to the amount of work necessary for this second part.

Regards,
Dario

Signature · 4 lines
<<This happens because I choose it to happen!>> (Raistlin Majere)
-----------------------------------------------------------------
Dario Faggioli, Ph.D, http://about.me/dario.faggioli
Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)
#31 lore raw
In reply to Tony S
Quoted 24 lines · Tony S
Quoted 22 lines · Tony S
Quoted 5 lines · Boris Ostrovsky

Tony narrowed the problem down to update_curr() where vruntime is calculated, based on runqueue's clock_task value. That value is computed
in update_rq_clock_task(), which needs paravirt_steal_rq_enabled.

Hi Boris,

You are right.

The real problem is steal_clock in pv_time_ops is implemented in KVM
but not in Xen.

[...]

Therefore, even though update_rq_clock_task() calculates the value and
paravirt_steal_rq_enabled option is enabled, the steal value just returns 0. This will cause the problem which I mentioned.

Quoted 6 lines
Quoted 6 lines

update_rq_clock_task
--> paravirt_steal_clock
--> pv_time_ops.steal_clock
--> native_steal_clock (if in Xen)
--> 0

Ok, thanks again for confirming this.

Quoted 3 lines

Also, I tried the latest long term version of Linux 4.4, this issue still exists there. Hoping the next version can add this patch.

Yes, as Juergen said here:

 http://lists.xen.org/archives/html/xen-devel/2016-05/msg01712.html

he has in his plans to implement the full mechanism. It will just take a bit longer, due to the amount of work necessary for this second part.

Regards,
Dario

Signature · 4 lines
<<This happens because I choose it to happen!>> (Raistlin Majere)
-----------------------------------------------------------------
Dario Faggioli, Ph.D, http://about.me/dario.faggioli
Senior Software Engineer, Citrix Systems R&D Ltd., Cambridge (UK)

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel

#32 lore raw
In reply to Tony S
Quoted 84 lines · Tony S
Quoted 27 lines · Boris Ostrovsky
Quoted 19 lines · Dario Faggioli
Quoted 11 lines · Juergen Gross
Quoted 4 lines · Boris Ostrovsky

Won't we be accounting for stolen cycles twice now --- once from steal_account_process_tick()->steal_clock() and second time from do_stolen_accounting()?

Uuh, yes.

I guess I should rip do_stolen_accounting() out, too? It is a Xen-specific hack, so I guess nobody will cry. Maybe it would be a good idea to select CONFIG_PARAVIRT_TIME_ACCOUNTING for XEN then?

So, config options aside, if I understand this correctly, it looks like we were actually already doing steal time accounting, although in a non-standard way.

And yet, people seem to have issues relating to lack of (proper?) steal time accounting (Cc-ing Tony).

Quoted 6 lines

I guess this means that, either:
- the issue being reported is actually not caused by the lack of steal time accounting,
- our current (Xen specific) steal time accounting solution is flawed,
- the issue is caused by the lack of the bit of steal time accounting that we do not support yet,

Quoted 6 lines

I believe it's this one.

Tony narrowed the problem down to update_curr() where vruntime is calculated, based on runqueue's clock_task value. That value is computed in update_rq_clock_task(), which needs paravirt_steal_rq_enabled.

Hi Boris,

You are right.

The real problem is steal_clock in pv_time_ops is implemented in KVM but not in Xen.

arch/x86/include/asm/paravirt_types.h

struct pv_time_ops {
        unsigned long long (*sched_clock)(void);
        unsigned long long (*steal_clock)(int cpu);
        unsigned long (*get_tsc_khz)(void);
};

(1) KVM implemented both sched_clock and steal_clock.

arch/x86/kernel/kvmclock.c
pv_time_ops.sched_clock = kvm_clock_read;

arch/x86/kernel/kvm.c
pv_time_ops.steal_clock = kvm_steal_clock;

(2) However, Xen just implemented sched_clock while the steal_clock is still native_steal_clock(). The function native_steal_clock() just simply return 0.

arch/x86/xen/time.c
.sched_clock = xen_clocksource_read;

arch/x86/kernel/paravirt.c
static u64 native_steal_clock(int cpu)

{
         return 0;
}

Therefore, even though update_rq_clock_task() calculates the value and paravirt_steal_rq_enabled option is enabled, the steal value just returns 0. This will cause the problem which I mentioned.

update_rq_clock_task
--> paravirt_steal_clock
--> pv_time_ops.steal_clock
--> native_steal_clock (if in Xen)
--> 0

Quoted 5 lines

The fundamental solution is to implement a steal_clock in Xen(learn from KVM implementation) instead of using the native one.

Tony

Also, I tried the latest long term version of Linux 4.4, this issue still exists there. Hoping the next version can add this patch.

Tony

Quoted 7 lines
Quoted 7 lines

-boris

Quoted 3 lines

- other ideas? Tony?

Dario

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xen.org/xen-devel