Skip to contentSkip to conversation

KVM virtual PTP driver

PATCH In review Marcelo Tosatti 6 participants 54 messages Jan 13, 2017 – Jan 20, 2017

Subsystems (7)
Network drivers Documentation x86 KVM +3 more subsystems
lore
Share & tools
lore

Marcelo Tosatti

author
#0 lore raw

cover KVM virtual PTP driver

This patchset implements a virtual PTP driver which allows guest to sync its clock to the host clock with high precision ( error is < 10ns).

author
#1 lore raw

PATCH 1/3 KVM: x86: provide realtime host clock via vsyscall notifiers

Expose the realtime host clock and save the TSC value used for the clock calculation.

Signed-off-by Marcelo Tosatti <[email protected]>
---
 arch/x86/kvm/x86.c |   38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
arch/x86/kvm/x86.c +38 -0
Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================
@@ -1139,6 +1139,8 @@
 
 	u64		boot_ns;
 	u64		nsec_base;
+	u64		wall_time_sec;
+	u64		wall_time_snsec;
 };
 
 static struct pvclock_gtod_data pvclock_gtod_data;

@@ -1162,6 +1164,9 @@
 	vdata->boot_ns			= boot_ns;
 	vdata->nsec_base		= tk->tkr_mono.xtime_nsec;
 
+	vdata->wall_time_sec            = tk->xtime_sec;
+	vdata->wall_time_snsec          = tk->tkr_mono.xtime_nsec;
+
 	write_seqcount_end(&vdata->seq);
 }
 #endif

@@ -1623,6 +1628,28 @@
 	return mode;
 }
 
+static int do_realtime(struct timespec *ts, cycle_t *cycle_now)
+{
+	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
+	unsigned long seq;
+	int mode;
+	u64 ns;
+
+	do {
+		seq = read_seqcount_begin(&gtod->seq);
+		mode = gtod->clock.vclock_mode;
+		ts->tv_sec = gtod->wall_time_sec;
+		ns = gtod->wall_time_snsec;
+		ns += vgettsc(cycle_now);
+		ns >>= gtod->clock.shift;
+	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
+
+	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
+	ts->tv_nsec = ns;
+
+	return mode;
+}
+
 /* returns true if host is using tsc clocksource */
 static bool kvm_get_time_and_clockread(s64 *kernel_ns, cycle_t *cycle_now)
 {

@@ -1632,6 +1659,17 @@
 
 	return do_monotonic_boot(kernel_ns, cycle_now) == VCLOCK_TSC;
 }
+
+/* returns true if host is using tsc clocksource */
+static bool kvm_get_walltime_and_clockread(struct timespec *ts,
+					   cycle_t *cycle_now)
+{
+	/* checked again under seqlock below */
+	if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
+		return false;
+
+	return do_realtime(ts, cycle_now) == VCLOCK_TSC;
+}
 #endif
 
 /*
#2 lore raw

Re: [patch 1/3] KVM: x86: provide realtime host clock via vsyscall notifiers

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 14 lines

Expose the realtime host clock and save the TSC value used for the clock calculation.

Signed-off-by Marcelo Tosatti <[email protected]>
---
 arch/x86/kvm/x86.c |   38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c line 1143

Quoted 5 lines
arch/x86/kvm/x86.c +2 -0
@@ -1139,6 +1139,8 @@
 
 	u64		boot_ns;
 	u64		nsec_base;
+	u64		wall_time_sec;
+	u64		wall_time_snsec;

The leading "s" in "snsec" looks like a copy-paste residue.

arch/x86/kvm/x86.c line 1168

Quoted 9 lines
arch/x86/kvm/x86.c +2 -0
 };
 
 static struct pvclock_gtod_data pvclock_gtod_data;

@@ -1162,6 +1164,9 @@
 	vdata->boot_ns			= boot_ns;
 	vdata->nsec_base		= tk->tkr_mono.xtime_nsec;
 
+	vdata->wall_time_sec            = tk->xtime_sec;
+	vdata->wall_time_snsec          = tk->tkr_mono.xtime_nsec;

Using tk->tkr_mono offsets for real time seems wrong -- what happens if the real time is half a second shifted from monotonic time?

If it's ok, then vdata->nsec_base == vdata->wall_time_snsec, so we don't need it.

arch/x86/kvm/x86.c line 1631

Quoted 9 lines
arch/x86/kvm/x86.c +2 -0
+
 	write_seqcount_end(&vdata->seq);
 }
 #endif

@@ -1623,6 +1628,28 @@
 	return mode;
 }
 
+static int do_realtime(struct timespec *ts, cycle_t *cycle_now)

This is too similar to do_monotonic_boot(), but I don't see a solution that is both nice and efficient. :(

(It usually means macros or copying pvclock_gtod_data.)

Quoted 44 lines
arch/x86/kvm/x86.c +32 -0
+{
+	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
+	unsigned long seq;
+	int mode;
+	u64 ns;
+
+	do {
+		seq = read_seqcount_begin(&gtod->seq);
+		mode = gtod->clock.vclock_mode;
+		ts->tv_sec = gtod->wall_time_sec;
+		ns = gtod->wall_time_snsec;
+		ns += vgettsc(cycle_now);
+		ns >>= gtod->clock.shift;
+	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
+
+	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
+	ts->tv_nsec = ns;
+
+	return mode;
+}
+
 /* returns true if host is using tsc clocksource */
 static bool kvm_get_time_and_clockread(s64 *kernel_ns, cycle_t *cycle_now)
 {

@@ -1632,6 +1659,17 @@
 
 	return do_monotonic_boot(kernel_ns, cycle_now) == VCLOCK_TSC;
 }
+
+/* returns true if host is using tsc clocksource */
+static bool kvm_get_walltime_and_clockread(struct timespec *ts,
+					   cycle_t *cycle_now)
+{
+	/* checked again under seqlock below */
+	if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
+		return false;
+
+	return do_realtime(ts, cycle_now) == VCLOCK_TSC;
+}
 #endif
 
 /*
author
#3 lore raw

Re: [patch 1/3] KVM: x86: provide realtime host clock via vsyscall notifiers

Quoted 16 lines · Radim Krcmar

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 15 lines

Expose the realtime host clock and save the TSC value used for the clock calculation.

Signed-off-by Marcelo Tosatti <[email protected]>
---
 arch/x86/kvm/x86.c |   38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c +0 -0
@@ -1139,6 +1139,8 @@
Quoted 6 lines

arch/x86/kvm/x86.c line 1143

Quoted 4 lines
arch/x86/kvm/x86.c +2 -0
 	u64		boot_ns;
 	u64		nsec_base;
+	u64		wall_time_sec;
+	u64		wall_time_snsec;

The leading "s" in "snsec" looks like a copy-paste residue.

Just copying the userspace vsyscall interface.

Quoted 7 lines
Quoted 7 lines
arch/x86/kvm/x86.c +0 -0
 };
 
 static struct pvclock_gtod_data pvclock_gtod_data;

@@ -1162,6 +1164,9 @@
 	vdata->boot_ns			= boot_ns;
 	vdata->nsec_base		= tk->tkr_mono.xtime_nsec;
Quoted 5 lines

arch/x86/kvm/x86.c line 1168

Quoted 2 lines
arch/x86/kvm/x86.c +2 -0
+	vdata->wall_time_sec            = tk->xtime_sec;
+	vdata->wall_time_snsec          = tk->tkr_mono.xtime_nsec;

Using tk->tkr_mono offsets for real time seems wrong -- what happens if the real time is half a second shifted from monotonic time?

Both the userspace vsyscall interface and getnstimeofday use it for realtime clock.

Monotonic clock adds the offset:

        vdata->monotonic_time_snsec     = tk->tkr_mono.xtime_nsec
                                        +
((u64)tk->wall_to_monotonic.tv_nsec
                                                << tk->tkr_mono.shift);
Quoted 2 lines

If it's ok, then vdata->nsec_base == vdata->wall_time_snsec, so we don't need it.

Just copying the userspace vsyscall interface.

Do you actually want to change the "s" and unify wall_time_snsec with nsec_base?

Quoted 8 lines
Quoted 8 lines
arch/x86/kvm/x86.c +1 -0
+
 	write_seqcount_end(&vdata->seq);
 }
 #endif

@@ -1623,6 +1628,28 @@
 	return mode;
 }
Quoted 6 lines

arch/x86/kvm/x86.c line 1631

Quoted 1 line
arch/x86/kvm/x86.c +1 -0
+static int do_realtime(struct timespec *ts, cycle_t *cycle_now)

This is too similar to do_monotonic_boot(), but I don't see a solution that is both nice and efficient. :(

(It usually means macros or copying pvclock_gtod_data.)

Yep.

#4 lore raw

Re: [patch 1/3] KVM: x86: provide realtime host clock via vsyscall notifiers

2017-01-13 13:34-0200, Marcelo Tosatti:

Quoted 17 lines · Radim Krcmar
Quoted 16 lines

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 15 lines

Expose the realtime host clock and save the TSC value used for the clock calculation.

Signed-off-by Marcelo Tosatti <[email protected]>
---
 arch/x86/kvm/x86.c |   38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c +0 -0
@@ -1139,6 +1139,8 @@
Quoted 8 lines
Quoted 6 lines

arch/x86/kvm/x86.c line 1143

Quoted 4 lines
arch/x86/kvm/x86.c +2 -0
 	u64		boot_ns;
 	u64		nsec_base;
+	u64		wall_time_sec;
+	u64		wall_time_snsec;

The leading "s" in "snsec" looks like a copy-paste residue.

Just copying the userspace vsyscall interface.

Oh, so the "s" means "sub-" for sub-nanosecond precision.

Quoted 16 lines
Quoted 7 lines
Quoted 7 lines
arch/x86/kvm/x86.c +0 -0
 };
 
 static struct pvclock_gtod_data pvclock_gtod_data;

@@ -1162,6 +1164,9 @@
 	vdata->boot_ns			= boot_ns;
 	vdata->nsec_base		= tk->tkr_mono.xtime_nsec;
Quoted 5 lines

arch/x86/kvm/x86.c line 1168

Quoted 2 lines
arch/x86/kvm/x86.c +2 -0
+	vdata->wall_time_sec            = tk->xtime_sec;
+	vdata->wall_time_snsec          = tk->tkr_mono.xtime_nsec;

Using tk->tkr_mono offsets for real time seems wrong -- what happens if the real time is half a second shifted from monotonic time?

Both the userspace vsyscall interface and getnstimeofday use it for realtime clock.

Quoted 6 lines

Monotonic clock adds the offset:

        vdata->monotonic_time_snsec     = tk->tkr_mono.xtime_nsec
                                        +
((u64)tk->wall_to_monotonic.tv_nsec
                                                << tk->tkr_mono.shift);

I see, thanks. Makes me wonder why our monotonic time is correct then, but that is problably thanks to boot_ns.

Quoted 7 lines
Quoted 2 lines

If it's ok, then vdata->nsec_base == vdata->wall_time_snsec, so we don't need it.

Just copying the userspace vsyscall interface.

Do you actually want to change the "s" and unify wall_time_snsec with nsec_base?

The "s" isn't important, even though I don't think we do anything that would justify it, but make use just 8 bytes for both. Renaming nsec_base is ok, but I'm not sure what tk->tkr_mono.xtime_nsec is anymore.

author
#5 lore raw

Re: [patch 1/3] KVM: x86: provide realtime host clock via vsyscall notifiers

Quoted 25 lines · Radim Krcmar

2017-01-13 13:34-0200, Marcelo Tosatti:

Quoted 24 lines · Radim Krcmar
Quoted 22 lines

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 14 lines

Expose the realtime host clock and save the TSC value used for the clock calculation.

Signed-off-by Marcelo Tosatti <[email protected]>
---
 arch/x86/kvm/x86.c |   38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c line 1143

Quoted 5 lines
arch/x86/kvm/x86.c +2 -0
@@ -1139,6 +1139,8 @@
 
 	u64		boot_ns;
 	u64		nsec_base;
+	u64		wall_time_sec;
+	u64		wall_time_snsec;

The leading "s" in "snsec" looks like a copy-paste residue.

Quoted 3 lines
Quoted 1 line

Just copying the userspace vsyscall interface.

Oh, so the "s" means "sub-" for sub-nanosecond precision.

It only counts nanoseconds, how can it be sub nanosecond precise?

Quoted 18 lines
Quoted 18 lines
Quoted 7 lines
Quoted 7 lines
arch/x86/kvm/x86.c +0 -0
 };
 
 static struct pvclock_gtod_data pvclock_gtod_data;

@@ -1162,6 +1164,9 @@
 	vdata->boot_ns			= boot_ns;
 	vdata->nsec_base		= tk->tkr_mono.xtime_nsec;
Quoted 5 lines

arch/x86/kvm/x86.c line 1168

Quoted 2 lines
arch/x86/kvm/x86.c +2 -0
+	vdata->wall_time_sec            = tk->xtime_sec;
+	vdata->wall_time_snsec          = tk->tkr_mono.xtime_nsec;

Using tk->tkr_mono offsets for real time seems wrong -- what happens if the real time is half a second shifted from monotonic time?

Both the userspace vsyscall interface and getnstimeofday use it for realtime clock.

Monotonic clock adds the offset:

Quoted 7 lines
Quoted 4 lines
        vdata->monotonic_time_snsec     = tk->tkr_mono.xtime_nsec
                                        +
((u64)tk->wall_to_monotonic.tv_nsec
                                                << tk->tkr_mono.shift);

I see, thanks. Makes me wonder why our monotonic time is correct then, but that is problably thanks to boot_ns.

The actual starting point of the system_timestamp part of kvmclock does not matter, all it matters is that it counts in nanoseconds.

Quoted 10 lines
Quoted 7 lines
Quoted 2 lines

If it's ok, then vdata->nsec_base == vdata->wall_time_snsec, so we don't need it.

Just copying the userspace vsyscall interface.

Do you actually want to change the "s" and unify wall_time_snsec with nsec_base?

The "s" isn't important, even though I don't think we do anything that would justify it, but make use just 8 bytes for both.

Unified.

Quoted 2 lines

Renaming nsec_base is ok, but I'm not sure what tk->tkr_mono.xtime_nsec is anymore.

Is the nsec part of tk->xtime_sec. See accumulate_nsecs_to_secs (which is called from the timer interrupt handler).

/**
* struct timekeeper - Structure holding internal timekeeping values.
* @tkr_mono: The readout base structure for CLOCK_MONOTONIC
* @tkr_raw: The readout base structure for
* CLOCK_MONOTONIC_RAW

 * @xtime_sec:          Current CLOCK_REALTIME time in seconds
 * @ktime_sec:          Current CLOCK_MONOTONIC time in seconds
 * @wall_to_monotonic:  CLOCK_REALTIME to CLOCK_MONOTONIC offset
 * @offs_real:          Offset clock monotonic -> clock realtime
 * @offs_boot:          Offset clock monotonic -> clock boottime
 * @offs_tai:           Offset clock monotonic -> clock tai
 * @tai_offset:         The current UTC to TAI offset in seconds
#6 lore raw

Re: [patch 1/3] KVM: x86: provide realtime host clock via vsyscall notifiers

2017-01-13 15:51-0200, Marcelo Tosatti:

Quoted 26 lines · Radim Krcmar
Quoted 25 lines

2017-01-13 13:34-0200, Marcelo Tosatti:

Quoted 24 lines · Radim Krcmar
Quoted 22 lines

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 14 lines

Expose the realtime host clock and save the TSC value used for the clock calculation.

Signed-off-by Marcelo Tosatti <[email protected]>
---
 arch/x86/kvm/x86.c |   38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c line 1143

Quoted 5 lines
arch/x86/kvm/x86.c +2 -0
@@ -1139,6 +1139,8 @@
 
 	u64		boot_ns;
 	u64		nsec_base;
+	u64		wall_time_sec;
+	u64		wall_time_snsec;

The leading "s" in "snsec" looks like a copy-paste residue.

Quoted 5 lines
Quoted 3 lines
Quoted 1 line

Just copying the userspace vsyscall interface.

Oh, so the "s" means "sub-" for sub-nanosecond precision.

It only counts nanoseconds, how can it be sub nanosecond precise?

Because it doesn't count nanoseconds. In update_vsyscall(), the *_snsec are shifted by tk->tkr_mono.shift bits to the left and that precision goes to sub-nanoseconds.

64 bit value makes sense then -- would be nice if we could pass that extra precision to the guest.

Quoted 23 lines
Quoted 23 lines
Quoted 22 lines
Quoted 7 lines
Quoted 7 lines
arch/x86/kvm/x86.c +0 -0
 };
 
 static struct pvclock_gtod_data pvclock_gtod_data;

@@ -1162,6 +1164,9 @@
 	vdata->boot_ns			= boot_ns;
 	vdata->nsec_base		= tk->tkr_mono.xtime_nsec;
Quoted 5 lines

arch/x86/kvm/x86.c line 1168

Quoted 2 lines
arch/x86/kvm/x86.c +2 -0
+	vdata->wall_time_sec            = tk->xtime_sec;
+	vdata->wall_time_snsec          = tk->tkr_mono.xtime_nsec;

Using tk->tkr_mono offsets for real time seems wrong -- what happens if the real time is half a second shifted from monotonic time?

Both the userspace vsyscall interface and getnstimeofday use it for realtime clock.

Monotonic clock adds the offset:

        vdata->monotonic_time_snsec     = tk->tkr_mono.xtime_nsec
                                        +
((u64)tk->wall_to_monotonic.tv_nsec
                                                << tk->tkr_mono.shift);
Quoted 5 lines
Quoted 2 lines

I see, thanks. Makes me wonder why our monotonic time is correct then, but that is problably thanks to boot_ns.

The actual starting point of the system_timestamp part of kvmclock does not matter, all it matters is that it counts in nanoseconds.

True.

Quoted 11 lines
Quoted 10 lines
Quoted 7 lines
Quoted 2 lines

If it's ok, then vdata->nsec_base == vdata->wall_time_snsec, so we don't need it.

Just copying the userspace vsyscall interface.

Do you actually want to change the "s" and unify wall_time_snsec with nsec_base?

The "s" isn't important, even though I don't think we do anything that would justify it, but make use just 8 bytes for both.

Quoted 7 lines

Unified.

Quoted 2 lines

Renaming nsec_base is ok, but I'm not sure what tk->tkr_mono.xtime_nsec is anymore.

Is the nsec part of tk->xtime_sec. See accumulate_nsecs_to_secs (which is called from the timer interrupt handler).

I see, thanks. They are not nanoseconds as the sub-nanosecond shift is there as well:

  u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
  while (tk->tkr_mono.xtime_nsec >= nsecps) {
  	tk->tkr_mono.xtime_nsec -= nsecps;
  	tk->xtime_sec++;
  }
#7 lore raw

Re: [patch 1/3] KVM: x86: provide realtime host clock via vsyscall notifiers

In reply to Radim Krcmar
Quoted 47 lines · Radim Krcmar

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 14 lines

Expose the realtime host clock and save the TSC value used for the clock calculation.

Signed-off-by Marcelo Tosatti <[email protected]>
---
 arch/x86/kvm/x86.c |   38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c line 1143

Quoted 5 lines
arch/x86/kvm/x86.c +2 -0
@@ -1139,6 +1139,8 @@
 
 	u64		boot_ns;
 	u64		nsec_base;
+	u64		wall_time_sec;
+	u64		wall_time_snsec;

The leading "s" in "snsec" looks like a copy-paste residue.

arch/x86/kvm/x86.c line 1168

Quoted 9 lines
arch/x86/kvm/x86.c +2 -0
 };
 
 static struct pvclock_gtod_data pvclock_gtod_data;

@@ -1162,6 +1164,9 @@
 	vdata->boot_ns			= boot_ns;
 	vdata->nsec_base		= tk->tkr_mono.xtime_nsec;
 
+	vdata->wall_time_sec            = tk->xtime_sec;
+	vdata->wall_time_snsec          = tk->tkr_mono.xtime_nsec;

Using tk->tkr_mono offsets for real time seems wrong -- what happens if the real time is half a second shifted from monotonic time?

If it's ok, then vdata->nsec_base == vdata->wall_time_snsec, so we don't need it.

Quoted 8 lines
arch/x86/kvm/x86.c +1 -0
+
 	write_seqcount_end(&vdata->seq);
 }
 #endif

@@ -1623,6 +1628,28 @@
 	return mode;
 }
Quoted 6 lines

arch/x86/kvm/x86.c line 1631

Quoted 1 line
arch/x86/kvm/x86.c +1 -0
+static int do_realtime(struct timespec *ts, cycle_t *cycle_now)

This is too similar to do_monotonic_boot(), but I don't see a solution that is both nice and efficient. :(

(It usually means macros or copying pvclock_gtod_data.)

PV Clock is hypervisor agnostic so both KVM and Xen can use it. Is this clock interface suppose to follow that?

Thanks.

Quoted 49 lines
Quoted 35 lines
arch/x86/kvm/x86.c +25 -0
+{
+	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
+	unsigned long seq;
+	int mode;
+	u64 ns;
+
+	do {
+		seq = read_seqcount_begin(&gtod->seq);
+		mode = gtod->clock.vclock_mode;
+		ts->tv_sec = gtod->wall_time_sec;
+		ns = gtod->wall_time_snsec;
+		ns += vgettsc(cycle_now);
+		ns >>= gtod->clock.shift;
+	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
+
+	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
+	ts->tv_nsec = ns;
+
+	return mode;
+}
+
 /* returns true if host is using tsc clocksource */
 static bool kvm_get_time_and_clockread(s64 *kernel_ns, cycle_t *cycle_now)
 {

@@ -1632,6 +1659,17 @@
 
 	return do_monotonic_boot(kernel_ns, cycle_now) == VCLOCK_TSC;
 }
+
+/* returns true if host is using tsc clocksource */
+static bool kvm_get_walltime_and_clockread(struct timespec *ts,
+					   cycle_t *cycle_now)

arch/x86/kvm/x86.c line 1675

Quoted 10 lines
arch/x86/kvm/x86.c +7 -0
+{
+	/* checked again under seqlock below */
+	if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
+		return false;
+
+	return do_realtime(ts, cycle_now) == VCLOCK_TSC;
+}
 #endif
 
 /*

--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

author
#8 lore raw

Re: [patch 1/3] KVM: x86: provide realtime host clock via vsyscall notifiers

Quoted 53 lines · Konrad Rzeszutek Wilk
Quoted 52 lines · Radim Krcmar

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 14 lines

Expose the realtime host clock and save the TSC value used for the clock calculation.

Signed-off-by Marcelo Tosatti <[email protected]>
---
 arch/x86/kvm/x86.c |   38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c line 1143

Quoted 5 lines
arch/x86/kvm/x86.c +2 -0
@@ -1139,6 +1139,8 @@
 
 	u64		boot_ns;
 	u64		nsec_base;
+	u64		wall_time_sec;
+	u64		wall_time_snsec;

The leading "s" in "snsec" looks like a copy-paste residue.

arch/x86/kvm/x86.c line 1168

Quoted 9 lines
arch/x86/kvm/x86.c +2 -0
 };
 
 static struct pvclock_gtod_data pvclock_gtod_data;

@@ -1162,6 +1164,9 @@
 	vdata->boot_ns			= boot_ns;
 	vdata->nsec_base		= tk->tkr_mono.xtime_nsec;
 
+	vdata->wall_time_sec            = tk->xtime_sec;
+	vdata->wall_time_snsec          = tk->tkr_mono.xtime_nsec;

Using tk->tkr_mono offsets for real time seems wrong -- what happens if the real time is half a second shifted from monotonic time?

If it's ok, then vdata->nsec_base == vdata->wall_time_snsec, so we don't need it.

arch/x86/kvm/x86.c line 1631

Quoted 9 lines
arch/x86/kvm/x86.c +2 -0
+
 	write_seqcount_end(&vdata->seq);
 }
 #endif

@@ -1623,6 +1628,28 @@
 	return mode;
 }
 
+static int do_realtime(struct timespec *ts, cycle_t *cycle_now)

This is too similar to do_monotonic_boot(), but I don't see a solution that is both nice and efficient. :(

Quoted 5 lines
Quoted 1 line

(It usually means macros or copying pvclock_gtod_data.)

PV Clock is hypervisor agnostic so both KVM and Xen can use it. Is this clock interface suppose to follow that?

If Xen implements the KVM_HC_CLOCK_OFFSET hypercall, Xen guests can use the kvm ptp driver, yes.

_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xen.org/xen-devel

#9 lore raw

Re: [patch 1/3] KVM: x86: provide realtime host clock via vsyscall notifiers

In reply to Radim Krcmar
Quoted 47 lines · Radim Krcmar

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 14 lines

Expose the realtime host clock and save the TSC value used for the clock calculation.

Signed-off-by Marcelo Tosatti <[email protected]>
---
 arch/x86/kvm/x86.c |   38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c line 1143

Quoted 5 lines
arch/x86/kvm/x86.c +2 -0
@@ -1139,6 +1139,8 @@
 
 	u64		boot_ns;
 	u64		nsec_base;
+	u64		wall_time_sec;
+	u64		wall_time_snsec;

The leading "s" in "snsec" looks like a copy-paste residue.

arch/x86/kvm/x86.c line 1168

Quoted 9 lines
arch/x86/kvm/x86.c +2 -0
 };
 
 static struct pvclock_gtod_data pvclock_gtod_data;

@@ -1162,6 +1164,9 @@
 	vdata->boot_ns			= boot_ns;
 	vdata->nsec_base		= tk->tkr_mono.xtime_nsec;
 
+	vdata->wall_time_sec            = tk->xtime_sec;
+	vdata->wall_time_snsec          = tk->tkr_mono.xtime_nsec;

Using tk->tkr_mono offsets for real time seems wrong -- what happens if the real time is half a second shifted from monotonic time?

If it's ok, then vdata->nsec_base == vdata->wall_time_snsec, so we don't need it.

Quoted 8 lines
arch/x86/kvm/x86.c +1 -0
+
 	write_seqcount_end(&vdata->seq);
 }
 #endif

@@ -1623,6 +1628,28 @@
 	return mode;
 }
Quoted 6 lines

arch/x86/kvm/x86.c line 1631

Quoted 1 line
arch/x86/kvm/x86.c +1 -0
+static int do_realtime(struct timespec *ts, cycle_t *cycle_now)

This is too similar to do_monotonic_boot(), but I don't see a solution that is both nice and efficient. :(

(It usually means macros or copying pvclock_gtod_data.)

PV Clock is hypervisor agnostic so both KVM and Xen can use it. Is this clock interface suppose to follow that?

Thanks.

Quoted 42 lines
Quoted 42 lines
arch/x86/kvm/x86.c +32 -0
+{
+	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
+	unsigned long seq;
+	int mode;
+	u64 ns;
+
+	do {
+		seq = read_seqcount_begin(&gtod->seq);
+		mode = gtod->clock.vclock_mode;
+		ts->tv_sec = gtod->wall_time_sec;
+		ns = gtod->wall_time_snsec;
+		ns += vgettsc(cycle_now);
+		ns >>= gtod->clock.shift;
+	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
+
+	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
+	ts->tv_nsec = ns;
+
+	return mode;
+}
+
 /* returns true if host is using tsc clocksource */
 static bool kvm_get_time_and_clockread(s64 *kernel_ns, cycle_t *cycle_now)
 {

@@ -1632,6 +1659,17 @@
 
 	return do_monotonic_boot(kernel_ns, cycle_now) == VCLOCK_TSC;
 }
+
+/* returns true if host is using tsc clocksource */
+static bool kvm_get_walltime_and_clockread(struct timespec *ts,
+					   cycle_t *cycle_now)
+{
+	/* checked again under seqlock below */
+	if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
+		return false;
+
+	return do_realtime(ts, cycle_now) == VCLOCK_TSC;
+}
 #endif
Quoted 7 lines

arch/x86/kvm/x86.c line 1675

Quoted 3 lines
arch/x86/kvm/x86.c +0 -0
 /*

--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xen.org/xen-devel

author
#10 lore raw

PATCH 2/3 KVM: x86: add KVM_HC_CLOCK_OFFSET hypercall

In reply to Marcelo Tosatti

Add a hypercall to retrieve the host realtime clock and the TSC value used to calculate that clock read.

Used to implement clock synchronization between
host and guest.

Signed-off-by Marcelo Tosatti <[email protected]>
---
 Documentation/virtual/kvm/hypercalls.txt |   30 ++++++++++++++++++++++
 arch/x86/include/uapi/asm/kvm_para.h     |    9 ++++++
 arch/x86/kvm/x86.c                       |   41 +++++++++++++++++++++++++++++++
 include/uapi/linux/kvm_para.h            |    3 ++
 4 files changed, 83 insertions(+)
arch/x86/include/uapi/asm/kvm_para.h +9 -0
Index: kvm-ptpdriver/arch/x86/include/uapi/asm/kvm_para.h
===================================================================
@@ -50,6 +50,15 @@
 	__u32 pad[11];
 };
 
+#define KVM_CLOCK_OFFSET_WALLCLOCK 0
+struct kvm_clock_offset {
+	__s64 sec;
+	__s64 nsec;
+	__u64 tsc;
+	__u32 flags;
+	__u32 pad[1];
+};
+
 #define KVM_STEAL_ALIGNMENT_BITS 5
 #define KVM_STEAL_VALID_BITS ((-1ULL << (KVM_STEAL_ALIGNMENT_BITS + 1)))
 #define KVM_STEAL_RESERVED_MASK (((1 << KVM_STEAL_ALIGNMENT_BITS) - 1 ) << 1)
Documentation/virtual/kvm/hypercalls.txt +30 -0
Index: kvm-ptpdriver/Documentation/virtual/kvm/hypercalls.txt
===================================================================
@@ -81,3 +81,33 @@
 same guest can wakeup the sleeping vcpu by issuing KVM_HC_KICK_CPU hypercall,
 specifying APIC ID (a1) of the vcpu to be woken up. An additional argument (a0)
 is used in the hypercall for future use.
+
+
+6. KVM_HC_CLOCK_OFFSET
+------------------------
+Architecture: x86
+Status: active
+Purpose: Hypercall used to synchronize host and guest clocks.
+Usage:
+
+a0: guest physical address where host copies
+"struct kvm_clock_offset" structure.
+
+a1: clock_type, ATM only KVM_CLOCK_OFFSET_WALLCLOCK (0)
+is supported (hosts CLOCK_REALTIME clock).
+
+		struct kvm_clock_offset {
+			__s64 sec;
+			__s64 nsec;
+			__u64 tsc;
+			__u32 flags;
+			__u32 pad;
+		};
+
+       Where:
+               * sec: seconds from clock_type clock.
+               * nsec: nanoseconds from clock_type clock.
+               * tsc: TSC value used to calculate sec/nsec pair
+                 (this hypercall only works when host uses TSC clocksource).
+               * flags: flags, unused at the moment.
+
arch/x86/kvm/x86.c +41 -0

53 lines not on this page · raw message

include/uapi/linux/kvm_para.h +3 -0
Index: kvm-ptpdriver/include/uapi/linux/kvm_para.h
===================================================================
@@ -14,6 +14,8 @@
 #define KVM_EFAULT		EFAULT
 #define KVM_E2BIG		E2BIG
 #define KVM_EPERM		EPERM
+#define KVM_EOPNOTSUPP		EOPNOTSUPP
+#define KVM_ENOMEM		ENOMEM
 
 #define KVM_HC_VAPIC_POLL_IRQ		1
 #define KVM_HC_MMU_OP			2

@@ -23,6 +25,7 @@
 #define KVM_HC_MIPS_GET_CLOCK_FREQ	6
 #define KVM_HC_MIPS_EXIT_VM		7
 #define KVM_HC_MIPS_CONSOLE_OUTPUT	8
+#define KVM_HC_CLOCK_OFFSET		9
 
 /*
  * hypercalls use architecture specific
#11 lore raw

Re: [patch 2/3] KVM: x86: add KVM_HC_CLOCK_OFFSET hypercall

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 51 lines

Add a hypercall to retrieve the host realtime clock and the TSC value used to calculate that clock read.

Used to implement clock synchronization between
host and guest.

Signed-off-by Marcelo Tosatti <[email protected]>
---
 Documentation/virtual/kvm/hypercalls.txt |   30 ++++++++++++++++++++++
 arch/x86/include/uapi/asm/kvm_para.h     |    9 ++++++
 arch/x86/kvm/x86.c                       |   41 +++++++++++++++++++++++++++++++
 include/uapi/linux/kvm_para.h            |    3 ++
 4 files changed, 83 insertions(+)

Index: kvm-ptpdriver/arch/x86/include/uapi/asm/kvm_para.h
===================================================================

arch/x86/include/uapi/asm/kvm_para.h +9 -0
@@ -50,6 +50,15 @@
 	__u32 pad[11];
 };
 
+#define KVM_CLOCK_OFFSET_WALLCLOCK 0
+struct kvm_clock_offset {
+	__s64 sec;
+	__s64 nsec;
+	__u64 tsc;
+	__u32 flags;
+	__u32 pad[1];
+};
+
 #define KVM_STEAL_ALIGNMENT_BITS 5
 #define KVM_STEAL_VALID_BITS ((-1ULL << (KVM_STEAL_ALIGNMENT_BITS + 1)))
 #define KVM_STEAL_RESERVED_MASK (((1 << KVM_STEAL_ALIGNMENT_BITS) - 1 ) << 1)

Index: kvm-ptpdriver/Documentation/virtual/kvm/hypercalls.txt
===================================================================

Documentation/virtual/kvm/hypercalls.txt +8 -0
@@ -81,3 +81,33 @@
 same guest can wakeup the sleeping vcpu by issuing KVM_HC_KICK_CPU hypercall,
 specifying APIC ID (a1) of the vcpu to be woken up. An additional argument (a0)
 is used in the hypercall for future use.
+
+
+6. KVM_HC_CLOCK_OFFSET
+------------------------
+Architecture: x86
+Status: active
+Purpose: Hypercall used to synchronize host and guest clocks.
+Usage:

Documentation/virtual/kvm/hypercalls.txt line 101

Quoted 10 lines
Documentation/virtual/kvm/hypercalls.txt +10 -0
+
+a0: guest physical address where host copies
+"struct kvm_clock_offset" structure.
+
+a1: clock_type, ATM only KVM_CLOCK_OFFSET_WALLCLOCK (0)
+is supported (hosts CLOCK_REALTIME clock).
+
+		struct kvm_clock_offset {
+			__s64 sec;
+			__s64 nsec;

Why is nsec:
1) signed -- it is a remainder after division by NSEC_PER_SEC
2) bigger than 32 bit -- NSEC_PER_SEC < 2^32
?

Documentation/virtual/kvm/hypercalls.txt line 109

Quoted 8 lines
Documentation/virtual/kvm/hypercalls.txt +8 -0
+			__u64 tsc;
+			__u32 flags;
+			__u32 pad;
+		};
+
+       Where:
+               * sec: seconds from clock_type clock.
+               * nsec: nanoseconds from clock_type clock.

The important part of an offset is the starting point -- I assume it is the the usual one, but documentation better be explicit.

Documentation/virtual/kvm/hypercalls.txt line 110

Quoted 1 line
Documentation/virtual/kvm/hypercalls.txt +1 -0
+               * tsc: TSC value used to calculate sec/nsec pair

This hypercall could in theory work even when computing the offset with a different clocksource on the host and the estimating TSC at that time.

Documentation/virtual/kvm/hypercalls.txt line 111

Quoted 1 line
Documentation/virtual/kvm/hypercalls.txt +1 -0
+                 (this hypercall only works when host uses TSC clocksource).

Describe that error as a return value.

Documentation/virtual/kvm/hypercalls.txt line 112

Quoted 1 line
Documentation/virtual/kvm/hypercalls.txt +1 -0
+               * flags: flags, unused at the moment.

unused = 0.

Quoted 14 lines
Documentation/virtual/kvm/hypercalls.txt +1 -0
+

Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c +5 -0
@@ -6121,6 +6121,44 @@
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
 
+static int kvm_pv_clock_offset(struct kvm_vcpu *vcpu, gpa_t paddr,
+			       unsigned long clock_type)
+{
+	struct kvm_clock_offset *clock_offset;
+	struct timespec ts;

arch/x86/kvm/x86.c line 6138

Quoted 10 lines
arch/x86/kvm/x86.c +10 -0
+	cycle_t cycle;
+	int ret;
+
+	if (clock_type != KVM_CLOCK_OFFSET_WALLCLOCK)
+		return -KVM_EOPNOTSUPP;
+
+	if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
+		return -KVM_EOPNOTSUPP;
+
+	clock_offset = kzalloc(sizeof(struct kvm_clock_offset), GFP_KERNEL);

Can't clock_offset be on the stack?

Quoted 59 lines
arch/x86/kvm/x86.c +26 -0
+	if (clock_offset == NULL)
+		return -KVM_ENOMEM;
+
+	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false) {
+		kfree(clock_offset);
+		return -KVM_EOPNOTSUPP;
+	}
+
+	clock_offset->sec = ts.tv_sec;
+	clock_offset->nsec = ts.tv_nsec;
+	clock_offset->tsc = kvm_read_l1_tsc(vcpu, cycle);
+	clock_offset->flags = 0;
+
+	ret = 0;
+	if (kvm_write_guest(vcpu->kvm, paddr, clock_offset,
+			      sizeof(struct kvm_clock_offset)))
+		ret = -KVM_EFAULT;
+
+	kfree(clock_offset);
+
+	return ret;
+}
+
 /*
  * kvm_pv_kick_cpu_op:  Kick a vcpu.
  *

@@ -6185,6 +6223,9 @@
 		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
 		ret = 0;
 		break;
+	case KVM_HC_CLOCK_OFFSET:
+		ret = kvm_pv_clock_offset(vcpu, a0, a1);
+		break;
 	default:
 		ret = -KVM_ENOSYS;
 		break;

Index: kvm-ptpdriver/include/uapi/linux/kvm_para.h
===================================================================

include/uapi/linux/kvm_para.h +3 -0
@@ -14,6 +14,8 @@
 #define KVM_EFAULT		EFAULT
 #define KVM_E2BIG		E2BIG
 #define KVM_EPERM		EPERM
+#define KVM_EOPNOTSUPP		EOPNOTSUPP
+#define KVM_ENOMEM		ENOMEM
 
 #define KVM_HC_VAPIC_POLL_IRQ		1
 #define KVM_HC_MMU_OP			2

@@ -23,6 +25,7 @@
 #define KVM_HC_MIPS_GET_CLOCK_FREQ	6
 #define KVM_HC_MIPS_EXIT_VM		7
 #define KVM_HC_MIPS_CONSOLE_OUTPUT	8
+#define KVM_HC_CLOCK_OFFSET		9
 
 /*
  * hypercalls use architecture specific
author
#12 lore raw

Re: [patch 2/3] KVM: x86: add KVM_HC_CLOCK_OFFSET hypercall

Quoted 57 lines · Radim Krcmar

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 56 lines

Add a hypercall to retrieve the host realtime clock and the TSC value used to calculate that clock read.

Used to implement clock synchronization between
host and guest.

Signed-off-by Marcelo Tosatti <[email protected]>
---
 Documentation/virtual/kvm/hypercalls.txt |   30 ++++++++++++++++++++++
 arch/x86/include/uapi/asm/kvm_para.h     |    9 ++++++
 arch/x86/kvm/x86.c                       |   41 +++++++++++++++++++++++++++++++
 include/uapi/linux/kvm_para.h            |    3 ++
 4 files changed, 83 insertions(+)

Index: kvm-ptpdriver/arch/x86/include/uapi/asm/kvm_para.h
===================================================================

arch/x86/include/uapi/asm/kvm_para.h +9 -0
@@ -50,6 +50,15 @@
 	__u32 pad[11];
 };
 
+#define KVM_CLOCK_OFFSET_WALLCLOCK 0
+struct kvm_clock_offset {
+	__s64 sec;
+	__s64 nsec;
+	__u64 tsc;
+	__u32 flags;
+	__u32 pad[1];
+};
+
 #define KVM_STEAL_ALIGNMENT_BITS 5
 #define KVM_STEAL_VALID_BITS ((-1ULL << (KVM_STEAL_ALIGNMENT_BITS + 1)))
 #define KVM_STEAL_RESERVED_MASK (((1 << KVM_STEAL_ALIGNMENT_BITS) - 1 ) << 1)

Index: kvm-ptpdriver/Documentation/virtual/kvm/hypercalls.txt
===================================================================

Documentation/virtual/kvm/hypercalls.txt +13 -0
@@ -81,3 +81,33 @@
 same guest can wakeup the sleeping vcpu by issuing KVM_HC_KICK_CPU hypercall,
 specifying APIC ID (a1) of the vcpu to be woken up. An additional argument (a0)
 is used in the hypercall for future use.
+
+
+6. KVM_HC_CLOCK_OFFSET
+------------------------
+Architecture: x86
+Status: active
+Purpose: Hypercall used to synchronize host and guest clocks.
+Usage:
+
+a0: guest physical address where host copies
+"struct kvm_clock_offset" structure.
+
+a1: clock_type, ATM only KVM_CLOCK_OFFSET_WALLCLOCK (0)
Quoted 8 lines

Documentation/virtual/kvm/hypercalls.txt line 101

Quoted 5 lines
Documentation/virtual/kvm/hypercalls.txt +5 -0
+is supported (hosts CLOCK_REALTIME clock).
+
+		struct kvm_clock_offset {
+			__s64 sec;
+			__s64 nsec;

Why is nsec:
1) signed -- it is a remainder after division by NSEC_PER_SEC

Because "struct timespec" is signed because it can be used for time deltas (you won't actually get signed values for kvm_get_walltime_and_clockread).

Just wanted to match "struct timespec".

Quoted 2 lines

2) bigger than 32 bit -- NSEC_PER_SEC < 2^32
?

Again matching struct timespec.

Quoted 11 lines

Documentation/virtual/kvm/hypercalls.txt line 109

Quoted 8 lines
Documentation/virtual/kvm/hypercalls.txt +8 -0
+			__u64 tsc;
+			__u32 flags;
+			__u32 pad;
+		};
+
+       Where:
+               * sec: seconds from clock_type clock.
+               * nsec: nanoseconds from clock_type clock.

The important part of an offset is the starting point -- I assume it is the the usual one, but documentation better be explicit.

Don't get what you mean? (the values have same meaning as hosts clock_gettime(CLOCK_REALTIME), supposedly that is clear).

Quoted 4 lines

Documentation/virtual/kvm/hypercalls.txt line 110

Quoted 1 line
Documentation/virtual/kvm/hypercalls.txt +1 -0
+               * tsc: TSC value used to calculate sec/nsec pair

This hypercall could in theory work even when computing the offset with a different clocksource on the host and the estimating TSC at that time.

Yes it could.

Quoted 3 lines

Documentation/virtual/kvm/hypercalls.txt line 111

Quoted 1 line
Documentation/virtual/kvm/hypercalls.txt +1 -0
+                 (this hypercall only works when host uses TSC clocksource).

Describe that error as a return value.

Fixed.

+
+ if (kvm_get_walltime_and_clockread(&ts, &cycle) == false) {
+ kfree(clock_offset);
+ return -KVM_EOPNOTSUPP;
+ }

Quoted 4 lines

Documentation/virtual/kvm/hypercalls.txt line 112

Quoted 2 lines
Documentation/virtual/kvm/hypercalls.txt +1 -0
+               * flags: flags, unused at the moment.

unused = 0.

Fixed.

Quoted 18 lines
Quoted 18 lines
Documentation/virtual/kvm/hypercalls.txt +1 -0
+

Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c +9 -0
@@ -6121,6 +6121,44 @@
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
 
+static int kvm_pv_clock_offset(struct kvm_vcpu *vcpu, gpa_t paddr,
+			       unsigned long clock_type)
+{
+	struct kvm_clock_offset *clock_offset;
+	struct timespec ts;
+	cycle_t cycle;
+	int ret;
+
+	if (clock_type != KVM_CLOCK_OFFSET_WALLCLOCK)
Quoted 8 lines

arch/x86/kvm/x86.c line 6138

Quoted 6 lines
arch/x86/kvm/x86.c +6 -0
+		return -KVM_EOPNOTSUPP;
+
+	if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
+		return -KVM_EOPNOTSUPP;
+
+	clock_offset = kzalloc(sizeof(struct kvm_clock_offset), GFP_KERNEL);

Can't clock_offset be on the stack?

Well since the kernel stack is limited, thought it would be safer to allocate it.

Quoted 59 lines
Quoted 59 lines
arch/x86/kvm/x86.c +26 -0
+	if (clock_offset == NULL)
+		return -KVM_ENOMEM;
+
+	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false) {
+		kfree(clock_offset);
+		return -KVM_EOPNOTSUPP;
+	}
+
+	clock_offset->sec = ts.tv_sec;
+	clock_offset->nsec = ts.tv_nsec;
+	clock_offset->tsc = kvm_read_l1_tsc(vcpu, cycle);
+	clock_offset->flags = 0;
+
+	ret = 0;
+	if (kvm_write_guest(vcpu->kvm, paddr, clock_offset,
+			      sizeof(struct kvm_clock_offset)))
+		ret = -KVM_EFAULT;
+
+	kfree(clock_offset);
+
+	return ret;
+}
+
 /*
  * kvm_pv_kick_cpu_op:  Kick a vcpu.
  *

@@ -6185,6 +6223,9 @@
 		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
 		ret = 0;
 		break;
+	case KVM_HC_CLOCK_OFFSET:
+		ret = kvm_pv_clock_offset(vcpu, a0, a1);
+		break;
 	default:
 		ret = -KVM_ENOSYS;
 		break;

Index: kvm-ptpdriver/include/uapi/linux/kvm_para.h
===================================================================

include/uapi/linux/kvm_para.h +3 -0
@@ -14,6 +14,8 @@
 #define KVM_EFAULT		EFAULT
 #define KVM_E2BIG		E2BIG
 #define KVM_EPERM		EPERM
+#define KVM_EOPNOTSUPP		EOPNOTSUPP
+#define KVM_ENOMEM		ENOMEM
 
 #define KVM_HC_VAPIC_POLL_IRQ		1
 #define KVM_HC_MMU_OP			2

@@ -23,6 +25,7 @@
 #define KVM_HC_MIPS_GET_CLOCK_FREQ	6
 #define KVM_HC_MIPS_EXIT_VM		7
 #define KVM_HC_MIPS_CONSOLE_OUTPUT	8
+#define KVM_HC_CLOCK_OFFSET		9
 
 /*
  * hypercalls use architecture specific
#13 lore raw

Re: [patch 2/3] KVM: x86: add KVM_HC_CLOCK_OFFSET hypercall

2017-01-13 13:43-0200, Marcelo Tosatti:

Quoted 37 lines · Radim Krcmar
Quoted 23 lines

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 22 lines

Add a hypercall to retrieve the host realtime clock and the TSC value used to calculate that clock read.

Used to implement clock synchronization between
host and guest.

Signed-off-by Marcelo Tosatti <[email protected]>

---
Index: kvm-ptpdriver/Documentation/virtual/kvm/hypercalls.txt

Documentation/virtual/kvm/hypercalls.txt +11 -0
@@ -81,3 +81,33 @@
+6. KVM_HC_CLOCK_OFFSET
+------------------------
+Architecture: x86
+Status: active
+Purpose: Hypercall used to synchronize host and guest clocks.
+Usage:
+
+a0: guest physical address where host copies
+"struct kvm_clock_offset" structure.
+
+a1: clock_type, ATM only KVM_CLOCK_OFFSET_WALLCLOCK (0)
Quoted 8 lines

Documentation/virtual/kvm/hypercalls.txt line 101

Quoted 5 lines
Documentation/virtual/kvm/hypercalls.txt +5 -0
+is supported (hosts CLOCK_REALTIME clock).
+
+		struct kvm_clock_offset {
+			__s64 sec;
+			__s64 nsec;

Why is nsec:
1) signed -- it is a remainder after division by NSEC_PER_SEC

Because "struct timespec" is signed because it can be used for time deltas (you won't actually get signed values for kvm_get_walltime_and_clockread).

Quoted 6 lines

Just wanted to match "struct timespec".

Quoted 2 lines

2) bigger than 32 bit -- NSEC_PER_SEC < 2^32
?

Again matching struct timespec.

It is "long" in struct timespec, which could also be "s32" ... I'd rather waste those 8 bytes inside padding -- its purpose is clear there. :)

Quoted 12 lines
Quoted 11 lines

Documentation/virtual/kvm/hypercalls.txt line 109

Quoted 8 lines
Documentation/virtual/kvm/hypercalls.txt +8 -0
+			__u64 tsc;
+			__u32 flags;
+			__u32 pad;
+		};
+
+       Where:
+               * sec: seconds from clock_type clock.
+               * nsec: nanoseconds from clock_type clock.

The important part of an offset is the starting point -- I assume it is the the usual one, but documentation better be explicit.

Quoted 2 lines

Don't get what you mean? (the values have same meaning as hosts clock_gettime(CLOCK_REALTIME), supposedly that is clear).

Ah, I didn't understand that clock_type was refering to CLOCK_REALTIME.

I'd drop offset from the hypercall name. IIUC, all various clock types would be compared to TSC, so we could name the hypercall somewhat like KVM_HC_CLOCK_AT_TSC -- we want to know what was the time on the selected clock when TSC was at value __u64 tsc.
The only offset is between sec+nsec and the beginning of time (and tsc and 0), but we don't care about that offset by itself -- we care about relation of sec+nsec to tsc, which isn't a simple offset as they flow differently.

If we wanted to be more generic, then KVM_HC_CLOCK_PAIRING/SNAPSHOT/... and argument 1 would contain clock_types, here REALTIME and TSC.

author
#14 lore raw

Re: [patch 2/3] KVM: x86: add KVM_HC_CLOCK_OFFSET hypercall

Quoted 38 lines · Radim Krcmar

2017-01-13 13:43-0200, Marcelo Tosatti:

Quoted 37 lines · Radim Krcmar
Quoted 23 lines

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 22 lines

Add a hypercall to retrieve the host realtime clock and the TSC value used to calculate that clock read.

Used to implement clock synchronization between
host and guest.

Signed-off-by Marcelo Tosatti <[email protected]>

---
Index: kvm-ptpdriver/Documentation/virtual/kvm/hypercalls.txt

Documentation/virtual/kvm/hypercalls.txt +11 -0
@@ -81,3 +81,33 @@
+6. KVM_HC_CLOCK_OFFSET
+------------------------
+Architecture: x86
+Status: active
+Purpose: Hypercall used to synchronize host and guest clocks.
+Usage:
+
+a0: guest physical address where host copies
+"struct kvm_clock_offset" structure.
+
+a1: clock_type, ATM only KVM_CLOCK_OFFSET_WALLCLOCK (0)
Quoted 8 lines

Documentation/virtual/kvm/hypercalls.txt line 101

Quoted 5 lines
Documentation/virtual/kvm/hypercalls.txt +5 -0
+is supported (hosts CLOCK_REALTIME clock).
+
+		struct kvm_clock_offset {
+			__s64 sec;
+			__s64 nsec;

Why is nsec:
1) signed -- it is a remainder after division by NSEC_PER_SEC

Because "struct timespec" is signed because it can be used for time deltas (you won't actually get signed values for kvm_get_walltime_and_clockread).

Quoted 8 lines
Quoted 6 lines

Just wanted to match "struct timespec".

Quoted 2 lines

2) bigger than 32 bit -- NSEC_PER_SEC < 2^32
?

Again matching struct timespec.

It is "long" in struct timespec, which could also be "s32" ...

Which fits in a s64.

Quoted 2 lines

I'd rather waste those 8 bytes inside padding -- its purpose is clear there. :)

The purpose of the s64 is to match timespec.

Quoted 15 lines
Quoted 12 lines
Quoted 11 lines

Documentation/virtual/kvm/hypercalls.txt line 109

Quoted 8 lines
Documentation/virtual/kvm/hypercalls.txt +8 -0
+			__u64 tsc;
+			__u32 flags;
+			__u32 pad;
+		};
+
+       Where:
+               * sec: seconds from clock_type clock.
+               * nsec: nanoseconds from clock_type clock.

The important part of an offset is the starting point -- I assume it is the the usual one, but documentation better be explicit.

Quoted 2 lines

Don't get what you mean? (the values have same meaning as hosts clock_gettime(CLOCK_REALTIME), supposedly that is clear).

Quoted 6 lines

Ah, I didn't understand that clock_type was refering to CLOCK_REALTIME.

I'd drop offset from the hypercall name. IIUC, all various clock types would be compared to TSC, so we could name the hypercall somewhat like KVM_HC_CLOCK_AT_TSC -- we want to know what was the time on the selected clock when TSC was at value __u64 tsc.

Well its the offset from TSC, so thats why "offset" in the name.

And as you say, the interface could be extended to support other clocks (using the available data in pad, i'll make it larger BTW).

Which would render the "TSC" from the hypercall name invalid.

Quoted 7 lines

The only offset is between sec+nsec and the beginning of time (and tsc and 0), but we don't care about that offset by itself -- we care about relation of sec+nsec to tsc, which isn't a simple offset as they flow differently.

If we wanted to be more generic, then KVM_HC_CLOCK_PAIRING/SNAPSHOT/... and argument 1 would contain clock_types, here REALTIME and TSC.

Pairing is fine, changed.

author
#15 lore raw

PATCH 3/3 PTP: add kvm PTP driver

In reply to Marcelo Tosatti

Add a driver with gettime method returning hosts realtime clock. This allows Chrony to synchronize host and guest clocks with high precision (see results below).

chronyc> sources
MS Name/IP address Stratum Poll Reach LastRx Last sample
===============================================================================
#* PHC0 0 3 377 6 +4ns[ +4ns] +/- 3ns

To configure Chronyd to use PHC refclock, add the
following line to its configuration file:

refclock PHC /dev/ptpX poll 3 dpoll -2 offset 0

Where /dev/ptpX is the kvmclock PTP clock.

---
 drivers/ptp/Kconfig   |   12 +++
 drivers/ptp/Makefile  |    1 
 drivers/ptp/ptp_kvm.c |  180 ++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 193 insertions(+)
drivers/ptp/Kconfig +12 -0
Index: kvm-ptpdriver/drivers/ptp/Kconfig
===================================================================
@@ -90,4 +90,16 @@
 	  To compile this driver as a module, choose M here: the module
 	  will be called ptp_pch.
 
+config PTP_1588_CLOCK_KVM
+	tristate "KVM virtual PTP clock"
+	depends on PTP_1588_CLOCK
+	depends on KVM_GUEST
+	default y
+	help
+	  This driver adds support for using kvm infrastructure as a PTP
+	  clock. This clock is only useful if you are using KVM guests.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called ptp_kvm.
+
 endmenu
drivers/ptp/ptp_kvm.c new file +180 -0

180 lines not on this page · raw message

drivers/ptp/Makefile +1 -0
Index: kvm-ptpdriver/drivers/ptp/Makefile
===================================================================
@@ -6,3 +6,4 @@
 obj-$(CONFIG_PTP_1588_CLOCK)		+= ptp.o
 obj-$(CONFIG_PTP_1588_CLOCK_IXP46X)	+= ptp_ixp46x.o
 obj-$(CONFIG_PTP_1588_CLOCK_PCH)	+= ptp_pch.o
+obj-$(CONFIG_PTP_1588_CLOCK_KVM)	+= ptp_kvm.o
#16 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 119 lines

Add a driver with gettime method returning hosts realtime clock. This allows Chrony to synchronize host and guest clocks with high precision (see results below).

chronyc> sources
MS Name/IP address Stratum Poll Reach LastRx Last sample
===============================================================================
#* PHC0 0 3 377 6 +4ns[ +4ns] +/- 3ns

To configure Chronyd to use PHC refclock, add the
following line to its configuration file:

refclock PHC /dev/ptpX poll 3 dpoll -2 offset 0

Where /dev/ptpX is the kvmclock PTP clock.

---
 drivers/ptp/Kconfig   |   12 +++
 drivers/ptp/Makefile  |    1 
 drivers/ptp/ptp_kvm.c |  180 ++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 193 insertions(+)

Index: kvm-ptpdriver/drivers/ptp/Kconfig
===================================================================

drivers/ptp/Kconfig +12 -0
@@ -90,4 +90,16 @@
 	  To compile this driver as a module, choose M here: the module
 	  will be called ptp_pch.
 
+config PTP_1588_CLOCK_KVM
+	tristate "KVM virtual PTP clock"
+	depends on PTP_1588_CLOCK
+	depends on KVM_GUEST
+	default y
+	help
+	  This driver adds support for using kvm infrastructure as a PTP
+	  clock. This clock is only useful if you are using KVM guests.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called ptp_kvm.
+
 endmenu

Index: kvm-ptpdriver/drivers/ptp/ptp_kvm.c
===================================================================

drivers/ptp/ptp_kvm.c new file +70 -0
@@ -0,0 +1,180 @@
+/*
+ * Virtual PTP 1588 clock for use with KVM guests
+ *
+ * Copyright (C) 2017 Red Hat Inc.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ */
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <uapi/linux/kvm_para.h>
+#include <asm/kvm_para.h>
+#include <asm/pvclock.h>
+#include <uapi/asm/kvm_para.h>
+
+#include <linux/ptp_clock_kernel.h>
+
+struct kvm_ptp_clock {
+	struct ptp_clock *ptp_clock;
+	struct ptp_clock_info caps;
+};
+
+DEFINE_SPINLOCK(kvm_ptp_lock);
+
+static struct pvclock_vsyscall_time_info *hv_clock;
+
+/*
+ * PTP clock operations
+ */
+
+static int ptp_kvm_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
+{
+	return -EOPNOTSUPP;
+}
+
+static int ptp_kvm_adjtime(struct ptp_clock_info *ptp, s64 delta)
+{
+	return -EOPNOTSUPP;
+}
+
+static struct kvm_clock_offset clock_off;
+static phys_addr_t clock_off_gpa;
+
+static int ptp_kvm_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts)
+{
+	unsigned long ret;
+	struct timespec64 tspec;
+	u64 delta;
+	cycle_t offset;
+	unsigned version;
+	int cpu;
+	struct pvclock_vcpu_time_info *src;
+
+	preempt_disable_notrace();
+	cpu = smp_processor_id();
+	src = &hv_clock[cpu].pvti;
+
+	spin_lock(&kvm_ptp_lock);
+

drivers/ptp/ptp_kvm.c line 80

Quoted 10 lines
drivers/ptp/ptp_kvm.c new file +10 -0
+	do {
+		/*
+		 * We are measuring the delay between
+		 * kvm_hypercall and rdtsc using TSC,
+		 * and converting that delta to
+		 * tsc_to_system_mul and tsc_shift
+		 * So any changes to tsc_to_system_mul
+		 * and tsc_shift in this region
+		 * invalidate the measurement.
+		 */

This assumes that the host uses kvmclock, but the guest can be using just TSC and even have kvmclock disabled. This code should at least check that kvmclock enabled.

(If we made kvmclock mandatory, then we could fix pvclock_wall_clock
interface, because it is already defined to provide real time based on kvmclock ...)

Quoted 26 lines
drivers/ptp/ptp_kvm.c new file +26 -0
+		version = pvclock_read_begin(src);
+
+		ret = kvm_hypercall2(KVM_HC_CLOCK_OFFSET,
+				     clock_off_gpa,
+				     KVM_CLOCK_OFFSET_WALLCLOCK);
+		if (ret != 0) {
+			pr_err("clock offset hypercall ret %lu\n", ret);
+			spin_unlock(&kvm_ptp_lock);
+			preempt_enable_notrace();
+			return -EOPNOTSUPP;
+		}
+
+		tspec.tv_sec = clock_off.sec;
+		tspec.tv_nsec = clock_off.nsec;
+
+		delta = rdtsc_ordered() - clock_off.tsc;
+
+		offset = pvclock_scale_delta(delta, src->tsc_to_system_mul,
+					     src->tsc_shift);
+
+	} while (pvclock_read_retry(src, version));
+
+	preempt_enable_notrace();
+
+	tspec.tv_nsec = tspec.tv_nsec + offset;
+

drivers/ptp/ptp_kvm.c line 116

Quoted 10 lines
drivers/ptp/ptp_kvm.c new file +10 -0
+	spin_unlock(&kvm_ptp_lock);
+
+	if (tspec.tv_nsec >= NSEC_PER_SEC) {
+		u64 secs = tspec.tv_nsec;
+
+		tspec.tv_nsec = do_div(secs, NSEC_PER_SEC);
+		tspec.tv_sec += secs;
+	}
+
+	memcpy(ts, &tspec, sizeof(struct timespec64));

But the whole idea is of improving the time by reading tsc a bit later is just weird ... why is it better to provide

tsc + x, time + tsc_delta_to_time(x)

than just

tsc, time

?

Because we'll always be quering the time at tsc + y, where y >> x, and we'd likely have other problems if shifting the time base by few thousand cycles made a difference.

Quoted 37 lines
drivers/ptp/ptp_kvm.c new file +37 -0
+
+	return 0;
+}
+
+static int ptp_kvm_settime(struct ptp_clock_info *ptp,
+			   const struct timespec64 *ts)
+{
+	return -EOPNOTSUPP;
+}
+
+static int ptp_kvm_enable(struct ptp_clock_info *ptp,
+			  struct ptp_clock_request *rq, int on)
+{
+	return -EOPNOTSUPP;
+}
+
+static struct ptp_clock_info ptp_kvm_caps = {
+	.owner		= THIS_MODULE,
+	.name		= "KVM virtual PTP",
+	.max_adj	= 0,
+	.n_ext_ts	= 0,
+	.n_pins		= 0,
+	.pps		= 0,
+	.adjfreq	= ptp_kvm_adjfreq,
+	.adjtime	= ptp_kvm_adjtime,
+	.gettime64	= ptp_kvm_gettime,
+	.settime64	= ptp_kvm_settime,
+	.enable		= ptp_kvm_enable,
+};
+
+/* module operations */
+
+static struct kvm_ptp_clock kvm_ptp_clock;
+
+static void __exit ptp_kvm_exit(void)
+{
+	ptp_clock_unregister(kvm_ptp_clock.ptp_clock);

drivers/ptp/ptp_kvm.c line 163

Quoted 10 lines
drivers/ptp/ptp_kvm.c new file +10 -0
+}
+
+static int __init ptp_kvm_init(void)
+{
+	if (!kvm_para_available())
+		return -ENODEV;
+
+	kvm_ptp_clock.caps = ptp_kvm_caps;
+
+	kvm_ptp_clock.ptp_clock = ptp_clock_register(&kvm_ptp_clock.caps, NULL);

It is a shame that the infrastructure uses polling when the guest could be notified on every host real time change, but this should be good enough.

drivers/ptp/ptp_kvm.c line 170

Quoted 7 lines
drivers/ptp/ptp_kvm.c new file +7 -0
+
+	if (IS_ERR(kvm_ptp_clock.ptp_clock))
+		return PTR_ERR(kvm_ptp_clock.ptp_clock);
+
+	clock_off_gpa = slow_virt_to_phys(&clock_off);
+
+	hv_clock = pvclock_pvti_cpu0_va();

Would safer to assign required globals before the registration -- races could get ugly.

Thanks.

Quoted 21 lines
drivers/ptp/ptp_kvm.c new file +10 -0
+
+	return 0;
+}
+
+module_init(ptp_kvm_init);
+module_exit(ptp_kvm_exit);
+
+MODULE_AUTHOR("Marcelo Tosatti <[email protected]>");
+MODULE_DESCRIPTION("PTP clock using KVMCLOCK");
+MODULE_LICENSE("GPL");

Index: kvm-ptpdriver/drivers/ptp/Makefile
===================================================================

drivers/ptp/Makefile +1 -0
@@ -6,3 +6,4 @@
 obj-$(CONFIG_PTP_1588_CLOCK)		+= ptp.o
 obj-$(CONFIG_PTP_1588_CLOCK_IXP46X)	+= ptp_ixp46x.o
 obj-$(CONFIG_PTP_1588_CLOCK_PCH)	+= ptp_pch.o
+obj-$(CONFIG_PTP_1588_CLOCK_KVM)	+= ptp_kvm.o
author
#17 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 126 lines · Radim Krcmar

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 125 lines

Add a driver with gettime method returning hosts realtime clock. This allows Chrony to synchronize host and guest clocks with high precision (see results below).

chronyc> sources
MS Name/IP address Stratum Poll Reach LastRx Last sample
===============================================================================
#* PHC0 0 3 377 6 +4ns[ +4ns] +/- 3ns

To configure Chronyd to use PHC refclock, add the
following line to its configuration file:

refclock PHC /dev/ptpX poll 3 dpoll -2 offset 0

Where /dev/ptpX is the kvmclock PTP clock.

---
 drivers/ptp/Kconfig   |   12 +++
 drivers/ptp/Makefile  |    1 
 drivers/ptp/ptp_kvm.c |  180 ++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 193 insertions(+)

Index: kvm-ptpdriver/drivers/ptp/Kconfig
===================================================================

drivers/ptp/Kconfig +12 -0
@@ -90,4 +90,16 @@
 	  To compile this driver as a module, choose M here: the module
 	  will be called ptp_pch.
 
+config PTP_1588_CLOCK_KVM
+	tristate "KVM virtual PTP clock"
+	depends on PTP_1588_CLOCK
+	depends on KVM_GUEST
+	default y
+	help
+	  This driver adds support for using kvm infrastructure as a PTP
+	  clock. This clock is only useful if you are using KVM guests.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called ptp_kvm.
+
 endmenu

Index: kvm-ptpdriver/drivers/ptp/ptp_kvm.c
===================================================================

drivers/ptp/ptp_kvm.c new file +76 -0
@@ -0,0 +1,180 @@
+/*
+ * Virtual PTP 1588 clock for use with KVM guests
+ *
+ * Copyright (C) 2017 Red Hat Inc.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ */
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <uapi/linux/kvm_para.h>
+#include <asm/kvm_para.h>
+#include <asm/pvclock.h>
+#include <uapi/asm/kvm_para.h>
+
+#include <linux/ptp_clock_kernel.h>
+
+struct kvm_ptp_clock {
+	struct ptp_clock *ptp_clock;
+	struct ptp_clock_info caps;
+};
+
+DEFINE_SPINLOCK(kvm_ptp_lock);
+
+static struct pvclock_vsyscall_time_info *hv_clock;
+
+/*
+ * PTP clock operations
+ */
+
+static int ptp_kvm_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
+{
+	return -EOPNOTSUPP;
+}
+
+static int ptp_kvm_adjtime(struct ptp_clock_info *ptp, s64 delta)
+{
+	return -EOPNOTSUPP;
+}
+
+static struct kvm_clock_offset clock_off;
+static phys_addr_t clock_off_gpa;
+
+static int ptp_kvm_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts)
+{
+	unsigned long ret;
+	struct timespec64 tspec;
+	u64 delta;
+	cycle_t offset;
+	unsigned version;
+	int cpu;
+	struct pvclock_vcpu_time_info *src;
+
+	preempt_disable_notrace();
+	cpu = smp_processor_id();
+	src = &hv_clock[cpu].pvti;
+
+	spin_lock(&kvm_ptp_lock);
+
+	do {
+		/*
+		 * We are measuring the delay between
+		 * kvm_hypercall and rdtsc using TSC,
+		 * and converting that delta to
+		 * tsc_to_system_mul and tsc_shift
Quoted 8 lines

drivers/ptp/ptp_kvm.c line 80

Quoted 4 lines
drivers/ptp/ptp_kvm.c new file +4 -0
+		 * So any changes to tsc_to_system_mul
+		 * and tsc_shift in this region
+		 * invalidate the measurement.
+		 */

This assumes that the host uses kvmclock, but the guest can be using just TSC and even have kvmclock disabled. This code should at least check that kvmclock enabled.

Fixed.

Quoted 44 lines

(If we made kvmclock mandatory, then we could fix pvclock_wall_clock
interface, because it is already defined to provide real time based on kvmclock ...)

Quoted 26 lines
drivers/ptp/ptp_kvm.c new file +26 -0
+		version = pvclock_read_begin(src);
+
+		ret = kvm_hypercall2(KVM_HC_CLOCK_OFFSET,
+				     clock_off_gpa,
+				     KVM_CLOCK_OFFSET_WALLCLOCK);
+		if (ret != 0) {
+			pr_err("clock offset hypercall ret %lu\n", ret);
+			spin_unlock(&kvm_ptp_lock);
+			preempt_enable_notrace();
+			return -EOPNOTSUPP;
+		}
+
+		tspec.tv_sec = clock_off.sec;
+		tspec.tv_nsec = clock_off.nsec;
+
+		delta = rdtsc_ordered() - clock_off.tsc;
+
+		offset = pvclock_scale_delta(delta, src->tsc_to_system_mul,
+					     src->tsc_shift);
+
+	} while (pvclock_read_retry(src, version));
+
+	preempt_enable_notrace();
+
+	tspec.tv_nsec = tspec.tv_nsec + offset;
+

drivers/ptp/ptp_kvm.c line 116

Quoted 10 lines
drivers/ptp/ptp_kvm.c new file +10 -0
+	spin_unlock(&kvm_ptp_lock);
+
+	if (tspec.tv_nsec >= NSEC_PER_SEC) {
+		u64 secs = tspec.tv_nsec;
+
+		tspec.tv_nsec = do_div(secs, NSEC_PER_SEC);
+		tspec.tv_sec += secs;
+	}
+
+	memcpy(ts, &tspec, sizeof(struct timespec64));

But the whole idea is of improving the time by reading tsc a bit later is just weird ... why is it better to provide

Quoted 7 lines

tsc + x, time + tsc_delta_to_time(x)

than just

tsc, time

?

Because you want to calculate the value of the host realtime clock at the moment of ptp_kvm_gettime.

We do:

1. kvm_hypercall.
2. get {sec, nsec, guest_tsc}.
3. kvm_hypercall returns.
4. delay = rdtsc() - guest_tsc.

Where delay is the delta (measured with the TSC) between points 2 and 4.

Quoted 3 lines

Because we'll always be quering the time at tsc + y, where y >> x, and we'd likely have other problems if shifting the time base by few thousand cycles made a difference.

Radim, i didnt get your "tsc + x", "time + tsc_delta_to_time(x)" formulas above. Can you be more verbose please?

Quoted 43 lines
Quoted 43 lines
drivers/ptp/ptp_kvm.c new file +43 -0
+
+	return 0;
+}
+
+static int ptp_kvm_settime(struct ptp_clock_info *ptp,
+			   const struct timespec64 *ts)
+{
+	return -EOPNOTSUPP;
+}
+
+static int ptp_kvm_enable(struct ptp_clock_info *ptp,
+			  struct ptp_clock_request *rq, int on)
+{
+	return -EOPNOTSUPP;
+}
+
+static struct ptp_clock_info ptp_kvm_caps = {
+	.owner		= THIS_MODULE,
+	.name		= "KVM virtual PTP",
+	.max_adj	= 0,
+	.n_ext_ts	= 0,
+	.n_pins		= 0,
+	.pps		= 0,
+	.adjfreq	= ptp_kvm_adjfreq,
+	.adjtime	= ptp_kvm_adjtime,
+	.gettime64	= ptp_kvm_gettime,
+	.settime64	= ptp_kvm_settime,
+	.enable		= ptp_kvm_enable,
+};
+
+/* module operations */
+
+static struct kvm_ptp_clock kvm_ptp_clock;
+
+static void __exit ptp_kvm_exit(void)
+{
+	ptp_clock_unregister(kvm_ptp_clock.ptp_clock);
+}
+
+static int __init ptp_kvm_init(void)
+{
+	if (!kvm_para_available())
+		return -ENODEV;
Quoted 8 lines

drivers/ptp/ptp_kvm.c line 163

Quoted 4 lines
drivers/ptp/ptp_kvm.c new file +4 -0
+
+	kvm_ptp_clock.caps = ptp_kvm_caps;
+
+	kvm_ptp_clock.ptp_clock = ptp_clock_register(&kvm_ptp_clock.caps, NULL);

It is a shame that the infrastructure uses polling when the guest could be notified on every host real time change, but this should be good enough.

Well, the whole NTP scheme is a feedback loop, so it wants to query the value of the clock at the moments the feedback loop logic decides it.

Quoted 9 lines

drivers/ptp/ptp_kvm.c line 170

Quoted 6 lines
drivers/ptp/ptp_kvm.c new file +6 -0
+	if (IS_ERR(kvm_ptp_clock.ptp_clock))
+		return PTR_ERR(kvm_ptp_clock.ptp_clock);
+
+	clock_off_gpa = slow_virt_to_phys(&clock_off);
+
+	hv_clock = pvclock_pvti_cpu0_va();

Would safer to assign required globals before the registration -- races could get ugly.

Fixed, thanks.

#18 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

2017-01-13 15:40-0200, Marcelo Tosatti:

Quoted 53 lines · Radim Krcmar
Quoted 41 lines

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 26 lines
drivers/ptp/ptp_kvm.c new file +26 -0
+		version = pvclock_read_begin(src);
+
+		ret = kvm_hypercall2(KVM_HC_CLOCK_OFFSET,
+				     clock_off_gpa,
+				     KVM_CLOCK_OFFSET_WALLCLOCK);
+		if (ret != 0) {
+			pr_err("clock offset hypercall ret %lu\n", ret);
+			spin_unlock(&kvm_ptp_lock);
+			preempt_enable_notrace();
+			return -EOPNOTSUPP;
+		}
+
+		tspec.tv_sec = clock_off.sec;
+		tspec.tv_nsec = clock_off.nsec;
+
+		delta = rdtsc_ordered() - clock_off.tsc;
+
+		offset = pvclock_scale_delta(delta, src->tsc_to_system_mul,
+					     src->tsc_shift);
+
+	} while (pvclock_read_retry(src, version));
+
+	preempt_enable_notrace();
+
+	tspec.tv_nsec = tspec.tv_nsec + offset;
+

drivers/ptp/ptp_kvm.c line 116

Quoted 10 lines
drivers/ptp/ptp_kvm.c new file +10 -0
+	spin_unlock(&kvm_ptp_lock);
+
+	if (tspec.tv_nsec >= NSEC_PER_SEC) {
+		u64 secs = tspec.tv_nsec;
+
+		tspec.tv_nsec = do_div(secs, NSEC_PER_SEC);
+		tspec.tv_sec += secs;
+	}
+
+	memcpy(ts, &tspec, sizeof(struct timespec64));

But the whole idea is of improving the time by reading tsc a bit later is just weird ... why is it better to provide

Quoted 7 lines

tsc + x, time + tsc_delta_to_time(x)

than just

tsc, time

?

Because you want to calculate the value of the host realtime clock at the moment of ptp_kvm_gettime.

Quoted 8 lines

We do:

1. kvm_hypercall.
2. get {sec, nsec, guest_tsc}.
3. kvm_hypercall returns.
4. delay = rdtsc() - guest_tsc.

Where delay is the delta (measured with the TSC) between points 2 and 4.

I see now ... the PTP interface is just not good for our purposes. We don't return {sec, nsec, guest_tsc}, we just return {sec, nsec} at some random time in the past. And to make it a bit more accurate, you add a best-effort delta before returning, which makes sense.

When we have to depend on pvclock, what are the advantages of not using the existing pvclock API for wall clock?
(You mentioned some extensions.)

  struct pvclock_wall_clock {
  	u32   version;
  	u32   sec;
  	u32   nsec;
  } __attribute__((__packed__));

It gives the wall clock when pvclock was 0, so you just add current kvmclock and get the host wall clock. Without a VM exit.

And how often is ptp_kvm_gettime() usually called?

Thanks.

Quoted 6 lines
Quoted 3 lines

Because we'll always be quering the time at tsc + y, where y >> x, and we'd likely have other problems if shifting the time base by few thousand cycles made a difference.

Radim, i didnt get your "tsc + x", "time + tsc_delta_to_time(x)" formulas above. Can you be more verbose please?

x is the delta, tsc_delta_to_time() is what pvclock_scale_delta() does.

I assumed that we set precise time with TSC, so the delta wouldn't matter, because PTP would either get {sec, nsec, guest_tsc}, or the same, but just shifted by delta, hence
{sec + tsc_delta_to_time(x) / NSEC_PER_SEC,
nsec + tsc_delta_to_time(x) % NSEC_PER_SEC,
guest_tsc + x}.

#19 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

2017-01-16 17:26+0100, Radim Krcmar:

Quoted 56 lines

2017-01-13 15:40-0200, Marcelo Tosatti:

Quoted 55 lines · Radim Krcmar
Quoted 41 lines

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 26 lines
drivers/ptp/ptp_kvm.c new file +26 -0
+		version = pvclock_read_begin(src);
+
+		ret = kvm_hypercall2(KVM_HC_CLOCK_OFFSET,
+				     clock_off_gpa,
+				     KVM_CLOCK_OFFSET_WALLCLOCK);
+		if (ret != 0) {
+			pr_err("clock offset hypercall ret %lu\n", ret);
+			spin_unlock(&kvm_ptp_lock);
+			preempt_enable_notrace();
+			return -EOPNOTSUPP;
+		}
+
+		tspec.tv_sec = clock_off.sec;
+		tspec.tv_nsec = clock_off.nsec;
+
+		delta = rdtsc_ordered() - clock_off.tsc;
+
+		offset = pvclock_scale_delta(delta, src->tsc_to_system_mul,
+					     src->tsc_shift);
+
+	} while (pvclock_read_retry(src, version));
+
+	preempt_enable_notrace();
+
+	tspec.tv_nsec = tspec.tv_nsec + offset;
+

drivers/ptp/ptp_kvm.c line 116

Quoted 10 lines
drivers/ptp/ptp_kvm.c new file +10 -0
+	spin_unlock(&kvm_ptp_lock);
+
+	if (tspec.tv_nsec >= NSEC_PER_SEC) {
+		u64 secs = tspec.tv_nsec;
+
+		tspec.tv_nsec = do_div(secs, NSEC_PER_SEC);
+		tspec.tv_sec += secs;
+	}
+
+	memcpy(ts, &tspec, sizeof(struct timespec64));

But the whole idea is of improving the time by reading tsc a bit later is just weird ... why is it better to provide

Quoted 7 lines

tsc + x, time + tsc_delta_to_time(x)

than just

tsc, time

?

Because you want to calculate the value of the host realtime clock at the moment of ptp_kvm_gettime.

We do:

Quoted 8 lines
Quoted 6 lines

1. kvm_hypercall.
2. get {sec, nsec, guest_tsc}.
3. kvm_hypercall returns.
4. delay = rdtsc() - guest_tsc.

Where delay is the delta (measured with the TSC) between points 2 and 4.

I see now ... the PTP interface is just not good for our purposes.

There is getcrosststamp() callback in PTP, which seems to be exactly what we want when pairing with TSC, so the pvclock delay fixup can be dropped when using it.

author
#20 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 62 lines · Radim Krcmar

2017-01-16 17:26+0100, Radim Krcmar:

Quoted 61 lines

2017-01-13 15:40-0200, Marcelo Tosatti:

Quoted 60 lines · Radim Krcmar
Quoted 41 lines

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 26 lines
drivers/ptp/ptp_kvm.c new file +26 -0
+		version = pvclock_read_begin(src);
+
+		ret = kvm_hypercall2(KVM_HC_CLOCK_OFFSET,
+				     clock_off_gpa,
+				     KVM_CLOCK_OFFSET_WALLCLOCK);
+		if (ret != 0) {
+			pr_err("clock offset hypercall ret %lu\n", ret);
+			spin_unlock(&kvm_ptp_lock);
+			preempt_enable_notrace();
+			return -EOPNOTSUPP;
+		}
+
+		tspec.tv_sec = clock_off.sec;
+		tspec.tv_nsec = clock_off.nsec;
+
+		delta = rdtsc_ordered() - clock_off.tsc;
+
+		offset = pvclock_scale_delta(delta, src->tsc_to_system_mul,
+					     src->tsc_shift);
+
+	} while (pvclock_read_retry(src, version));
+
+	preempt_enable_notrace();
+
+	tspec.tv_nsec = tspec.tv_nsec + offset;
+

drivers/ptp/ptp_kvm.c line 116

Quoted 10 lines
drivers/ptp/ptp_kvm.c new file +10 -0
+	spin_unlock(&kvm_ptp_lock);
+
+	if (tspec.tv_nsec >= NSEC_PER_SEC) {
+		u64 secs = tspec.tv_nsec;
+
+		tspec.tv_nsec = do_div(secs, NSEC_PER_SEC);
+		tspec.tv_sec += secs;
+	}
+
+	memcpy(ts, &tspec, sizeof(struct timespec64));

But the whole idea is of improving the time by reading tsc a bit later is just weird ... why is it better to provide

Quoted 7 lines

tsc + x, time + tsc_delta_to_time(x)

than just

tsc, time

?

Because you want to calculate the value of the host realtime clock at the moment of ptp_kvm_gettime.

We do:

1. kvm_hypercall.
2. get {sec, nsec, guest_tsc}.
3. kvm_hypercall returns.
4. delay = rdtsc() - guest_tsc.

Quoted 7 lines
Quoted 3 lines
Quoted 1 line

Where delay is the delta (measured with the TSC) between points 2 and 4.

I see now ... the PTP interface is just not good for our purposes.

There is getcrosststamp() callback in PTP, which seems to be exactly what we want when pairing with TSC, so the pvclock delay fixup can be dropped when using it.

What pvclock delay fixup you refer to? The "rdtsc() - clock_offset.tsc" part? You can't drop it, because if you do then your "host realtime clock read" will be behind by "rdtsc() - clock_offset.tsc" TSC cycles. We want the highest precision as possible.

#21 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

2017-01-16 15:08-0200, Marcelo Tosatti:

Quoted 65 lines · Radim Krcmar
Quoted 64 lines

2017-01-16 17:26+0100, Radim Krcmar:

Quoted 63 lines

2017-01-13 15:40-0200, Marcelo Tosatti:

Quoted 61 lines · Radim Krcmar
Quoted 41 lines

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 26 lines
drivers/ptp/ptp_kvm.c new file +26 -0
+		version = pvclock_read_begin(src);
+
+		ret = kvm_hypercall2(KVM_HC_CLOCK_OFFSET,
+				     clock_off_gpa,
+				     KVM_CLOCK_OFFSET_WALLCLOCK);
+		if (ret != 0) {
+			pr_err("clock offset hypercall ret %lu\n", ret);
+			spin_unlock(&kvm_ptp_lock);
+			preempt_enable_notrace();
+			return -EOPNOTSUPP;
+		}
+
+		tspec.tv_sec = clock_off.sec;
+		tspec.tv_nsec = clock_off.nsec;
+
+		delta = rdtsc_ordered() - clock_off.tsc;
+
+		offset = pvclock_scale_delta(delta, src->tsc_to_system_mul,
+					     src->tsc_shift);
+
+	} while (pvclock_read_retry(src, version));
+
+	preempt_enable_notrace();
+
+	tspec.tv_nsec = tspec.tv_nsec + offset;
+

drivers/ptp/ptp_kvm.c line 116

Quoted 10 lines
drivers/ptp/ptp_kvm.c new file +10 -0
+	spin_unlock(&kvm_ptp_lock);
+
+	if (tspec.tv_nsec >= NSEC_PER_SEC) {
+		u64 secs = tspec.tv_nsec;
+
+		tspec.tv_nsec = do_div(secs, NSEC_PER_SEC);
+		tspec.tv_sec += secs;
+	}
+
+	memcpy(ts, &tspec, sizeof(struct timespec64));

But the whole idea is of improving the time by reading tsc a bit later is just weird ... why is it better to provide

Quoted 7 lines

tsc + x, time + tsc_delta_to_time(x)

than just

tsc, time

?

Because you want to calculate the value of the host realtime clock at the moment of ptp_kvm_gettime.

We do:

1. kvm_hypercall.
2. get {sec, nsec, guest_tsc}.
3. kvm_hypercall returns.
4. delay = rdtsc() - guest_tsc.

Where delay is the delta (measured with the TSC) between points 2 and 4.

Quoted 8 lines
Quoted 5 lines
Quoted 1 line

I see now ... the PTP interface is just not good for our purposes.

There is getcrosststamp() callback in PTP, which seems to be exactly what we want when pairing with TSC, so the pvclock delay fixup can be dropped when using it.

What pvclock delay fixup you refer to? The "rdtsc() - clock_offset.tsc" part?

Yes.

Quoted 2 lines

You can't drop it, because if you do then your "host realtime
clock read" will be behind by "rdtsc() - clock_offset.tsc" TSC cycles.

The TSC read will be some cycles old when the hypercall ends, but that doesn't matter, because we will pass {sec, nsec, guest_tsc} to PTP and PTP should plug them into kernel's realtime clock roughly like this:

sec/nsec + (rdtsc() - guest_tsc) * tsc_freq

Adding delay to guest_tsc and sec/nsec cannot improve precision. (And will likely degrade it as kvmclock's frequency is incorrect.)

Quoted 1 line

We want the highest precision as possible.

I agree, which is why we don't want to lose precision in the delay guesswork because of gettime64().

author
#22 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 86 lines · Radim Krcmar

2017-01-16 15:08-0200, Marcelo Tosatti:

Quoted 65 lines · Radim Krcmar
Quoted 64 lines

2017-01-16 17:26+0100, Radim Krcmar:

Quoted 63 lines

2017-01-13 15:40-0200, Marcelo Tosatti:

Quoted 61 lines · Radim Krcmar
Quoted 41 lines

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 26 lines
(quoted diff) +26 -0
+		version = pvclock_read_begin(src);
+
+		ret = kvm_hypercall2(KVM_HC_CLOCK_OFFSET,
+				     clock_off_gpa,
+				     KVM_CLOCK_OFFSET_WALLCLOCK);
+		if (ret != 0) {
+			pr_err("clock offset hypercall ret %lu\n", ret);
+			spin_unlock(&kvm_ptp_lock);
+			preempt_enable_notrace();
+			return -EOPNOTSUPP;
+		}
+
+		tspec.tv_sec = clock_off.sec;
+		tspec.tv_nsec = clock_off.nsec;
+
+		delta = rdtsc_ordered() - clock_off.tsc;
+
+		offset = pvclock_scale_delta(delta, src->tsc_to_system_mul,
+					     src->tsc_shift);
+
+	} while (pvclock_read_retry(src, version));
+
+	preempt_enable_notrace();
+
+	tspec.tv_nsec = tspec.tv_nsec + offset;
+

Review

Quoted 10 lines
(quoted diff) +10 -0
+	spin_unlock(&kvm_ptp_lock);
+
+	if (tspec.tv_nsec >= NSEC_PER_SEC) {
+		u64 secs = tspec.tv_nsec;
+
+		tspec.tv_nsec = do_div(secs, NSEC_PER_SEC);
+		tspec.tv_sec += secs;
+	}
+
+	memcpy(ts, &tspec, sizeof(struct timespec64));

But the whole idea is of improving the time by reading tsc a bit later is just weird ... why is it better to provide

Quoted 7 lines

tsc + x, time + tsc_delta_to_time(x)

than just

tsc, time

?

Because you want to calculate the value of the host realtime clock at the moment of ptp_kvm_gettime.

We do:

1. kvm_hypercall.
2. get {sec, nsec, guest_tsc}.
3. kvm_hypercall returns.
4. delay = rdtsc() - guest_tsc.

Where delay is the delta (measured with the TSC) between points 2 and 4.

Quoted 8 lines
Quoted 5 lines
Quoted 1 line

I see now ... the PTP interface is just not good for our purposes.

There is getcrosststamp() callback in PTP, which seems to be exactly what we want when pairing with TSC, so the pvclock delay fixup can be dropped when using it.

What pvclock delay fixup you refer to? The "rdtsc() - clock_offset.tsc" part?

Yes.

Quoted 2 lines

You can't drop it, because if you do then your "host realtime
clock read" will be behind by "rdtsc() - clock_offset.tsc" TSC cycles.

The TSC read will be some cycles old when the hypercall ends, but that doesn't matter, because we will pass {sec, nsec, guest_tsc} to PTP and PTP should plug them into kernel's realtime clock roughly like this:

sec/nsec + (rdtsc() - guest_tsc) * tsc_freq

Quoted 7 lines

Adding delay to guest_tsc and sec/nsec cannot improve precision. (And will likely degrade it as kvmclock's frequency is incorrect.)

Quoted 1 line

We want the highest precision as possible.

I agree, which is why we don't want to lose precision in the delay guesswork because of gettime64().

Sorry the clock difference is 10ns now. So the guest clock is off by _10 ns_ of the host clock.

You are suggesting to use getcrosststamp instead, to drop the (rdtsc() - guest_tsc) part ?

Please be more verbose.

#23 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

2017-01-16 15:39-0200, Marcelo Tosatti:

Quoted 90 lines · Radim Krcmar
Quoted 89 lines

2017-01-16 15:08-0200, Marcelo Tosatti:

Quoted 65 lines · Radim Krcmar
Quoted 64 lines

2017-01-16 17:26+0100, Radim Krcmar:

Quoted 63 lines

2017-01-13 15:40-0200, Marcelo Tosatti:

Quoted 61 lines · Radim Krcmar
Quoted 41 lines

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 26 lines
(quoted diff) +26 -0
+		version = pvclock_read_begin(src);
+
+		ret = kvm_hypercall2(KVM_HC_CLOCK_OFFSET,
+				     clock_off_gpa,
+				     KVM_CLOCK_OFFSET_WALLCLOCK);
+		if (ret != 0) {
+			pr_err("clock offset hypercall ret %lu\n", ret);
+			spin_unlock(&kvm_ptp_lock);
+			preempt_enable_notrace();
+			return -EOPNOTSUPP;
+		}
+
+		tspec.tv_sec = clock_off.sec;
+		tspec.tv_nsec = clock_off.nsec;
+
+		delta = rdtsc_ordered() - clock_off.tsc;
+
+		offset = pvclock_scale_delta(delta, src->tsc_to_system_mul,
+					     src->tsc_shift);
+
+	} while (pvclock_read_retry(src, version));
+
+	preempt_enable_notrace();
+
+	tspec.tv_nsec = tspec.tv_nsec + offset;
+

Review

Quoted 10 lines
(quoted diff) +10 -0
+	spin_unlock(&kvm_ptp_lock);
+
+	if (tspec.tv_nsec >= NSEC_PER_SEC) {
+		u64 secs = tspec.tv_nsec;
+
+		tspec.tv_nsec = do_div(secs, NSEC_PER_SEC);
+		tspec.tv_sec += secs;
+	}
+
+	memcpy(ts, &tspec, sizeof(struct timespec64));

But the whole idea is of improving the time by reading tsc a bit later is just weird ... why is it better to provide

Quoted 7 lines

tsc + x, time + tsc_delta_to_time(x)

than just

tsc, time

?

Because you want to calculate the value of the host realtime clock at the moment of ptp_kvm_gettime.

We do:

1. kvm_hypercall.
2. get {sec, nsec, guest_tsc}.
3. kvm_hypercall returns.
4. delay = rdtsc() - guest_tsc.

Where delay is the delta (measured with the TSC) between points 2 and 4.

Quoted 8 lines
Quoted 5 lines
Quoted 1 line

I see now ... the PTP interface is just not good for our purposes.

There is getcrosststamp() callback in PTP, which seems to be exactly what we want when pairing with TSC, so the pvclock delay fixup can be dropped when using it.

What pvclock delay fixup you refer to? The "rdtsc() - clock_offset.tsc" part?

Yes.

Quoted 2 lines

You can't drop it, because if you do then your "host realtime
clock read" will be behind by "rdtsc() - clock_offset.tsc" TSC cycles.

The TSC read will be some cycles old when the hypercall ends, but that doesn't matter, because we will pass {sec, nsec, guest_tsc} to PTP and PTP should plug them into kernel's realtime clock roughly like this:

sec/nsec + (rdtsc() - guest_tsc) * tsc_freq

Adding delay to guest_tsc and sec/nsec cannot improve precision. (And will likely degrade it as kvmclock's frequency is incorrect.)

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

We want the highest precision as possible.

I agree, which is why we don't want to lose precision in the delay guesswork because of gettime64().

Sorry the clock difference is 10ns now. So the guest clock is off by _10 ns_ of the host clock.

That is pretty good.

Quoted 2 lines

You are suggesting to use getcrosststamp instead, to drop the (rdtsc() - guest_tsc) part ?

Yes, it results in simpler code, doesn't create dependency on the dreaded kvmclock, and is the best we can currently do wrt. precision.

Thanks.

author
#24 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 4 lines · Radim Krcmar
Quoted 2 lines

Sorry the clock difference is 10ns now. So the guest clock is off by _10 ns_ of the host clock.

That is pretty good.

Yes.

Quoted 5 lines
Quoted 2 lines

You are suggesting to use getcrosststamp instead, to drop the (rdtsc() - guest_tsc) part ?

Yes, it results in simpler code, doesn't create dependency on the dreaded kvmclock, and is the best we can currently do wrt. precision.

Sorry, unless i am misunderstanding how this works, it'll get the guest clock 2us behind, which is something not wanted.

Miroslav, if ->gettime64 returns the host realtime at 2us in the past, this means Chrony will sync the guest clock to

host realtime - 2us

Is that correct?

author
#25 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 13 lines · Marcelo Tosatti
Quoted 4 lines · Radim Krcmar
Quoted 2 lines

Sorry the clock difference is 10ns now. So the guest clock is off by _10 ns_ of the host clock.

That is pretty good.

Yes.

Quoted 5 lines
Quoted 2 lines

You are suggesting to use getcrosststamp instead, to drop the (rdtsc() - guest_tsc) part ?

Yes, it results in simpler code, doesn't create dependency on the dreaded kvmclock, and is the best we can currently do wrt. precision.

Even if the PHC sync algorithm manages to detect that the clock read is incorrect, consider the following:

Variability in the VM-entry code path, such as cache effects and interrupts would cause certain readings to be longer then the average (assuming an average
where cache is hot).

Using the TSC removes this variability, which can be large in case of non realtime guests, where you do:

1. kvm_hypercall.
2. read host realtime clock.
3. schedule out qemu-kvm vcpu.
4. schedule in qemu-kvm vcpu.

So using the delta between read host realtime and ->gettime64 increases precision and decreases variability.

Quoted 10 lines

Sorry, unless i am misunderstanding how this works, it'll get the guest clock 2us behind, which is something not wanted.

Miroslav, if ->gettime64 returns the host realtime at 2us in the past, this means Chrony will sync the guest clock to

host realtime - 2us

Is that correct?

author
#26 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 25 lines · Marcelo Tosatti
Quoted 13 lines · Marcelo Tosatti
Quoted 4 lines · Radim Krcmar
Quoted 2 lines

Sorry the clock difference is 10ns now. So the guest clock is off by _10 ns_ of the host clock.

That is pretty good.

Yes.

Quoted 5 lines
Quoted 2 lines

You are suggesting to use getcrosststamp instead, to drop the (rdtsc() - guest_tsc) part ?

Yes, it results in simpler code, doesn't create dependency on the dreaded kvmclock, and is the best we can currently do wrt. precision.

Even if the PHC sync algorithm manages to detect that the clock read is incorrect, consider the following:

Variability in the VM-entry code path, such as cache effects and interrupts would cause certain readings to be longer then the average (assuming an average
where cache is hot).

Using the TSC removes this variability, which can be large in case of non realtime guests, where you do:

Quoted 17 lines

1. kvm_hypercall.
2. read host realtime clock.
3. schedule out qemu-kvm vcpu.
4. schedule in qemu-kvm vcpu.

So using the delta between read host realtime and ->gettime64 increases precision and decreases variability.

Quoted 9 lines

Sorry, unless i am misunderstanding how this works, it'll get the guest clock 2us behind, which is something not wanted.

Miroslav, if ->gettime64 returns the host realtime at 2us in the past, this means Chrony will sync the guest clock to

host realtime - 2us

Is that correct?

Drop the offset correction and the following happens:

Clock offset seems to vary between negative hundreds of ns:

210 Number of sources = 1
MS Name/IP address Stratum Poll Reach LastRx Last sample
===============================================================================
#* PHC0 0 3 377 11 -131ns[ -309ns] +/-
3ns

And positive:

210 Number of sources = 1
MS Name/IP address Stratum Poll Reach LastRx Last sample
===============================================================================
#* PHC0 0 3 377 4 +79ns[ +155ns] +/-
3ns

#27 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 11 lines · Marcelo Tosatti
Quoted 10 lines · Marcelo Tosatti
Quoted 9 lines · Marcelo Tosatti

Sorry, unless i am misunderstanding how this works, it'll get the guest clock 2us behind, which is something not wanted.

Miroslav, if ->gettime64 returns the host realtime at 2us in the past, this means Chrony will sync the guest clock to

host realtime - 2us

Is that correct?

Probably. It depends on the error of both host and guest timestamps. If the error is the same on both sides, it will cancel out. An occasional spike in the delay shouldn't be a problem as the reading will be filtered out, but for best accuracy it's necessary that the host's timestamp is taken in the middle between the guest's timestamps.

Users of the PTP_SYS_OFFSET ioctl assume that (ts[0]+ts[2])/2 corresponds to ts[1], (ts[2]+ts[4])/2 corresponds to ts[3], and so on.

                    ts[1]     ts[3]
Host time    ---------+---------+........
                      |         |
                      |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]
Signature · 1 line
Miroslav Lichvar
author
#28 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 11 lines · Miroslav Lichvar
Quoted 10 lines · Marcelo Tosatti
Quoted 9 lines · Marcelo Tosatti
Quoted 8 lines · Marcelo Tosatti

Sorry, unless i am misunderstanding how this works, it'll get the guest clock 2us behind, which is something not wanted.

Miroslav, if ->gettime64 returns the host realtime at 2us in the past, this means Chrony will sync the guest clock to

host realtime - 2us

Quoted 8 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

Is that correct?

Probably. It depends on the error of both host and guest timestamps. If the error is the same on both sides, it will cancel out. An occasional spike in the delay shouldn't be a problem as the reading will be filtered out, but for best accuracy it's necessary that the host's timestamp is taken in the middle between the guest's timestamps.

The problem is that spikes can be far from occasional: it depends on activity of the host CPU and interrupts. Whose delay can be "intermittent": as long as interrupts are being sent to the host CPU, for example, the delay will be high (which can last minutes).

The TSC reading in the guest KVM PTP driver corrects for that delay.

Quoted 12 lines

Users of the PTP_SYS_OFFSET ioctl assume that (ts[0]+ts[2])/2 corresponds to ts[1], (ts[2]+ts[4])/2 corresponds to ts[3], and so on.

                    ts[1]     ts[3]
Host time    ---------+---------+........
                      |         |
                      |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]
Signature · 1 line
Miroslav Lichvar
#29 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

2017-01-17 09:30-0200, Marcelo Tosatti:

Quoted 21 lines · Miroslav Lichvar
Quoted 11 lines · Marcelo Tosatti
Quoted 10 lines · Marcelo Tosatti
Quoted 9 lines · Marcelo Tosatti
Quoted 8 lines

Sorry, unless i am misunderstanding how this works, it'll get the guest clock 2us behind, which is something not wanted.

Miroslav, if ->gettime64 returns the host realtime at 2us in the past, this means Chrony will sync the guest clock to

host realtime - 2us

Quoted 8 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

Is that correct?

Probably. It depends on the error of both host and guest timestamps. If the error is the same on both sides, it will cancel out. An occasional spike in the delay shouldn't be a problem as the reading will be filtered out, but for best accuracy it's necessary that the host's timestamp is taken in the middle between the guest's timestamps.

Quoted 16 lines

The problem is that spikes can be far from occasional: it depends on activity of the host CPU and interrupts. Whose delay can be "intermittent": as long as interrupts are being sent to the host CPU, for example, the delay will be high (which can last minutes).

The TSC reading in the guest KVM PTP driver corrects for that delay.

Quoted 9 lines

Users of the PTP_SYS_OFFSET ioctl assume that (ts[0]+ts[2])/2 corresponds to ts[1], (ts[2]+ts[4])/2 corresponds to ts[3], and so on.

                    ts[1]     ts[3]
Host time    ---------+---------+........
                      |         |
                      |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]

KVM PTP delay moves host ts[i] to be close to guest ts[i+1] and makes the offset very consistent, so the graph would look like:

                        ts[1]     ts[3]
Host time    -------------+---------+........
                          |         |
                          |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]

which doesn't sound good if users assume that the host reading is in the middle -- the guest time would be ahead of the host time.

I'm wondering why is the PTP precision around 10ns, when the hypercall takes around 2-3k cycles. Have you measured the guest<->host offset by getting the output of the hypercall, i.e.
{host_sec @ tsc, host_nsec @ tsc, tsc}
and comparing it with guest time computed from the same tsc, i.e.
{guest_sec @ tsc, guest_nsec @ tsc}
?

Thanks.

author
#30 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 60 lines · Radim Krcmar

2017-01-17 09:30-0200, Marcelo Tosatti:

Quoted 21 lines · Miroslav Lichvar
Quoted 11 lines · Marcelo Tosatti
Quoted 10 lines · Marcelo Tosatti
Quoted 9 lines · Marcelo Tosatti
Quoted 8 lines

Sorry, unless i am misunderstanding how this works, it'll get the guest clock 2us behind, which is something not wanted.

Miroslav, if ->gettime64 returns the host realtime at 2us in the past, this means Chrony will sync the guest clock to

host realtime - 2us

Quoted 8 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

Is that correct?

Probably. It depends on the error of both host and guest timestamps. If the error is the same on both sides, it will cancel out. An occasional spike in the delay shouldn't be a problem as the reading will be filtered out, but for best accuracy it's necessary that the host's timestamp is taken in the middle between the guest's timestamps.

Quoted 16 lines

The problem is that spikes can be far from occasional: it depends on activity of the host CPU and interrupts. Whose delay can be "intermittent": as long as interrupts are being sent to the host CPU, for example, the delay will be high (which can last minutes).

The TSC reading in the guest KVM PTP driver corrects for that delay.

Quoted 9 lines

Users of the PTP_SYS_OFFSET ioctl assume that (ts[0]+ts[2])/2 corresponds to ts[1], (ts[2]+ts[4])/2 corresponds to ts[3], and so on.

                    ts[1]     ts[3]
Host time    ---------+---------+........
                      |         |
                      |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]

KVM PTP delay moves host ts[i] to be close to guest ts[i+1] and makes the offset very consistent, so the graph would look like:

                        ts[1]     ts[3]
Host time    -------------+---------+........
                          |         |
                          |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]

which doesn't sound good if users assume that the host reading is in the middle -- the guest time would be ahead of the host time.

I'm wondering why is the PTP precision around 10ns, when the hypercall takes around 2-3k cycles. Have you measured the guest<->host offset by getting the output of the hypercall, i.e.
{host_sec @ tsc, host_nsec @ tsc, tsc}
and comparing it with guest time computed from the same tsc, i.e.
{guest_sec @ tsc, guest_nsec @ tsc}
?

Quoted 1 line

Thanks.

Testcase: run a guest and a loop sending SIGUSR1 to vcpu0 (emulating intense interrupts). Follows results:

Without TSC delta calculation:
=============================

#* PHC0                          0   3   377     2    -99ns[ +206ns] +/-  116ns
#* PHC0                          0   3   377     8   +202ns[ +249ns] +/-  111ns
#* PHC0                          0   3   377     8   -213ns[ +683ns] +/-   88ns
#* PHC0                          0   3   377     6    +77ns[ +319ns] +/-   56ns
#* PHC0                          0   3   377     4   -771ns[-1029ns] +/-   93ns
#* PHC0                          0   3   377    10    -49ns[  -58ns] +/-  121ns
#* PHC0                          0   3   377     9   +562ns[ +703ns] +/-  107ns
#* PHC0                          0   3   377     6     -2ns[   -3ns] +/-   94ns
#* PHC0                          0   3   377     4   +451ns[ +494ns] +/-  138ns
#* PHC0                          0   3   377    11    -67ns[  -74ns] +/-  113ns
#* PHC0                          0   3   377     8   +244ns[ +264ns] +/-  119ns
#* PHC0                          0   3   377     7   -696ns[ -890ns] +/-   89ns
#* PHC0                          0   3   377     4   +468ns[ +560ns] +/-  110ns
#* PHC0                          0   3   377    11   -310ns[ -430ns] +/-   72ns
#* PHC0                          0   3   377     9   +189ns[ +298ns] +/-   54ns
#* PHC0                          0   3   377     7   +594ns[ +473ns] +/-   96ns
#* PHC0                          0   3   377     5   +151ns[ +280ns] +/-   71ns
#* PHC0                          0   3   377    10   -590ns[ -696ns] +/-   94ns
#* PHC0                          0   3   377     8   +415ns[ +526ns] +/-   74ns
#* PHC0                          0   3   377     6  +1381ns[+1469ns] +/-  101ns
#* PHC0                          0   3   377     4   +571ns[+1304ns] +/-   54ns
#* PHC0                          0   3   377     8     -5ns[  +71ns] +/-  139ns
#* PHC0                          0   3   377     7   -247ns[ -502ns] +/-   69ns
#* PHC0                          0   3   377     5   -283ns[ +879ns] +/-   73ns
#* PHC0                          0   3   377     3   +148ns[ -109ns] +/-   61ns

With TSC delta calculation:
============================

#* PHC0                          0   3   377     7   +379ns[ +432ns] +/-   53ns
#* PHC0                          0   3   377     9   +106ns[ +420ns] +/-   42ns
#* PHC0                          0   3   377     7    -58ns[ -136ns] +/-   62ns
#* PHC0                          0   3   377    12    +93ns[  -38ns] +/-   64ns
#* PHC0                          0   3   377     8    +84ns[ +107ns] +/-   69ns
#* PHC0                          0   3   377     3    -76ns[ -103ns] +/-   52ns
#* PHC0                          0   3   377     7    +52ns[  +63ns] +/-   50ns
#* PHC0                          0   3   377    11    +29ns[  +31ns] +/-   70ns
#* PHC0                          0   3   377     7    -47ns[  -56ns] +/-   42ns
#* PHC0                          0   3   377    10    -35ns[  -42ns] +/-   33ns
#* PHC0                          0   3   377     7    -32ns[  -34ns] +/-   42ns
#* PHC0                          0   3   377    11   -172ns[ -173ns] +/-  118ns
#* PHC0                          0   3   377     6    +65ns[  +76ns] +/-   23ns
#* PHC0                          0   3   377     9    +18ns[  +23ns] +/-   37ns
#* PHC0                          0   3   377     6    +41ns[  -60ns] +/-   30ns
#* PHC0                          0   3   377    10    +39ns[ +183ns] +/-   42ns
#* PHC0                          0   3   377     6    +50ns[ +102ns] +/-   86ns
#* PHC0                          0   3   377    11    +50ns[  +75ns] +/-   52ns
#* PHC0                          0   3   377     6    +50ns[ +116ns] +/-  100ns
#* PHC0                          0   3   377    10    +46ns[  +65ns] +/-   79ns
#* PHC0                          0   3   377     7    -38ns[  -51ns] +/-   29ns
#* PHC0                          0   3   377    10    -11ns[  -12ns] +/-   32ns
#* PHC0                          0   3   377     7    -31ns[  -32ns] +/-   99ns
#* PHC0                          0   3   377    10   +222ns[ +238ns] +/-   58ns
#* PHC0                          0   3   377     6   +185ns[ +207ns] +/-   39ns
#* PHC0                          0   3   377    10   -392ns[ -394ns] +/-  118ns
#* PHC0                          0   3   377     6     -9ns[  -50ns] +/-   35ns
#* PHC0                          0   3   377    10   -346ns[ -355ns] +/-  111ns

Do you still want to drop it in favour of simplicity?

author
#31 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 128 lines · Marcelo Tosatti
Quoted 60 lines · Radim Krcmar

2017-01-17 09:30-0200, Marcelo Tosatti:

Quoted 21 lines · Miroslav Lichvar
Quoted 11 lines · Marcelo Tosatti
Quoted 10 lines · Marcelo Tosatti
Quoted 9 lines · Marcelo Tosatti
Quoted 8 lines

Sorry, unless i am misunderstanding how this works, it'll get the guest clock 2us behind, which is something not wanted.

Miroslav, if ->gettime64 returns the host realtime at 2us in the past, this means Chrony will sync the guest clock to

host realtime - 2us

Quoted 8 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line

Is that correct?

Probably. It depends on the error of both host and guest timestamps. If the error is the same on both sides, it will cancel out. An occasional spike in the delay shouldn't be a problem as the reading will be filtered out, but for best accuracy it's necessary that the host's timestamp is taken in the middle between the guest's timestamps.

Quoted 16 lines

The problem is that spikes can be far from occasional: it depends on activity of the host CPU and interrupts. Whose delay can be "intermittent": as long as interrupts are being sent to the host CPU, for example, the delay will be high (which can last minutes).

The TSC reading in the guest KVM PTP driver corrects for that delay.

Quoted 9 lines

Users of the PTP_SYS_OFFSET ioctl assume that (ts[0]+ts[2])/2 corresponds to ts[1], (ts[2]+ts[4])/2 corresponds to ts[3], and so on.

                    ts[1]     ts[3]
Host time    ---------+---------+........
                      |         |
                      |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]

KVM PTP delay moves host ts[i] to be close to guest ts[i+1] and makes the offset very consistent, so the graph would look like:

                        ts[1]     ts[3]
Host time    -------------+---------+........
                          |         |
                          |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]

which doesn't sound good if users assume that the host reading is in the middle -- the guest time would be ahead of the host time.

I'm wondering why is the PTP precision around 10ns, when the hypercall takes around 2-3k cycles. Have you measured the guest<->host offset by getting the output of the hypercall, i.e.
{host_sec @ tsc, host_nsec @ tsc, tsc}
and comparing it with guest time computed from the same tsc, i.e.
{guest_sec @ tsc, guest_nsec @ tsc}
?

Quoted 1 line

Thanks.

Testcase: run a guest and a loop sending SIGUSR1 to vcpu0 (emulating intense interrupts). Follows results:

Without TSC delta calculation:
=============================

#* PHC0                          0   3   377     2    -99ns[ +206ns] +/-  116ns
#* PHC0                          0   3   377     8   +202ns[ +249ns] +/-  111ns
#* PHC0                          0   3   377     8   -213ns[ +683ns] +/-   88ns
#* PHC0                          0   3   377     6    +77ns[ +319ns] +/-   56ns
#* PHC0                          0   3   377     4   -771ns[-1029ns] +/-   93ns
#* PHC0                          0   3   377    10    -49ns[  -58ns] +/-  121ns
#* PHC0                          0   3   377     9   +562ns[ +703ns] +/-  107ns
#* PHC0                          0   3   377     6     -2ns[   -3ns] +/-   94ns
#* PHC0                          0   3   377     4   +451ns[ +494ns] +/-  138ns
#* PHC0                          0   3   377    11    -67ns[  -74ns] +/-  113ns
#* PHC0                          0   3   377     8   +244ns[ +264ns] +/-  119ns
#* PHC0                          0   3   377     7   -696ns[ -890ns] +/-   89ns
#* PHC0                          0   3   377     4   +468ns[ +560ns] +/-  110ns
#* PHC0                          0   3   377    11   -310ns[ -430ns] +/-   72ns
#* PHC0                          0   3   377     9   +189ns[ +298ns] +/-   54ns
#* PHC0                          0   3   377     7   +594ns[ +473ns] +/-   96ns
#* PHC0                          0   3   377     5   +151ns[ +280ns] +/-   71ns
#* PHC0                          0   3   377    10   -590ns[ -696ns] +/-   94ns
#* PHC0                          0   3   377     8   +415ns[ +526ns] +/-   74ns
#* PHC0                          0   3   377     6  +1381ns[+1469ns] +/-  101ns
#* PHC0                          0   3   377     4   +571ns[+1304ns] +/-   54ns
#* PHC0                          0   3   377     8     -5ns[  +71ns] +/-  139ns
#* PHC0                          0   3   377     7   -247ns[ -502ns] +/-   69ns
#* PHC0                          0   3   377     5   -283ns[ +879ns] +/-   73ns
#* PHC0                          0   3   377     3   +148ns[ -109ns] +/-   61ns

With TSC delta calculation:
============================

#* PHC0                          0   3   377     7   +379ns[ +432ns] +/-   53ns
#* PHC0                          0   3   377     9   +106ns[ +420ns] +/-   42ns
#* PHC0                          0   3   377     7    -58ns[ -136ns] +/-   62ns
#* PHC0                          0   3   377    12    +93ns[  -38ns] +/-   64ns
#* PHC0                          0   3   377     8    +84ns[ +107ns] +/-   69ns
#* PHC0                          0   3   377     3    -76ns[ -103ns] +/-   52ns
#* PHC0                          0   3   377     7    +52ns[  +63ns] +/-   50ns
#* PHC0                          0   3   377    11    +29ns[  +31ns] +/-   70ns
#* PHC0                          0   3   377     7    -47ns[  -56ns] +/-   42ns
#* PHC0                          0   3   377    10    -35ns[  -42ns] +/-   33ns
#* PHC0                          0   3   377     7    -32ns[  -34ns] +/-   42ns
#* PHC0                          0   3   377    11   -172ns[ -173ns] +/-  118ns
#* PHC0                          0   3   377     6    +65ns[  +76ns] +/-   23ns
#* PHC0                          0   3   377     9    +18ns[  +23ns] +/-   37ns
#* PHC0                          0   3   377     6    +41ns[  -60ns] +/-   30ns
#* PHC0                          0   3   377    10    +39ns[ +183ns] +/-   42ns
#* PHC0                          0   3   377     6    +50ns[ +102ns] +/-   86ns
#* PHC0                          0   3   377    11    +50ns[  +75ns] +/-   52ns
#* PHC0                          0   3   377     6    +50ns[ +116ns] +/-  100ns
#* PHC0                          0   3   377    10    +46ns[  +65ns] +/-   79ns
#* PHC0                          0   3   377     7    -38ns[  -51ns] +/-   29ns
#* PHC0                          0   3   377    10    -11ns[  -12ns] +/-   32ns
#* PHC0                          0   3   377     7    -31ns[  -32ns] +/-   99ns
#* PHC0                          0   3   377    10   +222ns[ +238ns] +/-   58ns
#* PHC0                          0   3   377     6   +185ns[ +207ns] +/-   39ns
#* PHC0                          0   3   377    10   -392ns[ -394ns] +/-  118ns
#* PHC0                          0   3   377     6     -9ns[  -50ns] +/-   35ns
#* PHC0                          0   3   377    10   -346ns[ -355ns] +/-  111ns
Quoted 2 lines

Do you still want to drop it in favour of simplicity?

This is the output of "chronyc sources". See section "Time sources" of https://chrony.tuxfamily.org/doc/2.4/chronyc.html.

maintainer
#32 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 92 lines · Marcelo Tosatti
Quoted 91 lines · Marcelo Tosatti
Quoted 22 lines · Radim Krcmar

2017-01-17 09:30-0200, Marcelo Tosatti:

Quoted 10 lines · Miroslav Lichvar
Quoted 9 lines

Users of the PTP_SYS_OFFSET ioctl assume that (ts[0]+ts[2])/2 corresponds to ts[1], (ts[2]+ts[4])/2 corresponds to ts[3], and so on.

                    ts[1]     ts[3]
Host time    ---------+---------+........
                      |         |
                      |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]

KVM PTP delay moves host ts[i] to be close to guest ts[i+1] and makes the offset very consistent, so the graph would look like:

                        ts[1]     ts[3]
Host time    -------------+---------+........
                          |         |
                          |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]
Quoted 2 lines

which doesn't sound good if users assume that the host reading is in the middle -- the guest time would be ahead of the host time.

Testcase: run a guest and a loop sending SIGUSR1 to vcpu0 (emulating intense interrupts). Follows results:

Without TSC delta calculation:
=============================

#* PHC0                          0   3   377     2    -99ns[ +206ns] +/-  116ns
#* PHC0                          0   3   377     8   +202ns[ +249ns] +/-  111ns
#* PHC0                          0   3   377     8   -213ns[ +683ns] +/-   88ns
#* PHC0                          0   3   377     6    +77ns[ +319ns] +/-   56ns
#* PHC0                          0   3   377     4   -771ns[-1029ns] +/-   93ns
#* PHC0                          0   3   377    10    -49ns[  -58ns] +/-  121ns
#* PHC0                          0   3   377     9   +562ns[ +703ns] +/-  107ns
#* PHC0                          0   3   377     6     -2ns[   -3ns] +/-   94ns
#* PHC0                          0   3   377     4   +451ns[ +494ns] +/-  138ns
#* PHC0                          0   3   377    11    -67ns[  -74ns] +/-  113ns
#* PHC0                          0   3   377     8   +244ns[ +264ns] +/-  119ns
#* PHC0                          0   3   377     7   -696ns[ -890ns] +/-   89ns
#* PHC0                          0   3   377     4   +468ns[ +560ns] +/-  110ns
#* PHC0                          0   3   377    11   -310ns[ -430ns] +/-   72ns
#* PHC0                          0   3   377     9   +189ns[ +298ns] +/-   54ns
#* PHC0                          0   3   377     7   +594ns[ +473ns] +/-   96ns
#* PHC0                          0   3   377     5   +151ns[ +280ns] +/-   71ns
#* PHC0                          0   3   377    10   -590ns[ -696ns] +/-   94ns
#* PHC0                          0   3   377     8   +415ns[ +526ns] +/-   74ns
#* PHC0                          0   3   377     6  +1381ns[+1469ns] +/-  101ns
#* PHC0                          0   3   377     4   +571ns[+1304ns] +/-   54ns
#* PHC0                          0   3   377     8     -5ns[  +71ns] +/-  139ns
#* PHC0                          0   3   377     7   -247ns[ -502ns] +/-   69ns
#* PHC0                          0   3   377     5   -283ns[ +879ns] +/-   73ns
#* PHC0                          0   3   377     3   +148ns[ -109ns] +/-   61ns

With TSC delta calculation:
============================

#* PHC0                          0   3   377     7   +379ns[ +432ns] +/-   53ns
#* PHC0                          0   3   377     9   +106ns[ +420ns] +/-   42ns
#* PHC0                          0   3   377     7    -58ns[ -136ns] +/-   62ns
#* PHC0                          0   3   377    12    +93ns[  -38ns] +/-   64ns
#* PHC0                          0   3   377     8    +84ns[ +107ns] +/-   69ns
#* PHC0                          0   3   377     3    -76ns[ -103ns] +/-   52ns
#* PHC0                          0   3   377     7    +52ns[  +63ns] +/-   50ns
#* PHC0                          0   3   377    11    +29ns[  +31ns] +/-   70ns
#* PHC0                          0   3   377     7    -47ns[  -56ns] +/-   42ns
#* PHC0                          0   3   377    10    -35ns[  -42ns] +/-   33ns
#* PHC0                          0   3   377     7    -32ns[  -34ns] +/-   42ns
#* PHC0                          0   3   377    11   -172ns[ -173ns] +/-  118ns
#* PHC0                          0   3   377     6    +65ns[  +76ns] +/-   23ns
#* PHC0                          0   3   377     9    +18ns[  +23ns] +/-   37ns
#* PHC0                          0   3   377     6    +41ns[  -60ns] +/-   30ns
#* PHC0                          0   3   377    10    +39ns[ +183ns] +/-   42ns
#* PHC0                          0   3   377     6    +50ns[ +102ns] +/-   86ns
#* PHC0                          0   3   377    11    +50ns[  +75ns] +/-   52ns
#* PHC0                          0   3   377     6    +50ns[ +116ns] +/-  100ns
#* PHC0                          0   3   377    10    +46ns[  +65ns] +/-   79ns
#* PHC0                          0   3   377     7    -38ns[  -51ns] +/-   29ns
#* PHC0                          0   3   377    10    -11ns[  -12ns] +/-   32ns
#* PHC0                          0   3   377     7    -31ns[  -32ns] +/-   99ns
#* PHC0                          0   3   377    10   +222ns[ +238ns] +/-   58ns
#* PHC0                          0   3   377     6   +185ns[ +207ns] +/-   39ns
#* PHC0                          0   3   377    10   -392ns[ -394ns] +/-  118ns
#* PHC0                          0   3   377     6     -9ns[  -50ns] +/-   35ns
#* PHC0                          0   3   377    10   -346ns[ -355ns] +/-  111ns
Quoted 4 lines
Quoted 1 line

Do you still want to drop it in favour of simplicity?

This is the output of "chronyc sources". See section "Time sources" of https://chrony.tuxfamily.org/doc/2.4/chronyc.html.

It's just that it's not obvious why you get better results with biased host timestamps. What makes the biased host timestamp more precise?

I'd rather use PTP_SYS_OFFSET_PRECISE instead, but unfortunately chrony does not support it---but I would still prefer you to support PTP_SYS_OFFSET_PRECISE as well.

Paolo

#33 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 4 lines · Paolo Bonzini
Quoted 2 lines · Marcelo Tosatti
Quoted 2 lines

Testcase: run a guest and a loop sending SIGUSR1 to vcpu0 (emulating intense interrupts). Follows results:

Quoted 8 lines
Quoted 1 line
Quoted 1 line

Do you still want to drop it in favour of simplicity?

It's just that it's not obvious why you get better results with biased host timestamps. What makes the biased host timestamp more precise?

I'd rather use PTP_SYS_OFFSET_PRECISE instead, but unfortunately chrony does not support it---but I would still prefer you to support PTP_SYS_OFFSET_PRECISE as well.

Interesting. I wasn't aware that there is a new ioctl for measuring the HW-sys offset. Adding support to chrony shouldn't be difficult.

If I understand it correctly, PTP_SYS_OFFSET can be emulated on top of PTP_SYS_OFFSET_PRECISE simply by copying the sys_realtime and device fields to corresponding ts slots. The apparent delay will be zero, but that's ok if the conversion is really accurate.

I'm not sure if trying to do that in the opposite direction is a good idea. An application using PTP_SYS_OFFSET_PRECISE may assume the conversion is accurate and not include any delay/dispersion in an estimate of the maximum error, which is needed in NTP for instance.

If we know the host timestamp ts[1] is not in the middle between the guests timestamps ts[0] and ts[2], but rather closer to ts[2], why not simply shift ts[1] by (ts[2]-ts[0])/2 ?

Signature · 1 line
Miroslav Lichvar
maintainer
#34 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 14 lines · Miroslav Lichvar
Quoted 4 lines · Paolo Bonzini
Quoted 2 lines · Marcelo Tosatti
Quoted 2 lines

Testcase: run a guest and a loop sending SIGUSR1 to vcpu0 (emulating intense interrupts). Follows results:

Quoted 8 lines
Quoted 1 line
Quoted 1 line

Do you still want to drop it in favour of simplicity?

It's just that it's not obvious why you get better results with biased host timestamps. What makes the biased host timestamp more precise?

I'd rather use PTP_SYS_OFFSET_PRECISE instead, but unfortunately chrony does not support it---but I would still prefer you to support PTP_SYS_OFFSET_PRECISE as well.

Quoted 7 lines

Interesting. I wasn't aware that there is a new ioctl for measuring the HW-sys offset. Adding support to chrony shouldn't be difficult.

If I understand it correctly, PTP_SYS_OFFSET can be emulated on top of PTP_SYS_OFFSET_PRECISE simply by copying the sys_realtime and device fields to corresponding ts slots. The apparent delay will be zero, but that's ok if the conversion is really accurate.

Yes, for 1 sample only. Otherwise you'd have the same issue as in Marcelo's driver (the device aka guest timestamp from PTP_SYS_OFFSET_PRECISE would not be halfway between the system aka host timestamps), and your idea below could be applied.

Quoted 8 lines

I'm not sure if trying to do that in the opposite direction is a good idea. An application using PTP_SYS_OFFSET_PRECISE may assume the conversion is accurate and not include any delay/dispersion in an estimate of the maximum error, which is needed in NTP for instance.

If we know the host timestamp ts[1] is not in the middle between the guests timestamps ts[0] and ts[2], but rather closer to ts[2], why not simply shift ts[1] by (ts[2]-ts[0])/2 ?

Interesting idea! For this to work, KVM needs to implement getcrosstimestamp and ptp_chardev.c can then add an alternative implementation of PTP_SYS_OFFSET, based on precise cross timestamps.

Something like

                for (i = 0; i <= sysoff->n_samples; i++) {
			// ... call getcrosststamp ...
			sysns = ktime_to_ns(xtstamp.sys_realtime);
			if (i > 0) {
				devns = ktime_to_ns(xtstamp.device);
				devns -= (sysns - prev_sysns) / 2;
				devts = ns_to_timespec(devns);
				pct->sec = devts.tv_sec;
				pct->nsec = devts.tv_nsec;
	                        pct++;
			}
			systs = ns_to_timespec(sysns);
                        pct->sec = ts.tv_sec;
                        pct->nsec = ts.tv_nsec;
                        pct++;
			prev_sysns = sysns;
                }

Marcelo, can you give it a try?

Thanks,

Paolo

author
#35 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 66 lines · Paolo Bonzini
Quoted 14 lines · Miroslav Lichvar
Quoted 4 lines · Paolo Bonzini
Quoted 2 lines · Marcelo Tosatti
Quoted 2 lines

Testcase: run a guest and a loop sending SIGUSR1 to vcpu0 (emulating intense interrupts). Follows results:

Quoted 8 lines
Quoted 1 line
Quoted 1 line

Do you still want to drop it in favour of simplicity?

It's just that it's not obvious why you get better results with biased host timestamps. What makes the biased host timestamp more precise?

I'd rather use PTP_SYS_OFFSET_PRECISE instead, but unfortunately chrony does not support it---but I would still prefer you to support PTP_SYS_OFFSET_PRECISE as well.

Quoted 7 lines

Interesting. I wasn't aware that there is a new ioctl for measuring the HW-sys offset. Adding support to chrony shouldn't be difficult.

If I understand it correctly, PTP_SYS_OFFSET can be emulated on top of PTP_SYS_OFFSET_PRECISE simply by copying the sys_realtime and device fields to corresponding ts slots. The apparent delay will be zero, but that's ok if the conversion is really accurate.

Yes, for 1 sample only. Otherwise you'd have the same issue as in Marcelo's driver (the device aka guest timestamp from PTP_SYS_OFFSET_PRECISE would not be halfway between the system aka host timestamps), and your idea below could be applied.

Quoted 8 lines

I'm not sure if trying to do that in the opposite direction is a good idea. An application using PTP_SYS_OFFSET_PRECISE may assume the conversion is accurate and not include any delay/dispersion in an estimate of the maximum error, which is needed in NTP for instance.

If we know the host timestamp ts[1] is not in the middle between the guests timestamps ts[0] and ts[2], but rather closer to ts[2], why not simply shift ts[1] by (ts[2]-ts[0])/2 ?

Interesting idea! For this to work, KVM needs to implement getcrosstimestamp and ptp_chardev.c can then add an alternative implementation of PTP_SYS_OFFSET, based on precise cross timestamps.

Something like

                for (i = 0; i <= sysoff->n_samples; i++) {
			// ... call getcrosststamp ...
			sysns = ktime_to_ns(xtstamp.sys_realtime);
			if (i > 0) {
				devns = ktime_to_ns(xtstamp.device);
				devns -= (sysns - prev_sysns) / 2;
				devts = ns_to_timespec(devns);
				pct->sec = devts.tv_sec;
				pct->nsec = devts.tv_nsec;
	                        pct++;
			}
			systs = ns_to_timespec(sysns);
                        pct->sec = ts.tv_sec;
                        pct->nsec = ts.tv_nsec;
                        pct++;
			prev_sysns = sysns;
                }
Quoted 1 line

Marcelo, can you give it a try?

Can convert fine, but problem is the simultaneous read of host and guest clocks.

Quoted 3 lines

Thanks,

Paolo

It seems to me anything else other than using a single TSC read (for both host and guest clocks) is a poor PTP_SYS_OFFSET_PRECISE implementation (because it would claim to be similar to ART, where the timestamps are simultaneous), but not be.

#36 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

2017-01-18 12:50-0200, Marcelo Tosatti:

Quoted 58 lines · Paolo Bonzini
Quoted 57 lines · Miroslav Lichvar
Quoted 14 lines · Paolo Bonzini
Quoted 12 lines · Marcelo Tosatti
Quoted 4 lines
Quoted 4 lines

Testcase: run a guest and a loop sending SIGUSR1 to vcpu0 (emulating intense interrupts). Follows results:

Do you still want to drop it in favour of simplicity?

It's just that it's not obvious why you get better results with biased host timestamps. What makes the biased host timestamp more precise?

I'd rather use PTP_SYS_OFFSET_PRECISE instead, but unfortunately chrony does not support it---but I would still prefer you to support PTP_SYS_OFFSET_PRECISE as well.

Quoted 7 lines

Interesting. I wasn't aware that there is a new ioctl for measuring the HW-sys offset. Adding support to chrony shouldn't be difficult.

If I understand it correctly, PTP_SYS_OFFSET can be emulated on top of PTP_SYS_OFFSET_PRECISE simply by copying the sys_realtime and device fields to corresponding ts slots. The apparent delay will be zero, but that's ok if the conversion is really accurate.

Yes, for 1 sample only. Otherwise you'd have the same issue as in Marcelo's driver (the device aka guest timestamp from PTP_SYS_OFFSET_PRECISE would not be halfway between the system aka host timestamps), and your idea below could be applied.

Quoted 8 lines

I'm not sure if trying to do that in the opposite direction is a good idea. An application using PTP_SYS_OFFSET_PRECISE may assume the conversion is accurate and not include any delay/dispersion in an estimate of the maximum error, which is needed in NTP for instance.

If we know the host timestamp ts[1] is not in the middle between the guests timestamps ts[0] and ts[2], but rather closer to ts[2], why not simply shift ts[1] by (ts[2]-ts[0])/2 ?

Interesting idea! For this to work, KVM needs to implement getcrosstimestamp and ptp_chardev.c can then add an alternative implementation of PTP_SYS_OFFSET, based on precise cross timestamps.

Something like

                for (i = 0; i <= sysoff->n_samples; i++) {
			// ... call getcrosststamp ...
			sysns = ktime_to_ns(xtstamp.sys_realtime);
			if (i > 0) {
				devns = ktime_to_ns(xtstamp.device);
				devns -= (sysns - prev_sysns) / 2;
				devts = ns_to_timespec(devns);
				pct->sec = devts.tv_sec;
				pct->nsec = devts.tv_nsec;
Quoted 8 lines
Quoted 8 lines
	                        pct++;
			}
			systs = ns_to_timespec(sysns);
                        pct->sec = ts.tv_sec;
                        pct->nsec = ts.tv_nsec;
                        pct++;
			prev_sysns = sysns;
                }

Nice. PTP_SYS_OFFSET seems to be mandatory, so this hunk could be upstreamable as a fallback for PTP devices that have getcrosststamp and not gettime64. KVM PTP would be the only driver using it so far.

Quoted 5 lines
Quoted 1 line

Marcelo, can you give it a try?

Can convert fine, but problem is the simultaneous read of host and guest clocks.

Quoted 8 lines
Quoted 3 lines

Thanks,

Paolo

It seems to me anything else other than using a single TSC read (for both host and guest clocks) is a poor PTP_SYS_OFFSET_PRECISE implementation (because it would claim to be similar to ART, where the timestamps are simultaneous), but not be.

Yes, the guest should use the TSC returned by the hypercall to compute the corresponding guest time, which will allow us to know the host<->guest offset.

Is this impossible to do with the current API?

Thanks.

maintainer
#37 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

In reply to Marcelo Tosatti
Quoted 24 lines · Marcelo Tosatti
Quoted 24 lines

Interesting idea! For this to work, KVM needs to implement getcrosstimestamp and ptp_chardev.c can then add an alternative implementation of PTP_SYS_OFFSET, based on precise cross timestamps.

Something like

                for (i = 0; i <= sysoff->n_samples; i++) {
			// ... call getcrosststamp ...
			sysns = ktime_to_ns(xtstamp.sys_realtime);
			if (i > 0) {
				devns = ktime_to_ns(xtstamp.device);
				devns -= (sysns - prev_sysns) / 2;
				devts = ns_to_timespec(devns);
				pct->sec = devts.tv_sec;
				pct->nsec = devts.tv_nsec;
	                        pct++;
			}
			systs = ns_to_timespec(sysns);
                        pct->sec = ts.tv_sec;
                        pct->nsec = ts.tv_nsec;
                        pct++;
			prev_sysns = sysns;
                }
Quoted 4 lines
Quoted 1 line

Marcelo, can you give it a try?

Can convert fine, but problem is the simultaneous read of host and guest clocks.

Could the TSC from the hypercall be applied to kvmclock to do this? My understanding is that get_device_system_crosststamp (which is used in the sole in-tree implementation of getcrosststamp) already contains all the logic to do that.

Paolo

author
#38 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 32 lines · Paolo Bonzini
Quoted 24 lines · Marcelo Tosatti
Quoted 24 lines

Interesting idea! For this to work, KVM needs to implement getcrosstimestamp and ptp_chardev.c can then add an alternative implementation of PTP_SYS_OFFSET, based on precise cross timestamps.

Something like

                for (i = 0; i <= sysoff->n_samples; i++) {
			// ... call getcrosststamp ...
			sysns = ktime_to_ns(xtstamp.sys_realtime);
			if (i > 0) {
				devns = ktime_to_ns(xtstamp.device);
				devns -= (sysns - prev_sysns) / 2;
				devts = ns_to_timespec(devns);
				pct->sec = devts.tv_sec;
				pct->nsec = devts.tv_nsec;
	                        pct++;
			}
			systs = ns_to_timespec(sysns);
                        pct->sec = ts.tv_sec;
                        pct->nsec = ts.tv_nsec;
                        pct++;
			prev_sysns = sysns;
                }
Quoted 4 lines
Quoted 1 line

Marcelo, can you give it a try?

Can convert fine, but problem is the simultaneous read of host and guest clocks.

Quoted 6 lines

Could the TSC from the hypercall be applied to kvmclock to do this? My understanding is that get_device_system_crosststamp (which is used in the sole in-tree implementation of getcrosststamp) already contains all the logic to do that.

Paolo

Yeah feed that TSC to pvclock_clocksource_read.

Cool will take a look at that function thanks.

author
#39 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

In reply to Paolo Bonzini
Quoted 100 lines · Paolo Bonzini
Quoted 92 lines · Marcelo Tosatti
Quoted 91 lines · Marcelo Tosatti
Quoted 22 lines · Radim Krcmar

2017-01-17 09:30-0200, Marcelo Tosatti:

Quoted 10 lines · Miroslav Lichvar
Quoted 9 lines

Users of the PTP_SYS_OFFSET ioctl assume that (ts[0]+ts[2])/2 corresponds to ts[1], (ts[2]+ts[4])/2 corresponds to ts[3], and so on.

                    ts[1]     ts[3]
Host time    ---------+---------+........
                      |         |
                      |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]

KVM PTP delay moves host ts[i] to be close to guest ts[i+1] and makes the offset very consistent, so the graph would look like:

                        ts[1]     ts[3]
Host time    -------------+---------+........
                          |         |
                          |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]
Quoted 2 lines

which doesn't sound good if users assume that the host reading is in the middle -- the guest time would be ahead of the host time.

Testcase: run a guest and a loop sending SIGUSR1 to vcpu0 (emulating intense interrupts). Follows results:

Without TSC delta calculation:
=============================

#* PHC0                          0   3   377     2    -99ns[ +206ns] +/-  116ns
#* PHC0                          0   3   377     8   +202ns[ +249ns] +/-  111ns
#* PHC0                          0   3   377     8   -213ns[ +683ns] +/-   88ns
#* PHC0                          0   3   377     6    +77ns[ +319ns] +/-   56ns
#* PHC0                          0   3   377     4   -771ns[-1029ns] +/-   93ns
#* PHC0                          0   3   377    10    -49ns[  -58ns] +/-  121ns
#* PHC0                          0   3   377     9   +562ns[ +703ns] +/-  107ns
#* PHC0                          0   3   377     6     -2ns[   -3ns] +/-   94ns
#* PHC0                          0   3   377     4   +451ns[ +494ns] +/-  138ns
#* PHC0                          0   3   377    11    -67ns[  -74ns] +/-  113ns
#* PHC0                          0   3   377     8   +244ns[ +264ns] +/-  119ns
#* PHC0                          0   3   377     7   -696ns[ -890ns] +/-   89ns
#* PHC0                          0   3   377     4   +468ns[ +560ns] +/-  110ns
#* PHC0                          0   3   377    11   -310ns[ -430ns] +/-   72ns
#* PHC0                          0   3   377     9   +189ns[ +298ns] +/-   54ns
#* PHC0                          0   3   377     7   +594ns[ +473ns] +/-   96ns
#* PHC0                          0   3   377     5   +151ns[ +280ns] +/-   71ns
#* PHC0                          0   3   377    10   -590ns[ -696ns] +/-   94ns
#* PHC0                          0   3   377     8   +415ns[ +526ns] +/-   74ns
#* PHC0                          0   3   377     6  +1381ns[+1469ns] +/-  101ns
#* PHC0                          0   3   377     4   +571ns[+1304ns] +/-   54ns
#* PHC0                          0   3   377     8     -5ns[  +71ns] +/-  139ns
#* PHC0                          0   3   377     7   -247ns[ -502ns] +/-   69ns
#* PHC0                          0   3   377     5   -283ns[ +879ns] +/-   73ns
#* PHC0                          0   3   377     3   +148ns[ -109ns] +/-   61ns

With TSC delta calculation:
============================

#* PHC0                          0   3   377     7   +379ns[ +432ns] +/-   53ns
#* PHC0                          0   3   377     9   +106ns[ +420ns] +/-   42ns
#* PHC0                          0   3   377     7    -58ns[ -136ns] +/-   62ns
#* PHC0                          0   3   377    12    +93ns[  -38ns] +/-   64ns
#* PHC0                          0   3   377     8    +84ns[ +107ns] +/-   69ns
#* PHC0                          0   3   377     3    -76ns[ -103ns] +/-   52ns
#* PHC0                          0   3   377     7    +52ns[  +63ns] +/-   50ns
#* PHC0                          0   3   377    11    +29ns[  +31ns] +/-   70ns
#* PHC0                          0   3   377     7    -47ns[  -56ns] +/-   42ns
#* PHC0                          0   3   377    10    -35ns[  -42ns] +/-   33ns
#* PHC0                          0   3   377     7    -32ns[  -34ns] +/-   42ns
#* PHC0                          0   3   377    11   -172ns[ -173ns] +/-  118ns
#* PHC0                          0   3   377     6    +65ns[  +76ns] +/-   23ns
#* PHC0                          0   3   377     9    +18ns[  +23ns] +/-   37ns
#* PHC0                          0   3   377     6    +41ns[  -60ns] +/-   30ns
#* PHC0                          0   3   377    10    +39ns[ +183ns] +/-   42ns
#* PHC0                          0   3   377     6    +50ns[ +102ns] +/-   86ns
#* PHC0                          0   3   377    11    +50ns[  +75ns] +/-   52ns
#* PHC0                          0   3   377     6    +50ns[ +116ns] +/-  100ns
#* PHC0                          0   3   377    10    +46ns[  +65ns] +/-   79ns
#* PHC0                          0   3   377     7    -38ns[  -51ns] +/-   29ns
#* PHC0                          0   3   377    10    -11ns[  -12ns] +/-   32ns
#* PHC0                          0   3   377     7    -31ns[  -32ns] +/-   99ns
#* PHC0                          0   3   377    10   +222ns[ +238ns] +/-   58ns
#* PHC0                          0   3   377     6   +185ns[ +207ns] +/-   39ns
#* PHC0                          0   3   377    10   -392ns[ -394ns] +/-  118ns
#* PHC0                          0   3   377     6     -9ns[  -50ns] +/-   35ns
#* PHC0                          0   3   377    10   -346ns[ -355ns] +/-  111ns
Quoted 4 lines
Quoted 1 line

Do you still want to drop it in favour of simplicity?

This is the output of "chronyc sources". See section "Time sources" of https://chrony.tuxfamily.org/doc/2.4/chronyc.html.

Quoted 6 lines

It's just that it's not obvious why you get better results with biased host timestamps. What makes the biased host timestamp more precise?

I'd rather use PTP_SYS_OFFSET_PRECISE instead, but unfortunately chrony does not support it---but I would still prefer you to support PTP_SYS_OFFSET_PRECISE as well.

A single TSC read could be used to implement the PRECISE ioctl, but if a timer interrupt takes place on either the host or the guest, and that timer interrupt "adds" the TSC delta to xtime.nsec/xtime.sec, then that single TSC read cannot be used.

So you would have to stop timer interrupts (in guest and host) for the duration of the PRECISE ioctl in the guest to avoid that situation, which seems a bit overkill to me.

Any other ideas?

author
#40 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 113 lines · Marcelo Tosatti
Quoted 100 lines · Paolo Bonzini
Quoted 92 lines · Marcelo Tosatti
Quoted 91 lines · Marcelo Tosatti
Quoted 22 lines · Radim Krcmar

2017-01-17 09:30-0200, Marcelo Tosatti:

Quoted 10 lines · Miroslav Lichvar
Quoted 9 lines

Users of the PTP_SYS_OFFSET ioctl assume that (ts[0]+ts[2])/2 corresponds to ts[1], (ts[2]+ts[4])/2 corresponds to ts[3], and so on.

                    ts[1]     ts[3]
Host time    ---------+---------+........
                      |         |
                      |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]

KVM PTP delay moves host ts[i] to be close to guest ts[i+1] and makes the offset very consistent, so the graph would look like:

                        ts[1]     ts[3]
Host time    -------------+---------+........
                          |         |
                          |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]
Quoted 2 lines

which doesn't sound good if users assume that the host reading is in the middle -- the guest time would be ahead of the host time.

Testcase: run a guest and a loop sending SIGUSR1 to vcpu0 (emulating intense interrupts). Follows results:

Without TSC delta calculation:
=============================

#* PHC0                          0   3   377     2    -99ns[ +206ns] +/-  116ns
#* PHC0                          0   3   377     8   +202ns[ +249ns] +/-  111ns
#* PHC0                          0   3   377     8   -213ns[ +683ns] +/-   88ns
#* PHC0                          0   3   377     6    +77ns[ +319ns] +/-   56ns
#* PHC0                          0   3   377     4   -771ns[-1029ns] +/-   93ns
#* PHC0                          0   3   377    10    -49ns[  -58ns] +/-  121ns
#* PHC0                          0   3   377     9   +562ns[ +703ns] +/-  107ns
#* PHC0                          0   3   377     6     -2ns[   -3ns] +/-   94ns
#* PHC0                          0   3   377     4   +451ns[ +494ns] +/-  138ns
#* PHC0                          0   3   377    11    -67ns[  -74ns] +/-  113ns
#* PHC0                          0   3   377     8   +244ns[ +264ns] +/-  119ns
#* PHC0                          0   3   377     7   -696ns[ -890ns] +/-   89ns
#* PHC0                          0   3   377     4   +468ns[ +560ns] +/-  110ns
#* PHC0                          0   3   377    11   -310ns[ -430ns] +/-   72ns
#* PHC0                          0   3   377     9   +189ns[ +298ns] +/-   54ns
#* PHC0                          0   3   377     7   +594ns[ +473ns] +/-   96ns
#* PHC0                          0   3   377     5   +151ns[ +280ns] +/-   71ns
#* PHC0                          0   3   377    10   -590ns[ -696ns] +/-   94ns
#* PHC0                          0   3   377     8   +415ns[ +526ns] +/-   74ns
#* PHC0                          0   3   377     6  +1381ns[+1469ns] +/-  101ns
#* PHC0                          0   3   377     4   +571ns[+1304ns] +/-   54ns
#* PHC0                          0   3   377     8     -5ns[  +71ns] +/-  139ns
#* PHC0                          0   3   377     7   -247ns[ -502ns] +/-   69ns
#* PHC0                          0   3   377     5   -283ns[ +879ns] +/-   73ns
#* PHC0                          0   3   377     3   +148ns[ -109ns] +/-   61ns

With TSC delta calculation:
============================

#* PHC0                          0   3   377     7   +379ns[ +432ns] +/-   53ns
#* PHC0                          0   3   377     9   +106ns[ +420ns] +/-   42ns
#* PHC0                          0   3   377     7    -58ns[ -136ns] +/-   62ns
#* PHC0                          0   3   377    12    +93ns[  -38ns] +/-   64ns
#* PHC0                          0   3   377     8    +84ns[ +107ns] +/-   69ns
#* PHC0                          0   3   377     3    -76ns[ -103ns] +/-   52ns
#* PHC0                          0   3   377     7    +52ns[  +63ns] +/-   50ns
#* PHC0                          0   3   377    11    +29ns[  +31ns] +/-   70ns
#* PHC0                          0   3   377     7    -47ns[  -56ns] +/-   42ns
#* PHC0                          0   3   377    10    -35ns[  -42ns] +/-   33ns
#* PHC0                          0   3   377     7    -32ns[  -34ns] +/-   42ns
#* PHC0                          0   3   377    11   -172ns[ -173ns] +/-  118ns
#* PHC0                          0   3   377     6    +65ns[  +76ns] +/-   23ns
#* PHC0                          0   3   377     9    +18ns[  +23ns] +/-   37ns
#* PHC0                          0   3   377     6    +41ns[  -60ns] +/-   30ns
#* PHC0                          0   3   377    10    +39ns[ +183ns] +/-   42ns
#* PHC0                          0   3   377     6    +50ns[ +102ns] +/-   86ns
#* PHC0                          0   3   377    11    +50ns[  +75ns] +/-   52ns
#* PHC0                          0   3   377     6    +50ns[ +116ns] +/-  100ns
#* PHC0                          0   3   377    10    +46ns[  +65ns] +/-   79ns
#* PHC0                          0   3   377     7    -38ns[  -51ns] +/-   29ns
#* PHC0                          0   3   377    10    -11ns[  -12ns] +/-   32ns
#* PHC0                          0   3   377     7    -31ns[  -32ns] +/-   99ns
#* PHC0                          0   3   377    10   +222ns[ +238ns] +/-   58ns
#* PHC0                          0   3   377     6   +185ns[ +207ns] +/-   39ns
#* PHC0                          0   3   377    10   -392ns[ -394ns] +/-  118ns
#* PHC0                          0   3   377     6     -9ns[  -50ns] +/-   35ns
#* PHC0                          0   3   377    10   -346ns[ -355ns] +/-  111ns
Quoted 4 lines
Quoted 1 line

Do you still want to drop it in favour of simplicity?

This is the output of "chronyc sources". See section "Time sources" of https://chrony.tuxfamily.org/doc/2.4/chronyc.html.

Quoted 6 lines

It's just that it's not obvious why you get better results with biased host timestamps. What makes the biased host timestamp more precise?

I'd rather use PTP_SYS_OFFSET_PRECISE instead, but unfortunately chrony does not support it---but I would still prefer you to support PTP_SYS_OFFSET_PRECISE as well.

A single TSC read could be used to implement the PRECISE ioctl, but if a timer interrupt takes place on either the host or the guest, and that timer interrupt "adds" the TSC delta to xtime.nsec/xtime.sec, then that single TSC read cannot be used.

Quoted 5 lines

So you would have to stop timer interrupts (in guest and host) for the duration of the PRECISE ioctl in the guest to avoid that situation, which seems a bit overkill to me.

Any other ideas?

Could have a hypercall that disables host timer interrupts for a specified amount of time... But that does not scale with multiple VMs.

#41 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

2017-01-18 12:53-0200, Marcelo Tosatti:

Quoted 114 lines
Quoted 113 lines · Marcelo Tosatti
Quoted 100 lines · Paolo Bonzini
Quoted 92 lines · Marcelo Tosatti
Quoted 91 lines · Marcelo Tosatti
Quoted 22 lines · Radim Krcmar

2017-01-17 09:30-0200, Marcelo Tosatti:

Quoted 10 lines · Miroslav Lichvar
Quoted 9 lines

Users of the PTP_SYS_OFFSET ioctl assume that (ts[0]+ts[2])/2 corresponds to ts[1], (ts[2]+ts[4])/2 corresponds to ts[3], and so on.

                    ts[1]     ts[3]
Host time    ---------+---------+........
                      |         |
                      |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]

KVM PTP delay moves host ts[i] to be close to guest ts[i+1] and makes the offset very consistent, so the graph would look like:

                        ts[1]     ts[3]
Host time    -------------+---------+........
                          |         |
                          |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]
Quoted 2 lines

which doesn't sound good if users assume that the host reading is in the middle -- the guest time would be ahead of the host time.

Testcase: run a guest and a loop sending SIGUSR1 to vcpu0 (emulating intense interrupts). Follows results:

Without TSC delta calculation:
=============================

#* PHC0                          0   3   377     2    -99ns[ +206ns] +/-  116ns
#* PHC0                          0   3   377     8   +202ns[ +249ns] +/-  111ns
#* PHC0                          0   3   377     8   -213ns[ +683ns] +/-   88ns
#* PHC0                          0   3   377     6    +77ns[ +319ns] +/-   56ns
#* PHC0                          0   3   377     4   -771ns[-1029ns] +/-   93ns
#* PHC0                          0   3   377    10    -49ns[  -58ns] +/-  121ns
#* PHC0                          0   3   377     9   +562ns[ +703ns] +/-  107ns
#* PHC0                          0   3   377     6     -2ns[   -3ns] +/-   94ns
#* PHC0                          0   3   377     4   +451ns[ +494ns] +/-  138ns
#* PHC0                          0   3   377    11    -67ns[  -74ns] +/-  113ns
#* PHC0                          0   3   377     8   +244ns[ +264ns] +/-  119ns
#* PHC0                          0   3   377     7   -696ns[ -890ns] +/-   89ns
#* PHC0                          0   3   377     4   +468ns[ +560ns] +/-  110ns
#* PHC0                          0   3   377    11   -310ns[ -430ns] +/-   72ns
#* PHC0                          0   3   377     9   +189ns[ +298ns] +/-   54ns
#* PHC0                          0   3   377     7   +594ns[ +473ns] +/-   96ns
#* PHC0                          0   3   377     5   +151ns[ +280ns] +/-   71ns
#* PHC0                          0   3   377    10   -590ns[ -696ns] +/-   94ns
#* PHC0                          0   3   377     8   +415ns[ +526ns] +/-   74ns
#* PHC0                          0   3   377     6  +1381ns[+1469ns] +/-  101ns
#* PHC0                          0   3   377     4   +571ns[+1304ns] +/-   54ns
#* PHC0                          0   3   377     8     -5ns[  +71ns] +/-  139ns
#* PHC0                          0   3   377     7   -247ns[ -502ns] +/-   69ns
#* PHC0                          0   3   377     5   -283ns[ +879ns] +/-   73ns
#* PHC0                          0   3   377     3   +148ns[ -109ns] +/-   61ns

With TSC delta calculation:
============================

#* PHC0                          0   3   377     7   +379ns[ +432ns] +/-   53ns
#* PHC0                          0   3   377     9   +106ns[ +420ns] +/-   42ns
#* PHC0                          0   3   377     7    -58ns[ -136ns] +/-   62ns
#* PHC0                          0   3   377    12    +93ns[  -38ns] +/-   64ns
#* PHC0                          0   3   377     8    +84ns[ +107ns] +/-   69ns
#* PHC0                          0   3   377     3    -76ns[ -103ns] +/-   52ns
#* PHC0                          0   3   377     7    +52ns[  +63ns] +/-   50ns
#* PHC0                          0   3   377    11    +29ns[  +31ns] +/-   70ns
#* PHC0                          0   3   377     7    -47ns[  -56ns] +/-   42ns
#* PHC0                          0   3   377    10    -35ns[  -42ns] +/-   33ns
#* PHC0                          0   3   377     7    -32ns[  -34ns] +/-   42ns
#* PHC0                          0   3   377    11   -172ns[ -173ns] +/-  118ns
#* PHC0                          0   3   377     6    +65ns[  +76ns] +/-   23ns
#* PHC0                          0   3   377     9    +18ns[  +23ns] +/-   37ns
#* PHC0                          0   3   377     6    +41ns[  -60ns] +/-   30ns
#* PHC0                          0   3   377    10    +39ns[ +183ns] +/-   42ns
#* PHC0                          0   3   377     6    +50ns[ +102ns] +/-   86ns
#* PHC0                          0   3   377    11    +50ns[  +75ns] +/-   52ns
#* PHC0                          0   3   377     6    +50ns[ +116ns] +/-  100ns
#* PHC0                          0   3   377    10    +46ns[  +65ns] +/-   79ns
#* PHC0                          0   3   377     7    -38ns[  -51ns] +/-   29ns
#* PHC0                          0   3   377    10    -11ns[  -12ns] +/-   32ns
#* PHC0                          0   3   377     7    -31ns[  -32ns] +/-   99ns
#* PHC0                          0   3   377    10   +222ns[ +238ns] +/-   58ns
#* PHC0                          0   3   377     6   +185ns[ +207ns] +/-   39ns
#* PHC0                          0   3   377    10   -392ns[ -394ns] +/-  118ns
#* PHC0                          0   3   377     6     -9ns[  -50ns] +/-   35ns
#* PHC0                          0   3   377    10   -346ns[ -355ns] +/-  111ns
Quoted 4 lines
Quoted 1 line

Do you still want to drop it in favour of simplicity?

This is the output of "chronyc sources". See section "Time sources" of https://chrony.tuxfamily.org/doc/2.4/chronyc.html.

Quoted 6 lines

It's just that it's not obvious why you get better results with biased host timestamps. What makes the biased host timestamp more precise?

I'd rather use PTP_SYS_OFFSET_PRECISE instead, but unfortunately chrony does not support it---but I would still prefer you to support PTP_SYS_OFFSET_PRECISE as well.

A single TSC read could be used to implement the PRECISE ioctl, but if a timer interrupt takes place on either the host or the guest, and that timer interrupt "adds" the TSC delta to xtime.nsec/xtime.sec, then that single TSC read cannot be used.

Quoted 8 lines
Quoted 5 lines

So you would have to stop timer interrupts (in guest and host) for the duration of the PRECISE ioctl in the guest to avoid that situation, which seems a bit overkill to me.

Any other ideas?

Could have a hypercall that disables host timer interrupts for a specified amount of time... But that does not scale with multiple VMs.

No need to disable interrupts on guest nor host as both protect the time by a seqlock. Do timer interrupts always advance the time base? (We could disable guest interrupts if they were too disruptive because
of the long hypercall, but the host is fine in any case.)

The guest protects tsc->sec/nsec by tk_core.seq, so we can make sure that the guest computed sec/nsec from the host tsc read from hypercall is consistent.

The {sec @ tsc, nsec @ tsc, tsc} triple provided by the host is also correct. (PTP_SYS_OFFSET_PRECISE can't have any problems with the host that PTP_SYS_OFFSET wouldn't also have.)

author
#42 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 119 lines · Radim Krcmar

2017-01-18 12:53-0200, Marcelo Tosatti:

Quoted 118 lines
Quoted 117 lines · Marcelo Tosatti
Quoted 100 lines · Paolo Bonzini
Quoted 92 lines · Marcelo Tosatti
Quoted 91 lines · Marcelo Tosatti
Quoted 22 lines · Radim Krcmar

2017-01-17 09:30-0200, Marcelo Tosatti:

Quoted 10 lines · Miroslav Lichvar
Quoted 9 lines

Users of the PTP_SYS_OFFSET ioctl assume that (ts[0]+ts[2])/2 corresponds to ts[1], (ts[2]+ts[4])/2 corresponds to ts[3], and so on.

                    ts[1]     ts[3]
Host time    ---------+---------+........
                      |         |
                      |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]

KVM PTP delay moves host ts[i] to be close to guest ts[i+1] and makes the offset very consistent, so the graph would look like:

                        ts[1]     ts[3]
Host time    -------------+---------+........
                          |         |
                          |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]
Quoted 2 lines

which doesn't sound good if users assume that the host reading is in the middle -- the guest time would be ahead of the host time.

Testcase: run a guest and a loop sending SIGUSR1 to vcpu0 (emulating intense interrupts). Follows results:

Without TSC delta calculation:
=============================

#* PHC0                          0   3   377     2    -99ns[ +206ns] +/-  116ns
#* PHC0                          0   3   377     8   +202ns[ +249ns] +/-  111ns
#* PHC0                          0   3   377     8   -213ns[ +683ns] +/-   88ns
#* PHC0                          0   3   377     6    +77ns[ +319ns] +/-   56ns
#* PHC0                          0   3   377     4   -771ns[-1029ns] +/-   93ns
#* PHC0                          0   3   377    10    -49ns[  -58ns] +/-  121ns
#* PHC0                          0   3   377     9   +562ns[ +703ns] +/-  107ns
#* PHC0                          0   3   377     6     -2ns[   -3ns] +/-   94ns
#* PHC0                          0   3   377     4   +451ns[ +494ns] +/-  138ns
#* PHC0                          0   3   377    11    -67ns[  -74ns] +/-  113ns
#* PHC0                          0   3   377     8   +244ns[ +264ns] +/-  119ns
#* PHC0                          0   3   377     7   -696ns[ -890ns] +/-   89ns
#* PHC0                          0   3   377     4   +468ns[ +560ns] +/-  110ns
#* PHC0                          0   3   377    11   -310ns[ -430ns] +/-   72ns
#* PHC0                          0   3   377     9   +189ns[ +298ns] +/-   54ns
#* PHC0                          0   3   377     7   +594ns[ +473ns] +/-   96ns
#* PHC0                          0   3   377     5   +151ns[ +280ns] +/-   71ns
#* PHC0                          0   3   377    10   -590ns[ -696ns] +/-   94ns
#* PHC0                          0   3   377     8   +415ns[ +526ns] +/-   74ns
#* PHC0                          0   3   377     6  +1381ns[+1469ns] +/-  101ns
#* PHC0                          0   3   377     4   +571ns[+1304ns] +/-   54ns
#* PHC0                          0   3   377     8     -5ns[  +71ns] +/-  139ns
#* PHC0                          0   3   377     7   -247ns[ -502ns] +/-   69ns
#* PHC0                          0   3   377     5   -283ns[ +879ns] +/-   73ns
#* PHC0                          0   3   377     3   +148ns[ -109ns] +/-   61ns

With TSC delta calculation:
============================

#* PHC0                          0   3   377     7   +379ns[ +432ns] +/-   53ns
#* PHC0                          0   3   377     9   +106ns[ +420ns] +/-   42ns
#* PHC0                          0   3   377     7    -58ns[ -136ns] +/-   62ns
#* PHC0                          0   3   377    12    +93ns[  -38ns] +/-   64ns
#* PHC0                          0   3   377     8    +84ns[ +107ns] +/-   69ns
#* PHC0                          0   3   377     3    -76ns[ -103ns] +/-   52ns
#* PHC0                          0   3   377     7    +52ns[  +63ns] +/-   50ns
#* PHC0                          0   3   377    11    +29ns[  +31ns] +/-   70ns
#* PHC0                          0   3   377     7    -47ns[  -56ns] +/-   42ns
#* PHC0                          0   3   377    10    -35ns[  -42ns] +/-   33ns
#* PHC0                          0   3   377     7    -32ns[  -34ns] +/-   42ns
#* PHC0                          0   3   377    11   -172ns[ -173ns] +/-  118ns
#* PHC0                          0   3   377     6    +65ns[  +76ns] +/-   23ns
#* PHC0                          0   3   377     9    +18ns[  +23ns] +/-   37ns
#* PHC0                          0   3   377     6    +41ns[  -60ns] +/-   30ns
#* PHC0                          0   3   377    10    +39ns[ +183ns] +/-   42ns
#* PHC0                          0   3   377     6    +50ns[ +102ns] +/-   86ns
#* PHC0                          0   3   377    11    +50ns[  +75ns] +/-   52ns
#* PHC0                          0   3   377     6    +50ns[ +116ns] +/-  100ns
#* PHC0                          0   3   377    10    +46ns[  +65ns] +/-   79ns
#* PHC0                          0   3   377     7    -38ns[  -51ns] +/-   29ns
#* PHC0                          0   3   377    10    -11ns[  -12ns] +/-   32ns
#* PHC0                          0   3   377     7    -31ns[  -32ns] +/-   99ns
#* PHC0                          0   3   377    10   +222ns[ +238ns] +/-   58ns
#* PHC0                          0   3   377     6   +185ns[ +207ns] +/-   39ns
#* PHC0                          0   3   377    10   -392ns[ -394ns] +/-  118ns
#* PHC0                          0   3   377     6     -9ns[  -50ns] +/-   35ns
#* PHC0                          0   3   377    10   -346ns[ -355ns] +/-  111ns
Quoted 4 lines
Quoted 1 line

Do you still want to drop it in favour of simplicity?

This is the output of "chronyc sources". See section "Time sources" of https://chrony.tuxfamily.org/doc/2.4/chronyc.html.

Quoted 6 lines

It's just that it's not obvious why you get better results with biased host timestamps. What makes the biased host timestamp more precise?

I'd rather use PTP_SYS_OFFSET_PRECISE instead, but unfortunately chrony does not support it---but I would still prefer you to support PTP_SYS_OFFSET_PRECISE as well.

A single TSC read could be used to implement the PRECISE ioctl, but if a timer interrupt takes place on either the host or the guest, and that timer interrupt "adds" the TSC delta to xtime.nsec/xtime.sec, then that single TSC read cannot be used.

So you would have to stop timer interrupts (in guest and host) for the duration of the PRECISE ioctl in the guest to avoid that situation, which seems a bit overkill to me.

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

Any other ideas?

Could have a hypercall that disables host timer interrupts for a specified amount of time... But that does not scale with multiple VMs.

No need to disable interrupts on guest nor host as both protect the time by a seqlock.

Still not scalable with multiple VMs... so need a different solution.

Quoted 1 line

Do timer interrupts always advance the time base?

No only when a full "cycle_interval" has been accumulated.

Quoted 2 lines

(We could disable guest interrupts if they were too disruptive because
of the long hypercall, but the host is fine in any case.)

VMs can harm host interrupt processing, don't see that as being OK (even when rate limited does scale to high number of VMs).

Quoted 3 lines

The guest protects tsc->sec/nsec by tk_core.seq, so we can make sure that the guest computed sec/nsec from the host tsc read from hypercall is consistent.

Doh, of course.

Quoted 3 lines

The {sec @ tsc, nsec @ tsc, tsc} triple provided by the host is also correct. (PTP_SYS_OFFSET_PRECISE can't have any problems with the host that PTP_SYS_OFFSET wouldn't also have.)

Yep, should work, lets try.

#43 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

2017-01-18 13:28-0200, Marcelo Tosatti:

Quoted 122 lines · Radim Krcmar
Quoted 121 lines

2017-01-18 12:53-0200, Marcelo Tosatti:

Quoted 120 lines
Quoted 118 lines · Marcelo Tosatti
Quoted 100 lines · Paolo Bonzini
Quoted 92 lines · Marcelo Tosatti
Quoted 91 lines · Marcelo Tosatti
Quoted 22 lines · Radim Krcmar

2017-01-17 09:30-0200, Marcelo Tosatti:

Quoted 10 lines · Miroslav Lichvar
Quoted 9 lines

Users of the PTP_SYS_OFFSET ioctl assume that (ts[0]+ts[2])/2 corresponds to ts[1], (ts[2]+ts[4])/2 corresponds to ts[3], and so on.

                    ts[1]     ts[3]
Host time    ---------+---------+........
                      |         |
                      |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]

KVM PTP delay moves host ts[i] to be close to guest ts[i+1] and makes the offset very consistent, so the graph would look like:

                        ts[1]     ts[3]
Host time    -------------+---------+........
                          |         |
                          |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]
Quoted 2 lines

which doesn't sound good if users assume that the host reading is in the middle -- the guest time would be ahead of the host time.

Testcase: run a guest and a loop sending SIGUSR1 to vcpu0 (emulating intense interrupts). Follows results:

Without TSC delta calculation:
=============================

#* PHC0                          0   3   377     2    -99ns[ +206ns] +/-  116ns
#* PHC0                          0   3   377     8   +202ns[ +249ns] +/-  111ns
#* PHC0                          0   3   377     8   -213ns[ +683ns] +/-   88ns
#* PHC0                          0   3   377     6    +77ns[ +319ns] +/-   56ns
#* PHC0                          0   3   377     4   -771ns[-1029ns] +/-   93ns
#* PHC0                          0   3   377    10    -49ns[  -58ns] +/-  121ns
#* PHC0                          0   3   377     9   +562ns[ +703ns] +/-  107ns
#* PHC0                          0   3   377     6     -2ns[   -3ns] +/-   94ns
#* PHC0                          0   3   377     4   +451ns[ +494ns] +/-  138ns
#* PHC0                          0   3   377    11    -67ns[  -74ns] +/-  113ns
#* PHC0                          0   3   377     8   +244ns[ +264ns] +/-  119ns
#* PHC0                          0   3   377     7   -696ns[ -890ns] +/-   89ns
#* PHC0                          0   3   377     4   +468ns[ +560ns] +/-  110ns
#* PHC0                          0   3   377    11   -310ns[ -430ns] +/-   72ns
#* PHC0                          0   3   377     9   +189ns[ +298ns] +/-   54ns
#* PHC0                          0   3   377     7   +594ns[ +473ns] +/-   96ns
#* PHC0                          0   3   377     5   +151ns[ +280ns] +/-   71ns
#* PHC0                          0   3   377    10   -590ns[ -696ns] +/-   94ns
#* PHC0                          0   3   377     8   +415ns[ +526ns] +/-   74ns
#* PHC0                          0   3   377     6  +1381ns[+1469ns] +/-  101ns
#* PHC0                          0   3   377     4   +571ns[+1304ns] +/-   54ns
#* PHC0                          0   3   377     8     -5ns[  +71ns] +/-  139ns
#* PHC0                          0   3   377     7   -247ns[ -502ns] +/-   69ns
#* PHC0                          0   3   377     5   -283ns[ +879ns] +/-   73ns
#* PHC0                          0   3   377     3   +148ns[ -109ns] +/-   61ns

With TSC delta calculation:
============================

#* PHC0                          0   3   377     7   +379ns[ +432ns] +/-   53ns
#* PHC0                          0   3   377     9   +106ns[ +420ns] +/-   42ns
#* PHC0                          0   3   377     7    -58ns[ -136ns] +/-   62ns
#* PHC0                          0   3   377    12    +93ns[  -38ns] +/-   64ns
#* PHC0                          0   3   377     8    +84ns[ +107ns] +/-   69ns
#* PHC0                          0   3   377     3    -76ns[ -103ns] +/-   52ns
#* PHC0                          0   3   377     7    +52ns[  +63ns] +/-   50ns
#* PHC0                          0   3   377    11    +29ns[  +31ns] +/-   70ns
#* PHC0                          0   3   377     7    -47ns[  -56ns] +/-   42ns
#* PHC0                          0   3   377    10    -35ns[  -42ns] +/-   33ns
#* PHC0                          0   3   377     7    -32ns[  -34ns] +/-   42ns
#* PHC0                          0   3   377    11   -172ns[ -173ns] +/-  118ns
#* PHC0                          0   3   377     6    +65ns[  +76ns] +/-   23ns
#* PHC0                          0   3   377     9    +18ns[  +23ns] +/-   37ns
#* PHC0                          0   3   377     6    +41ns[  -60ns] +/-   30ns
#* PHC0                          0   3   377    10    +39ns[ +183ns] +/-   42ns
#* PHC0                          0   3   377     6    +50ns[ +102ns] +/-   86ns
#* PHC0                          0   3   377    11    +50ns[  +75ns] +/-   52ns
#* PHC0                          0   3   377     6    +50ns[ +116ns] +/-  100ns
#* PHC0                          0   3   377    10    +46ns[  +65ns] +/-   79ns
#* PHC0                          0   3   377     7    -38ns[  -51ns] +/-   29ns
#* PHC0                          0   3   377    10    -11ns[  -12ns] +/-   32ns
#* PHC0                          0   3   377     7    -31ns[  -32ns] +/-   99ns
#* PHC0                          0   3   377    10   +222ns[ +238ns] +/-   58ns
#* PHC0                          0   3   377     6   +185ns[ +207ns] +/-   39ns
#* PHC0                          0   3   377    10   -392ns[ -394ns] +/-  118ns
#* PHC0                          0   3   377     6     -9ns[  -50ns] +/-   35ns
#* PHC0                          0   3   377    10   -346ns[ -355ns] +/-  111ns
Quoted 4 lines
Quoted 1 line

Do you still want to drop it in favour of simplicity?

This is the output of "chronyc sources". See section "Time sources" of https://chrony.tuxfamily.org/doc/2.4/chronyc.html.

Quoted 6 lines

It's just that it's not obvious why you get better results with biased host timestamps. What makes the biased host timestamp more precise?

I'd rather use PTP_SYS_OFFSET_PRECISE instead, but unfortunately chrony does not support it---but I would still prefer you to support PTP_SYS_OFFSET_PRECISE as well.

A single TSC read could be used to implement the PRECISE ioctl, but if a timer interrupt takes place on either the host or the guest, and that timer interrupt "adds" the TSC delta to xtime.nsec/xtime.sec, then that single TSC read cannot be used.

So you would have to stop timer interrupts (in guest and host) for the duration of the PRECISE ioctl in the guest to avoid that situation, which seems a bit overkill to me.

Any other ideas?

Quoted 7 lines
Quoted 5 lines
Quoted 2 lines

Could have a hypercall that disables host timer interrupts for a specified amount of time... But that does not scale with multiple VMs.

No need to disable interrupts on guest nor host as both protect the time by a seqlock.

Still not scalable with multiple VMs... so need a different solution.

What doesn't scale?
The VM hypercall takes read on the tk_core.seq, which doesn't block other VMs from doing the same.

author
#44 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

In reply to Paolo Bonzini
Quoted 93 lines · Paolo Bonzini
Quoted 92 lines · Marcelo Tosatti
Quoted 91 lines · Marcelo Tosatti
Quoted 22 lines · Radim Krcmar

2017-01-17 09:30-0200, Marcelo Tosatti:

Quoted 10 lines · Miroslav Lichvar
Quoted 9 lines

Users of the PTP_SYS_OFFSET ioctl assume that (ts[0]+ts[2])/2 corresponds to ts[1], (ts[2]+ts[4])/2 corresponds to ts[3], and so on.

                    ts[1]     ts[3]
Host time    ---------+---------+........
                      |         |
                      |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]

KVM PTP delay moves host ts[i] to be close to guest ts[i+1] and makes the offset very consistent, so the graph would look like:

                        ts[1]     ts[3]
Host time    -------------+---------+........
                          |         |
                          |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]
Quoted 2 lines

which doesn't sound good if users assume that the host reading is in the middle -- the guest time would be ahead of the host time.

Testcase: run a guest and a loop sending SIGUSR1 to vcpu0 (emulating intense interrupts). Follows results:

Without TSC delta calculation:
=============================

#* PHC0                          0   3   377     2    -99ns[ +206ns] +/-  116ns
#* PHC0                          0   3   377     8   +202ns[ +249ns] +/-  111ns
#* PHC0                          0   3   377     8   -213ns[ +683ns] +/-   88ns
#* PHC0                          0   3   377     6    +77ns[ +319ns] +/-   56ns
#* PHC0                          0   3   377     4   -771ns[-1029ns] +/-   93ns
#* PHC0                          0   3   377    10    -49ns[  -58ns] +/-  121ns
#* PHC0                          0   3   377     9   +562ns[ +703ns] +/-  107ns
#* PHC0                          0   3   377     6     -2ns[   -3ns] +/-   94ns
#* PHC0                          0   3   377     4   +451ns[ +494ns] +/-  138ns
#* PHC0                          0   3   377    11    -67ns[  -74ns] +/-  113ns
#* PHC0                          0   3   377     8   +244ns[ +264ns] +/-  119ns
#* PHC0                          0   3   377     7   -696ns[ -890ns] +/-   89ns
#* PHC0                          0   3   377     4   +468ns[ +560ns] +/-  110ns
#* PHC0                          0   3   377    11   -310ns[ -430ns] +/-   72ns
#* PHC0                          0   3   377     9   +189ns[ +298ns] +/-   54ns
#* PHC0                          0   3   377     7   +594ns[ +473ns] +/-   96ns
#* PHC0                          0   3   377     5   +151ns[ +280ns] +/-   71ns
#* PHC0                          0   3   377    10   -590ns[ -696ns] +/-   94ns
#* PHC0                          0   3   377     8   +415ns[ +526ns] +/-   74ns
#* PHC0                          0   3   377     6  +1381ns[+1469ns] +/-  101ns
#* PHC0                          0   3   377     4   +571ns[+1304ns] +/-   54ns
#* PHC0                          0   3   377     8     -5ns[  +71ns] +/-  139ns
#* PHC0                          0   3   377     7   -247ns[ -502ns] +/-   69ns
#* PHC0                          0   3   377     5   -283ns[ +879ns] +/-   73ns
#* PHC0                          0   3   377     3   +148ns[ -109ns] +/-   61ns

With TSC delta calculation:
============================

#* PHC0                          0   3   377     7   +379ns[ +432ns] +/-   53ns
#* PHC0                          0   3   377     9   +106ns[ +420ns] +/-   42ns
#* PHC0                          0   3   377     7    -58ns[ -136ns] +/-   62ns
#* PHC0                          0   3   377    12    +93ns[  -38ns] +/-   64ns
#* PHC0                          0   3   377     8    +84ns[ +107ns] +/-   69ns
#* PHC0                          0   3   377     3    -76ns[ -103ns] +/-   52ns
#* PHC0                          0   3   377     7    +52ns[  +63ns] +/-   50ns
#* PHC0                          0   3   377    11    +29ns[  +31ns] +/-   70ns
#* PHC0                          0   3   377     7    -47ns[  -56ns] +/-   42ns
#* PHC0                          0   3   377    10    -35ns[  -42ns] +/-   33ns
#* PHC0                          0   3   377     7    -32ns[  -34ns] +/-   42ns
#* PHC0                          0   3   377    11   -172ns[ -173ns] +/-  118ns
#* PHC0                          0   3   377     6    +65ns[  +76ns] +/-   23ns
#* PHC0                          0   3   377     9    +18ns[  +23ns] +/-   37ns
#* PHC0                          0   3   377     6    +41ns[  -60ns] +/-   30ns
#* PHC0                          0   3   377    10    +39ns[ +183ns] +/-   42ns
#* PHC0                          0   3   377     6    +50ns[ +102ns] +/-   86ns
#* PHC0                          0   3   377    11    +50ns[  +75ns] +/-   52ns
#* PHC0                          0   3   377     6    +50ns[ +116ns] +/-  100ns
#* PHC0                          0   3   377    10    +46ns[  +65ns] +/-   79ns
#* PHC0                          0   3   377     7    -38ns[  -51ns] +/-   29ns
#* PHC0                          0   3   377    10    -11ns[  -12ns] +/-   32ns
#* PHC0                          0   3   377     7    -31ns[  -32ns] +/-   99ns
#* PHC0                          0   3   377    10   +222ns[ +238ns] +/-   58ns
#* PHC0                          0   3   377     6   +185ns[ +207ns] +/-   39ns
#* PHC0                          0   3   377    10   -392ns[ -394ns] +/-  118ns
#* PHC0                          0   3   377     6     -9ns[  -50ns] +/-   35ns
#* PHC0                          0   3   377    10   -346ns[ -355ns] +/-  111ns
Quoted 7 lines
Quoted 4 lines
Quoted 1 line

Do you still want to drop it in favour of simplicity?

This is the output of "chronyc sources". See section "Time sources" of https://chrony.tuxfamily.org/doc/2.4/chronyc.html.

It's just that it's not obvious why you get better results with biased host timestamps. What makes the biased host timestamp more precise?

The issue is that, without it you have a larger window for interruptions to take place and therefore the read value when ->gettime64 return is from longer time in the past.

Quoted 3 lines

I'd rather use PTP_SYS_OFFSET_PRECISE instead, but unfortunately chrony does not support it---but I would still prefer you to support PTP_SYS_OFFSET_PRECISE as well.

Sure, I'll check if it makes sense to implement PTP_SYS_OFFSET_PRECISE for KVM case.

#45 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 7 lines · Marcelo Tosatti
Quoted 3 lines · Paolo Bonzini

I'd rather use PTP_SYS_OFFSET_PRECISE instead, but unfortunately chrony does not support it---but I would still prefer you to support PTP_SYS_OFFSET_PRECISE as well.

Sure, I'll check if it makes sense to implement PTP_SYS_OFFSET_PRECISE for KVM case.

But is it really so precise that the application can safely assume there are no errors due to asymmetric delays, etc? I think PTP_SYS_OFFSET_PRECISE should be supported only if the accuracy of the offset measured between the HW and system clock is not worse than the precision of the system clock (typically few tens of nanoseconds).

It would be good to verify the accuracy of the offset when the host and guest clocks are synchronised to each other over PTP using two NICs with HW timestamping.

Signature · 1 line
Miroslav Lichvar
maintainer
#46 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 11 lines · Miroslav Lichvar
Quoted 2 lines

Sure, I'll check if it makes sense to implement PTP_SYS_OFFSET_PRECISE for KVM case.

But is it really so precise that the application can safely assume there are no errors due to asymmetric delays, etc? I think PTP_SYS_OFFSET_PRECISE should be supported only if the accuracy of the offset measured between the HW and system clock is not worse than the precision of the system clock (typically few tens of nanoseconds).

It would be good to verify the accuracy of the offset when the host and guest clocks are synchronised to each other over PTP using two NICs with HW timestamping.

PTP_SYS_OFFSET_PRECISE works if the guest can compute its own timestamp based on the same source as the device. On bare metal you have:

- the source for system clock is the TSC (with clocksource_tsc)
- device provides a (time, ART) tuple

You can convert ART->TSC and then ask clocksource_tsc for a system time based on the device-provided ART value. Likewise for KVM:

- the source for system clock is the guest TSC (with kvmclock)
- host can provide a (time, guest TSC) tuple

The PTP driver can take the host-provided guest TSC, and ask kvmclock for a system time based on that TSC value. It's even simpler because there's no ART->TSC conversion involved, and it will always be precise and independent of any vmexit or interrupt delay.

Paolo

#47 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

In reply to Miroslav Lichvar

2017-01-18 16:54+0100, Miroslav Lichvar:

Quoted 6 lines · Marcelo Tosatti
Quoted 5 lines · Paolo Bonzini
Quoted 3 lines

I'd rather use PTP_SYS_OFFSET_PRECISE instead, but unfortunately chrony does not support it---but I would still prefer you to support PTP_SYS_OFFSET_PRECISE as well.

Quoted 8 lines
Quoted 2 lines

Sure, I'll check if it makes sense to implement PTP_SYS_OFFSET_PRECISE for KVM case.

But is it really so precise that the application can safely assume there are no errors due to asymmetric delays, etc? I think PTP_SYS_OFFSET_PRECISE should be supported only if the accuracy of the offset measured between the HW and system clock is not worse than the precision of the system clock (typically few tens of nanoseconds).

KVM is actually the perfect user of PTP_SYS_OFFSET_PRECISE: the host and guest system clocks use the same hardware clock.

(We want to copy the host data about the clock to the guest, and we just
happen to use PTP for that.)

Quoted 3 lines

It would be good to verify the accuracy of the offset when the host and guest clocks are synchronised to each other over PTP using two NICs with HW timestamping.

We can verify the accuracy just by reading the host time and tsc computing guest time from that tsc -- they should be equal.

Well, we don't give host frequency to the guest, but I hope that the guest can compute it accurately after polling the host few times. (Still, a room for improvement.)

NICs with HW timestamping would be a bit more complicated to set up. :)

#48 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

In reply to Paolo Bonzini

2017-01-18 13:46+0100, Paolo Bonzini:

Quoted 95 lines · Marcelo Tosatti
Quoted 92 lines · Marcelo Tosatti
Quoted 91 lines · Radim Krcmar
Quoted 22 lines

2017-01-17 09:30-0200, Marcelo Tosatti:

Quoted 10 lines · Miroslav Lichvar
Quoted 9 lines

Users of the PTP_SYS_OFFSET ioctl assume that (ts[0]+ts[2])/2 corresponds to ts[1], (ts[2]+ts[4])/2 corresponds to ts[3], and so on.

                    ts[1]     ts[3]
Host time    ---------+---------+........
                      |         |
                      |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]

KVM PTP delay moves host ts[i] to be close to guest ts[i+1] and makes the offset very consistent, so the graph would look like:

                        ts[1]     ts[3]
Host time    -------------+---------+........
                          |         |
                          |         |
Guest time   ----+---------+---------+......
                ts[0]    ts[2]     ts[4]
Quoted 2 lines

which doesn't sound good if users assume that the host reading is in the middle -- the guest time would be ahead of the host time.

Testcase: run a guest and a loop sending SIGUSR1 to vcpu0 (emulating intense interrupts). Follows results:

Without TSC delta calculation:
=============================

#* PHC0                          0   3   377     2    -99ns[ +206ns] +/-  116ns
#* PHC0                          0   3   377     8   +202ns[ +249ns] +/-  111ns
#* PHC0                          0   3   377     8   -213ns[ +683ns] +/-   88ns
#* PHC0                          0   3   377     6    +77ns[ +319ns] +/-   56ns
#* PHC0                          0   3   377     4   -771ns[-1029ns] +/-   93ns
#* PHC0                          0   3   377    10    -49ns[  -58ns] +/-  121ns
#* PHC0                          0   3   377     9   +562ns[ +703ns] +/-  107ns
#* PHC0                          0   3   377     6     -2ns[   -3ns] +/-   94ns
#* PHC0                          0   3   377     4   +451ns[ +494ns] +/-  138ns
#* PHC0                          0   3   377    11    -67ns[  -74ns] +/-  113ns
#* PHC0                          0   3   377     8   +244ns[ +264ns] +/-  119ns
#* PHC0                          0   3   377     7   -696ns[ -890ns] +/-   89ns
#* PHC0                          0   3   377     4   +468ns[ +560ns] +/-  110ns
#* PHC0                          0   3   377    11   -310ns[ -430ns] +/-   72ns
#* PHC0                          0   3   377     9   +189ns[ +298ns] +/-   54ns
#* PHC0                          0   3   377     7   +594ns[ +473ns] +/-   96ns
#* PHC0                          0   3   377     5   +151ns[ +280ns] +/-   71ns
#* PHC0                          0   3   377    10   -590ns[ -696ns] +/-   94ns
#* PHC0                          0   3   377     8   +415ns[ +526ns] +/-   74ns
#* PHC0                          0   3   377     6  +1381ns[+1469ns] +/-  101ns
#* PHC0                          0   3   377     4   +571ns[+1304ns] +/-   54ns
#* PHC0                          0   3   377     8     -5ns[  +71ns] +/-  139ns
#* PHC0                          0   3   377     7   -247ns[ -502ns] +/-   69ns
#* PHC0                          0   3   377     5   -283ns[ +879ns] +/-   73ns
#* PHC0                          0   3   377     3   +148ns[ -109ns] +/-   61ns

With TSC delta calculation:
============================

#* PHC0                          0   3   377     7   +379ns[ +432ns] +/-   53ns
#* PHC0                          0   3   377     9   +106ns[ +420ns] +/-   42ns
#* PHC0                          0   3   377     7    -58ns[ -136ns] +/-   62ns
#* PHC0                          0   3   377    12    +93ns[  -38ns] +/-   64ns
#* PHC0                          0   3   377     8    +84ns[ +107ns] +/-   69ns
#* PHC0                          0   3   377     3    -76ns[ -103ns] +/-   52ns
#* PHC0                          0   3   377     7    +52ns[  +63ns] +/-   50ns
#* PHC0                          0   3   377    11    +29ns[  +31ns] +/-   70ns
#* PHC0                          0   3   377     7    -47ns[  -56ns] +/-   42ns
#* PHC0                          0   3   377    10    -35ns[  -42ns] +/-   33ns
#* PHC0                          0   3   377     7    -32ns[  -34ns] +/-   42ns
#* PHC0                          0   3   377    11   -172ns[ -173ns] +/-  118ns
#* PHC0                          0   3   377     6    +65ns[  +76ns] +/-   23ns
#* PHC0                          0   3   377     9    +18ns[  +23ns] +/-   37ns
#* PHC0                          0   3   377     6    +41ns[  -60ns] +/-   30ns
#* PHC0                          0   3   377    10    +39ns[ +183ns] +/-   42ns
#* PHC0                          0   3   377     6    +50ns[ +102ns] +/-   86ns
#* PHC0                          0   3   377    11    +50ns[  +75ns] +/-   52ns
#* PHC0                          0   3   377     6    +50ns[ +116ns] +/-  100ns
#* PHC0                          0   3   377    10    +46ns[  +65ns] +/-   79ns
#* PHC0                          0   3   377     7    -38ns[  -51ns] +/-   29ns
#* PHC0                          0   3   377    10    -11ns[  -12ns] +/-   32ns
#* PHC0                          0   3   377     7    -31ns[  -32ns] +/-   99ns
#* PHC0                          0   3   377    10   +222ns[ +238ns] +/-   58ns
#* PHC0                          0   3   377     6   +185ns[ +207ns] +/-   39ns
#* PHC0                          0   3   377    10   -392ns[ -394ns] +/-  118ns
#* PHC0                          0   3   377     6     -9ns[  -50ns] +/-   35ns
#* PHC0                          0   3   377    10   -346ns[ -355ns] +/-  111ns
Quoted 7 lines
Quoted 4 lines
Quoted 1 line

Do you still want to drop it in favour of simplicity?

This is the output of "chronyc sources". See section "Time sources" of https://chrony.tuxfamily.org/doc/2.4/chronyc.html.

It's just that it's not obvious why you get better results with biased host timestamps. What makes the biased host timestamp more precise?

I think that TSC offsetting is more precise (the difference between two samples smaller), but less accurate (the difference between guest and host clock is bigger).
And chronyc can't tell how much off from the host clock our guest clock is (that would be measured how I wrote above).

The main reason why it is more precise is because the TSC offset erases the effect of many latency sources that can happen before or after the host time read.

The precision with TSC offsetting is almost only affected by the duration between two guest time measurements, because the host time is shifted to be very close to the latter guest time measurement.

Without TSC offsetting, the host time will be closer to the middle between two guest reads (= better accuracy), but the moment of the tsc read on the host has more variables -- host interrupts and whatnot, which will result in lower precision.

Quoted 3 lines

I'd rather use PTP_SYS_OFFSET_PRECISE instead, but unfortunately chrony does not support it---but I would still prefer you to support PTP_SYS_OFFSET_PRECISE as well.

Oh, I didn't expect that userspace can't use PTP_SYS_OFFSET_PRECISE ...

author
#49 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

In reply to Radim Krcmar
Quoted 61 lines · Radim Krcmar

2017-01-13 15:40-0200, Marcelo Tosatti:

Quoted 60 lines · Radim Krcmar
Quoted 41 lines

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 26 lines
drivers/ptp/ptp_kvm.c new file +26 -0
+		version = pvclock_read_begin(src);
+
+		ret = kvm_hypercall2(KVM_HC_CLOCK_OFFSET,
+				     clock_off_gpa,
+				     KVM_CLOCK_OFFSET_WALLCLOCK);
+		if (ret != 0) {
+			pr_err("clock offset hypercall ret %lu\n", ret);
+			spin_unlock(&kvm_ptp_lock);
+			preempt_enable_notrace();
+			return -EOPNOTSUPP;
+		}
+
+		tspec.tv_sec = clock_off.sec;
+		tspec.tv_nsec = clock_off.nsec;
+
+		delta = rdtsc_ordered() - clock_off.tsc;
+
+		offset = pvclock_scale_delta(delta, src->tsc_to_system_mul,
+					     src->tsc_shift);
+
+	} while (pvclock_read_retry(src, version));
+
+	preempt_enable_notrace();
+
+	tspec.tv_nsec = tspec.tv_nsec + offset;
+

drivers/ptp/ptp_kvm.c line 116

Quoted 10 lines
drivers/ptp/ptp_kvm.c new file +10 -0
+	spin_unlock(&kvm_ptp_lock);
+
+	if (tspec.tv_nsec >= NSEC_PER_SEC) {
+		u64 secs = tspec.tv_nsec;
+
+		tspec.tv_nsec = do_div(secs, NSEC_PER_SEC);
+		tspec.tv_sec += secs;
+	}
+
+	memcpy(ts, &tspec, sizeof(struct timespec64));

But the whole idea is of improving the time by reading tsc a bit later is just weird ... why is it better to provide

Quoted 7 lines

tsc + x, time + tsc_delta_to_time(x)

than just

tsc, time

?

Because you want to calculate the value of the host realtime clock at the moment of ptp_kvm_gettime.

We do:

1. kvm_hypercall.
2. get {sec, nsec, guest_tsc}.
3. kvm_hypercall returns.
4. delay = rdtsc() - guest_tsc.

Quoted 6 lines
Quoted 1 line

Where delay is the delta (measured with the TSC) between points 2 and 4.

I see now ... the PTP interface is just not good for our purposes. We don't return {sec, nsec, guest_tsc}, we just return {sec, nsec} at some random time in the past. And to make it a bit more accurate, you add a best-effort delta before returning, which makes sense.

Not random time in the past. We return {sec, nsec} from the host realtime at the moment the user ran the hypercall.

Since PTP is very accurate, that "a bit more" counts, yes.

Quoted 4 lines

When we have to depend on pvclock, what are the advantages of not using the existing pvclock API for wall clock?
(You mentioned some extensions.)

Quoted 8 lines
  struct pvclock_wall_clock {
  	u32   version;
  	u32   sec;
  	u32   nsec;
  } __attribute__((__packed__));

It gives the wall clock when pvclock was 0, so you just add current kvmclock and get the host wall clock.

Well, no. For one, the TSC part of kvmclock:

	kvmclock-read = system_timestamp + convert-to-1GHz(rdtsc() - tsc_timestamp)
				           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Drifts relative to UTC. This part can be large.
The guests NTP is responsible for fixing
that drift of the guests realtime clock (talking about current setup, without KVM PTP driver).

Now, we want very high precision (less than 1us) for this driver. Very small TSC drifts on a large delta defeat the purpose.

Quoted 1 line

Without a VM exit.

Huge performance is not an issue. Accuracy (how different from the host realtime clock our "approximation" of the host realtime clock) is.

Quoted 1 line

And how often is ptp_kvm_gettime() usually called?

The PTP_SYS_OFFSET ioctl calls the following code in a loop:

struct ptp_sys_offset {
unsigned int n_samples; /* Desired number of measurements. */
unsigned int rsv[3]; /* Reserved for future use. */
/*
* Array of interleaved system/phc time stamps. The kernel
* will provide 2*n_samples + 1 time stamps, with the last
* one as a system time stamp.
*/
struct ptp_clock_time ts[2 * PTP_MAX_SAMPLES + 1];
};

#define PTP_MAX_SAMPLES 25 /* Maximum allowed offset measurement samples. */

case PTP_SYS_OFFSET:
sysoff = memdup_user((void __user *)arg,

sizeof(*sysoff));
                if (IS_ERR(sysoff)) {
                        err = PTR_ERR(sysoff);
                        sysoff = NULL;
                        break;
                }
                if (sysoff->n_samples > PTP_MAX_SAMPLES) {
                        err = -EINVAL;
                        break;
                }
                pct = &sysoff->ts[0];
                for (i = 0; i < sysoff->n_samples; i++) {
                        getnstimeofday64(&ts);
                        pct->sec = ts.tv_sec;
                        pct->nsec = ts.tv_nsec;
                        pct++;
                        ptp->info->gettime64(ptp->info, &ts);
                        pct->sec = ts.tv_sec;
                        pct->nsec = ts.tv_nsec;
                        pct++;
                }
                getnstimeofday64(&ts);
                pct->sec = ts.tv_sec;
                pct->nsec = ts.tv_nsec;

How often that ioctl is called depends on the parameters of the Chrony PHC code. Initially (to determine the clock difference Chrony should call it more frequently, later on it should call it less frequency).

Perhaps once every second initially (the ioctl). I'll confirm with the exact value for my setup and reply to this email.

Quoted 10 lines

Thanks.

Quoted 6 lines
Quoted 3 lines

Because we'll always be quering the time at tsc + y, where y >> x, and we'd likely have other problems if shifting the time base by few thousand cycles made a difference.

Radim, i didnt get your "tsc + x", "time + tsc_delta_to_time(x)" formulas above. Can you be more verbose please?

Quoted 8 lines

x is the delta, tsc_delta_to_time() is what pvclock_scale_delta() does.

I assumed that we set precise time with TSC, so the delta wouldn't matter, because PTP would either get {sec, nsec, guest_tsc}, or the same, but just shifted by delta, hence
{sec + tsc_delta_to_time(x) / NSEC_PER_SEC,
nsec + tsc_delta_to_time(x) % NSEC_PER_SEC,
guest_tsc + x}.

Ah, OK. I understand you now understood the meaning of "tsc" part of the {sec, nsec, guest_tsc} triple.

#50 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

2017-01-16 15:04-0200, Marcelo Tosatti:

Quoted 64 lines · Radim Krcmar
Quoted 63 lines

2017-01-13 15:40-0200, Marcelo Tosatti:

Quoted 61 lines · Radim Krcmar
Quoted 41 lines

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 26 lines
drivers/ptp/ptp_kvm.c new file +26 -0
+		version = pvclock_read_begin(src);
+
+		ret = kvm_hypercall2(KVM_HC_CLOCK_OFFSET,
+				     clock_off_gpa,
+				     KVM_CLOCK_OFFSET_WALLCLOCK);
+		if (ret != 0) {
+			pr_err("clock offset hypercall ret %lu\n", ret);
+			spin_unlock(&kvm_ptp_lock);
+			preempt_enable_notrace();
+			return -EOPNOTSUPP;
+		}
+
+		tspec.tv_sec = clock_off.sec;
+		tspec.tv_nsec = clock_off.nsec;
+
+		delta = rdtsc_ordered() - clock_off.tsc;
+
+		offset = pvclock_scale_delta(delta, src->tsc_to_system_mul,
+					     src->tsc_shift);
+
+	} while (pvclock_read_retry(src, version));
+
+	preempt_enable_notrace();
+
+	tspec.tv_nsec = tspec.tv_nsec + offset;
+

drivers/ptp/ptp_kvm.c line 116

Quoted 10 lines
drivers/ptp/ptp_kvm.c new file +10 -0
+	spin_unlock(&kvm_ptp_lock);
+
+	if (tspec.tv_nsec >= NSEC_PER_SEC) {
+		u64 secs = tspec.tv_nsec;
+
+		tspec.tv_nsec = do_div(secs, NSEC_PER_SEC);
+		tspec.tv_sec += secs;
+	}
+
+	memcpy(ts, &tspec, sizeof(struct timespec64));

But the whole idea is of improving the time by reading tsc a bit later is just weird ... why is it better to provide

Quoted 7 lines

tsc + x, time + tsc_delta_to_time(x)

than just

tsc, time

?

Because you want to calculate the value of the host realtime clock at the moment of ptp_kvm_gettime.

We do:

1. kvm_hypercall.
2. get {sec, nsec, guest_tsc}.
3. kvm_hypercall returns.
4. delay = rdtsc() - guest_tsc.

Where delay is the delta (measured with the TSC) between points 2 and 4.

Quoted 7 lines
Quoted 4 lines

I see now ... the PTP interface is just not good for our purposes. We don't return {sec, nsec, guest_tsc}, we just return {sec, nsec} at some random time in the past. And to make it a bit more accurate, you add a best-effort delta before returning, which makes sense.

Not random time in the past. We return {sec, nsec} from the host realtime at the moment the user ran the hypercall.

That is what we want, but {sec, nsec} is not anchored to any running time, so it is unavoidably late when we return it.

Quoted 19 lines

Since PTP is very accurate, that "a bit more" counts, yes.

Quoted 4 lines

When we have to depend on pvclock, what are the advantages of not using the existing pvclock API for wall clock?
(You mentioned some extensions.)

Quoted 8 lines
  struct pvclock_wall_clock {
  	u32   version;
  	u32   sec;
  	u32   nsec;
  } __attribute__((__packed__));

It gives the wall clock when pvclock was 0, so you just add current kvmclock and get the host wall clock.

Well, no. For one, the TSC part of kvmclock:

	kvmclock-read = system_timestamp + convert-to-1GHz(rdtsc() - tsc_timestamp)
				           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Quoted 7 lines

Drifts relative to UTC. This part can be large.
The guests NTP is responsible for fixing
that drift of the guests realtime clock (talking about current setup, without KVM PTP driver).

Now, we want very high precision (less than 1us) for this driver. Very small TSC drifts on a large delta defeat the purpose.

True.

Quoted 4 lines
Quoted 1 line

Without a VM exit.

Huge performance is not an issue. Accuracy (how different from the host realtime clock our "approximation" of the host realtime clock) is.

Ah, ok.

Quoted 32 lines
Quoted 1 line

And how often is ptp_kvm_gettime() usually called?

The PTP_SYS_OFFSET ioctl calls the following code in a loop:

struct ptp_sys_offset {
unsigned int n_samples; /* Desired number of measurements. */
unsigned int rsv[3]; /* Reserved for future use. */
/*
* Array of interleaved system/phc time stamps. The kernel
* will provide 2*n_samples + 1 time stamps, with the last
* one as a system time stamp.
*/
struct ptp_clock_time ts[2 * PTP_MAX_SAMPLES + 1];
};

#define PTP_MAX_SAMPLES 25 /* Maximum allowed offset measurement samples. */

case PTP_SYS_OFFSET:
sysoff = memdup_user((void __user *)arg,

sizeof(*sysoff));
                if (IS_ERR(sysoff)) {
                        err = PTR_ERR(sysoff);
                        sysoff = NULL;
                        break;
                }
                if (sysoff->n_samples > PTP_MAX_SAMPLES) {
                        err = -EINVAL;
                        break;
                }
                pct = &sysoff->ts[0];
                for (i = 0; i < sysoff->n_samples; i++) {
Quoted 8 lines
                        getnstimeofday64(&ts);
                        pct->sec = ts.tv_sec;
                        pct->nsec = ts.tv_nsec;
                        pct++;
                        ptp->info->gettime64(ptp->info, &ts);
                        pct->sec = ts.tv_sec;
                        pct->nsec = ts.tv_nsec;
                        pct++;

Hm, this loop alternates between system time and ptp time and I'd guess that userspace then manipulates deltas of these two readings and applies the result to system time.
I'm not even sure that pvclock delta improves the situation anymore -- it adds an offset to all gettime64() reads, but that can be computed from multiple datapoints. (And the delta also adds uncertainty to results.)

Quoted 11 lines
                }
                getnstimeofday64(&ts);
                pct->sec = ts.tv_sec;
                pct->nsec = ts.tv_nsec;

How often that ioctl is called depends on the parameters of the Chrony PHC code. Initially (to determine the clock difference Chrony should call it more frequently, later on it should call it less frequency).

Perhaps once every second initially (the ioctl). I'll confirm with the exact value for my setup and reply to this email.

Thanks.

author
#51 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

Quoted 153 lines · Radim Krcmar

2017-01-16 15:04-0200, Marcelo Tosatti:

Quoted 64 lines · Radim Krcmar
Quoted 63 lines

2017-01-13 15:40-0200, Marcelo Tosatti:

Quoted 61 lines · Radim Krcmar
Quoted 41 lines

2017-01-13 10:01-0200, Marcelo Tosatti:

Quoted 26 lines
drivers/ptp/ptp_kvm.c new file +26 -0
+		version = pvclock_read_begin(src);
+
+		ret = kvm_hypercall2(KVM_HC_CLOCK_OFFSET,
+				     clock_off_gpa,
+				     KVM_CLOCK_OFFSET_WALLCLOCK);
+		if (ret != 0) {
+			pr_err("clock offset hypercall ret %lu\n", ret);
+			spin_unlock(&kvm_ptp_lock);
+			preempt_enable_notrace();
+			return -EOPNOTSUPP;
+		}
+
+		tspec.tv_sec = clock_off.sec;
+		tspec.tv_nsec = clock_off.nsec;
+
+		delta = rdtsc_ordered() - clock_off.tsc;
+
+		offset = pvclock_scale_delta(delta, src->tsc_to_system_mul,
+					     src->tsc_shift);
+
+	} while (pvclock_read_retry(src, version));
+
+	preempt_enable_notrace();
+
+	tspec.tv_nsec = tspec.tv_nsec + offset;
+

drivers/ptp/ptp_kvm.c line 116

Quoted 10 lines
drivers/ptp/ptp_kvm.c new file +10 -0
+	spin_unlock(&kvm_ptp_lock);
+
+	if (tspec.tv_nsec >= NSEC_PER_SEC) {
+		u64 secs = tspec.tv_nsec;
+
+		tspec.tv_nsec = do_div(secs, NSEC_PER_SEC);
+		tspec.tv_sec += secs;
+	}
+
+	memcpy(ts, &tspec, sizeof(struct timespec64));

But the whole idea is of improving the time by reading tsc a bit later is just weird ... why is it better to provide

Quoted 7 lines

tsc + x, time + tsc_delta_to_time(x)

than just

tsc, time

?

Because you want to calculate the value of the host realtime clock at the moment of ptp_kvm_gettime.

We do:

1. kvm_hypercall.
2. get {sec, nsec, guest_tsc}.
3. kvm_hypercall returns.
4. delay = rdtsc() - guest_tsc.

Where delay is the delta (measured with the TSC) between points 2 and 4.

Quoted 7 lines
Quoted 4 lines

I see now ... the PTP interface is just not good for our purposes. We don't return {sec, nsec, guest_tsc}, we just return {sec, nsec} at some random time in the past. And to make it a bit more accurate, you add a best-effort delta before returning, which makes sense.

Not random time in the past. We return {sec, nsec} from the host realtime at the moment the user ran the hypercall.

That is what we want, but {sec, nsec} is not anchored to any running time, so it is unavoidably late when we return it.

Quoted 19 lines

Since PTP is very accurate, that "a bit more" counts, yes.

Quoted 4 lines

When we have to depend on pvclock, what are the advantages of not using the existing pvclock API for wall clock?
(You mentioned some extensions.)

Quoted 8 lines
  struct pvclock_wall_clock {
  	u32   version;
  	u32   sec;
  	u32   nsec;
  } __attribute__((__packed__));

It gives the wall clock when pvclock was 0, so you just add current kvmclock and get the host wall clock.

Well, no. For one, the TSC part of kvmclock:

	kvmclock-read = system_timestamp + convert-to-1GHz(rdtsc() - tsc_timestamp)
				           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Quoted 7 lines

Drifts relative to UTC. This part can be large.
The guests NTP is responsible for fixing
that drift of the guests realtime clock (talking about current setup, without KVM PTP driver).

Now, we want very high precision (less than 1us) for this driver. Very small TSC drifts on a large delta defeat the purpose.

True.

Quoted 4 lines
Quoted 1 line

Without a VM exit.

Huge performance is not an issue. Accuracy (how different from the host realtime clock our "approximation" of the host realtime clock) is.

Ah, ok.

Quoted 32 lines
Quoted 1 line

And how often is ptp_kvm_gettime() usually called?

The PTP_SYS_OFFSET ioctl calls the following code in a loop:

struct ptp_sys_offset {
unsigned int n_samples; /* Desired number of measurements. */
unsigned int rsv[3]; /* Reserved for future use. */
/*
* Array of interleaved system/phc time stamps. The kernel
* will provide 2*n_samples + 1 time stamps, with the last
* one as a system time stamp.
*/
struct ptp_clock_time ts[2 * PTP_MAX_SAMPLES + 1];
};

#define PTP_MAX_SAMPLES 25 /* Maximum allowed offset measurement samples. */

case PTP_SYS_OFFSET:
sysoff = memdup_user((void __user *)arg,

sizeof(*sysoff));
                if (IS_ERR(sysoff)) {
                        err = PTR_ERR(sysoff);
                        sysoff = NULL;
                        break;
                }
                if (sysoff->n_samples > PTP_MAX_SAMPLES) {
                        err = -EINVAL;
                        break;
                }
                pct = &sysoff->ts[0];
                for (i = 0; i < sysoff->n_samples; i++) {
Quoted 8 lines
                        getnstimeofday64(&ts);
                        pct->sec = ts.tv_sec;
                        pct->nsec = ts.tv_nsec;
                        pct++;
                        ptp->info->gettime64(ptp->info, &ts);
                        pct->sec = ts.tv_sec;
                        pct->nsec = ts.tv_nsec;
                        pct++;
Quoted 3 lines

Hm, this loop alternates between system time and ptp time and I'd guess that userspace then manipulates deltas of these two readings and applies the result to system time.

Yes.

Quoted 4 lines

I'm not even sure that pvclock delta improves the situation anymore -- it adds an offset to all gettime64() reads, but that can be computed from multiple datapoints. (And the delta also adds uncertainty to results.)

Sorry: the value returned from ->gettime64 should be the value of the clock at that instant.

If you remove the pvclock delta (lets assume its 2000ns), then you'll sync getnstimeofday64 to a clock thats 2000ns behind the host clock.

Does that make sense?

Quoted 5 lines
Quoted 5 lines
                }
                getnstimeofday64(&ts);
                pct->sec = ts.tv_sec;
                pct->nsec = ts.tv_nsec;
Quoted 8 lines
Quoted 6 lines

How often that ioctl is called depends on the parameters of the Chrony PHC code. Initially (to determine the clock difference Chrony should call it more frequently, later on it should call it less frequency).

Perhaps once every second initially (the ioctl). I'll confirm with the exact value for my setup and reply to this email.

Thanks.

Attached. 1551 hypercalls in 38 secs is about 40 times per second. The ioctl is calling ->gettime64 10 per ioctl, so thats about 4 ioctls per sec.

# tracer: nop
#
# entries-in-buffer/entries-written: 1540/1540 #P:4
#
# _-----=> irqs-off
# / _----=> need-resched

#                            | / _---=> hardirq/softirq
#                            || / _--=> preempt-depth
#                            ||| /     delay
#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION
#              | |       |   ||||       |         |
 qemu-system-x86-1630  [000] ....   612.087050: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.087059: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.087061: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.087064: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.087066: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.087068: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.087070: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.087073: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.087075: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.087077: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.337454: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.337461: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.337463: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.337465: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.337467: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.337469: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.337471: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.337473: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.337474: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.337476: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.587820: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.587825: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.587828: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.587829: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.587831: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.587833: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.587835: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.587836: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.587838: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.587840: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.838183: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.838189: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.838191: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.838193: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.838195: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.838197: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.838198: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.838200: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.838202: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   612.838204: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.088535: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.088541: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.088543: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.088545: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.088547: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.088549: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.088551: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.088553: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.088554: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.088579: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.338897: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.338904: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.338907: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.338909: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.338911: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.338913: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.338915: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.338916: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.338918: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.338920: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.589255: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.589260: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.589262: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.589264: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.589266: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.589268: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.589270: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.589272: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.589274: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.589276: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.839626: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.839633: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.839635: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.839637: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.839639: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.839641: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.839642: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.839645: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.839647: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   613.839649: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.089987: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.089993: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.089995: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.089997: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.089999: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.090001: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.090003: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.090005: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.090007: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.090009: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.340343: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.340348: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.340351: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.340352: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.340354: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.340356: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.340358: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.340360: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.340362: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   614.340364: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.590713: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.590719: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.590721: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.590722: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.590724: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.590726: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.590728: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.590730: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.590731: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.590733: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.841063: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.841068: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.841070: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.841072: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.841074: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.841076: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.841078: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.841080: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.841082: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   614.841084: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.091415: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.091420: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.091423: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.091425: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.091426: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.091428: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.091430: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.091432: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.091434: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.091436: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.341680: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.341685: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.341687: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.341689: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.341691: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.341693: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.341695: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.341697: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.341699: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.341701: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.592040: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.592045: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.592047: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.592049: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.592051: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.592052: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.592054: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.592056: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.592058: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.592060: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.842388: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.842393: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.842395: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.842397: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.842399: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.842401: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.842403: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.842405: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.842407: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   615.842409: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.092734: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.092739: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.092741: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.092743: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.092745: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.092746: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.092748: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.092750: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.092752: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.092754: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.343097: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.343104: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.343106: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.343108: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.343110: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.343112: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.343114: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.343116: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.343118: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.343120: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.593453: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.593459: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.593461: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.593463: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.593465: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.593467: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.593469: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.593470: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.593472: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.593474: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.843828: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.843835: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.843837: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.843839: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.843841: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.843843: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.843845: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.843847: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.843849: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   616.843850: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.094184: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.094190: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.094192: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.094194: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.094196: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.094198: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.094200: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.094201: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.094203: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.094205: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.344560: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.344565: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.344568: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.344570: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.344572: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.344574: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.344575: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.344577: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.344579: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.344581: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.594934: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.594939: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.594942: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.594944: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.594945: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.594947: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.594949: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.594951: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.594953: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.594955: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.845286: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.845291: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.845294: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.845296: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.845298: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.845299: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.845301: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.845303: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.845305: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   617.845307: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.095669: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.095674: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.095677: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.095678: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.095680: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.095682: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.095684: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.095685: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.095687: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.095689: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.346029: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.346035: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.346037: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.346038: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.346041: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.346043: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.346045: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.346047: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.346048: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.346050: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.596384: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.596389: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.596391: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.596393: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.596395: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.596397: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.596398: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.596400: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.596402: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.596404: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.846733: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.846739: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.846741: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.846743: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.846745: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.846747: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.846749: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.846751: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.846753: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   618.846755: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.097087: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.097092: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.097095: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.097096: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.097098: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.097100: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.097102: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.097104: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.097106: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.097108: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.347439: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.347445: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.347447: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.347449: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.347451: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.347452: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.347454: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.347456: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.347458: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.347460: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.597810: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.597817: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.597820: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.597823: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.597826: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.597829: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.597832: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.597835: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.597837: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.597840: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.848175: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.848180: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.848182: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.848184: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.848186: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.848188: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.848190: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.848192: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.848193: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   619.848195: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.098579: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.098586: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.098588: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.098590: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.098592: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.098594: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.098595: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.098597: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.098599: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.098601: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.348942: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.348948: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.348950: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.348952: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.348953: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.348955: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.348957: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.348959: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.348960: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.348962: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.599294: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.599300: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.599302: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.599304: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.599306: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.599308: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.599310: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.599312: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.599313: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.599315: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.849646: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.849652: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.849654: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.849656: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.849658: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.849660: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.849662: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.849664: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.849666: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   620.849667: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.100009: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.100016: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.100019: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.100021: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.100022: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.100024: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.100026: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.100028: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.100030: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.100032: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.350363: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.350369: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.350371: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.350373: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.350375: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.350377: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.350379: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.350381: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.350382: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.350384: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.600744: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.600749: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.600752: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.600753: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.600755: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.600757: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.600759: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.600760: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.600762: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.600764: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.851104: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.851110: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.851112: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.851114: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.851116: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.851118: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.851120: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.851122: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.851124: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   621.851126: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.101458: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.101464: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.101466: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.101468: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.101470: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.101472: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.101474: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.101476: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.101477: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.101479: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.351816: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.351822: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.351824: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.351826: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.351828: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.351830: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.351832: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.351834: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.351836: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.351838: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.602175: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.602181: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.602183: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.602185: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.602187: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.602188: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.602190: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.602192: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.602194: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.602196: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.852527: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.852533: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.852535: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.852537: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.852539: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.852541: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.852544: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.852547: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.852550: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   622.852553: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.102887: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.102892: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.102895: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.102897: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.102898: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.102900: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.102902: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.102904: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.102906: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.102908: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.353282: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.353289: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.353291: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.353293: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.353295: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.353297: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.353299: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.353300: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.353302: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.353304: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.603663: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.603669: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.603671: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.603673: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.603675: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.603677: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.603678: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.603680: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.603682: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.603684: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.854015: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.854021: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.854023: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.854025: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.854027: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.854029: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.854031: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.854033: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.854035: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   623.854037: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.104363: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.104369: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.104371: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.104373: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.104374: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.104376: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.104378: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.104380: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.104382: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.104383: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.354724: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.354731: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.354733: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.354735: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.354737: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.354739: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.354741: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.354742: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.354744: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   624.354746: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   624.605113: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   624.605120: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   624.605122: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   624.605124: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   624.605126: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   624.605128: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   624.605130: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   624.605132: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   624.605134: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   624.605136: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   624.855482: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   624.855488: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   624.855490: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   624.855491: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   624.855493: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   624.855495: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   624.855497: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   624.855498: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   624.855500: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   624.855502: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.105843: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.105848: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.105850: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.105852: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.105854: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.105856: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.105857: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.105859: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.105861: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.105863: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.356218: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.356225: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.356229: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.356232: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.356235: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.356238: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.356241: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.356244: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.356247: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.356250: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.606599: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.606606: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.606608: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.606610: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.606612: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.606614: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.606616: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.606618: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.606620: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.606621: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.856960: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.856966: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.856968: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.856970: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.856972: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.856974: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.856976: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.856978: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.856979: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   625.856981: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.107314: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.107319: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.107321: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.107323: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.107326: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.107328: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.107330: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.107332: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.107334: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.107335: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.357688: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.357693: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.357695: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.357697: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.357699: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.357701: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.357703: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.357705: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.357707: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.357708: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.608042: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.608048: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.608050: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.608052: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.608054: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.608056: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.608058: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.608059: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.608061: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.608063: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.858408: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.858417: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.858421: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.858423: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.858435: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.858437: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.858440: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.858442: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.858445: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   626.858447: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.108799: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.108829: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.108831: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.108833: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.108835: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.108837: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.108839: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.108840: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.108842: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.108844: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.359180: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.359186: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.359188: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.359190: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.359192: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.359194: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.359195: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.359197: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.359199: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.359201: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.609531: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.609537: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.609539: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.609541: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.609543: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.609545: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.609547: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.609549: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.609551: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.609553: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.859886: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.859891: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.859894: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.859895: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.859897: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.859899: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.859901: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.859903: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.859905: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   627.859907: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.110254: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.110260: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.110262: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.110264: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.110266: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.110268: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.110270: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.110271: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.110273: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.110275: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.360604: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.360609: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.360611: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.360613: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.360615: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.360617: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.360619: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.360621: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.360623: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.360625: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.610959: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.610965: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.610967: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.610969: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.610971: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.610973: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.610975: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.610977: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.610978: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.610980: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.861312: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.861317: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.861319: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.861321: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.861323: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.861325: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.861336: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.861338: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.861340: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   628.861341: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.111675: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.111689: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.111692: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.111693: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.111695: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.111697: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.111699: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.111701: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.111703: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.111705: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.362076: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.362082: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.362084: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.362086: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.362088: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.362090: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.362092: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.362094: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.362095: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.362097: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.612427: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.612433: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.612435: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.612437: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.612439: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.612441: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.612443: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.612444: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.612446: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.612448: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.862800: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.862806: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.862808: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.862810: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.862812: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.862814: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.862816: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.862818: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.862820: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   629.862822: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.113174: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.113180: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.113182: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.113184: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.113186: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.113188: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.113190: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.113191: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.113193: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.113195: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.363529: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.363535: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.363537: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.363539: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.363541: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.363543: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.363544: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.363546: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.363548: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.363550: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.613888: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.613893: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.613895: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.613897: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.613899: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.613901: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.613902: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.613904: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.613906: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.613908: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.864240: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.864246: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.864248: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.864250: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.864252: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.864254: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.864256: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.864257: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.864259: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   630.864261: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.114595: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.114601: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.114604: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.114606: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.114607: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.114609: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.114611: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.114613: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.114615: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.114617: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.364951: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.364956: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.364958: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.364960: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.364962: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.364964: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.364966: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.364968: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.364970: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.364972: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.615311: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.615317: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.615319: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.615321: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.615323: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.615325: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.615326: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.615328: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.615330: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.615332: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.865673: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.865679: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.865681: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.865683: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.865684: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.865686: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.865688: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.865690: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.865692: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   631.865693: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.116033: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.116039: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.116041: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.116043: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.116044: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.116046: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.116048: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.116050: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.116052: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.116053: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.366386: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.366401: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.366404: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.366407: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.366409: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.366412: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.366414: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.366417: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.366419: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.366423: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.616752: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.616758: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.616760: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.616762: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.616764: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.616765: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.616767: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.616769: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.616771: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.616797: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.867134: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.867140: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.867142: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.867144: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.867146: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.867148: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.867150: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.867152: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.867153: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   632.867155: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.117488: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.117494: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.117496: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.117498: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.117500: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.117502: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.117504: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.117506: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.117508: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.117510: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.367842: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.367847: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.367849: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.367851: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.367853: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.367855: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.367857: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.367859: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.367861: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.367863: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.618195: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.618201: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.618203: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.618205: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.618207: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.618208: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.618210: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.618212: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.618214: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.618216: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.868549: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.868555: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.868557: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.868559: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.868561: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.868563: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.868565: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.868566: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.868568: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   633.868570: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.118902: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.118908: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.118910: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.118912: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.118914: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.118916: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.118917: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.118919: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.118921: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.118939: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.369273: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.369279: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.369281: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.369283: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.369285: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.369287: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.369288: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.369290: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.369292: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.369294: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.619630: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.619636: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.619638: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.619640: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.619641: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.619643: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.619645: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.619647: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.619648: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.619650: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.869986: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.869991: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.869993: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.869995: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.869997: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.869999: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.870001: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.870002: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.870004: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   634.870006: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.120333: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.120338: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.120341: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.120343: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.120345: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.120347: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.120349: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.120351: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.120353: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.120354: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.370689: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.370694: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.370696: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.370698: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.370699: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.370701: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.370703: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.370704: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.370706: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.370708: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.621042: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.621048: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.621050: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.621052: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.621054: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.621056: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.621057: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.621059: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.621061: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.621063: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.871399: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.871406: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.871408: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.871410: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.871412: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.871414: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.871416: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.871417: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.871419: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   635.871421: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.121877: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.121882: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.121884: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.121886: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.121888: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.121890: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.121892: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.121894: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.121896: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.121898: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.372233: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.372239: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.372241: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.372243: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.372245: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.372247: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.372249: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.372250: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.372252: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.372254: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.622596: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.622602: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.622605: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.622607: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.622608: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.622610: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.622612: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.622614: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.622616: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.622618: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.872756: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.872763: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.872765: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.872767: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.872768: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.872770: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.872772: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.872774: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.872775: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   636.872778: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.123108: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.123114: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.123116: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.123118: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.123120: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.123122: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.123124: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.123125: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.123127: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.123129: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.373466: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.373471: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.373474: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.373476: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.373477: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.373479: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.373481: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.373483: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.373485: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.373487: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.623820: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.623825: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.623828: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.623830: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.623831: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.623833: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.623835: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.623837: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.623839: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.623841: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.873968: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.873974: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.873976: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.873978: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.873980: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.873982: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.873984: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.873986: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.873987: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   637.873989: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.124348: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.124355: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.124357: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.124359: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.124361: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.124363: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.124364: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.124366: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.124368: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.124370: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.374708: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.374713: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.374715: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.374717: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.374719: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.374721: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.374723: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.374724: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.374726: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.374728: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.625083: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.625089: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.625091: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.625093: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.625095: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.625096: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.625098: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.625100: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.625102: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.625103: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.875444: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.875450: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.875452: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.875454: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.875456: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.875458: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.875460: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.875462: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.875464: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   638.875466: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.125797: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.125803: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.125805: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.125807: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.125809: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.125810: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.125812: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.125814: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.125816: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.125818: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.376152: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.376158: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.376160: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.376162: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.376164: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.376166: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.376167: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.376169: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.376171: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.376173: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.626513: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.626519: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.626522: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.626524: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.626526: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.626527: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.626529: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.626531: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.626533: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.626535: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.876872: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.876879: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.876881: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.876883: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.876885: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.876887: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.876889: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.876891: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.876893: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   639.876894: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.127054: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.127061: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.127063: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.127065: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.127067: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.127069: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.127071: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.127073: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.127075: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.127076: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.377413: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.377420: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.377422: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.377424: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.377426: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.377428: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.377430: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.377432: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.377434: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.377436: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.627783: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.627790: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.627792: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.627794: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.627795: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.627797: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.627799: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.627801: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.627803: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.627805: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.878141: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.878148: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.878150: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.878153: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.878155: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.878157: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.878159: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.878160: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.878162: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   640.878164: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.128511: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.128518: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.128520: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.128522: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.128523: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.128525: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.128527: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.128529: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.128531: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.128533: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.378877: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.378884: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.378886: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.378888: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.378890: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.378892: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.378894: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.378896: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.378898: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.378899: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.629247: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.629254: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.629256: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.629258: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.629260: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.629262: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.629264: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.629266: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.629268: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.629269: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.879608: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.879615: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.879617: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.879619: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.879621: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.879622: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.879624: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.879626: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.879628: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   641.879630: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.129989: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.129996: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.129999: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.130000: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.130002: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.130004: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.130006: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.130008: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.130010: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.130012: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.380341: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.380347: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.380350: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.380352: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.380353: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.380355: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.380358: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.380361: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.380364: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.380367: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.630714: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.630720: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.630722: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.630724: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.630726: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.630728: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.630730: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.630732: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.630733: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.630735: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.881079: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.881099: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.881102: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.881104: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.881105: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.881107: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.881109: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.881111: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.881113: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   642.881115: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.131458: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.131466: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.131468: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.131469: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.131471: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.131473: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.131475: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.131477: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.131478: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.131480: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.381826: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.381832: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.381834: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.381836: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.381838: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.381840: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.381842: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.381843: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.381845: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   643.381847: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.632194: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.632201: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.632203: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.632205: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.632207: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.632209: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.632211: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.632212: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.632214: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.632216: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.882559: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.882566: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.882568: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.882570: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.882572: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.882573: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.882575: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.882577: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.882579: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   643.882581: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.132948: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.132955: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.132957: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.132959: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.132961: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.132963: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.132965: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.132967: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.132969: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.132971: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.383329: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.383335: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.383337: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.383339: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.383341: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.383343: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.383345: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.383347: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.383349: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.383351: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.633692: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.633698: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.633701: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.633702: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.633704: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.633706: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.633708: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.633710: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.633712: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.633714: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.883841: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.883850: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.883853: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.883856: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.883859: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.883862: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.883865: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.883875: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.883878: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   644.883880: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.134215: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.134222: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.134224: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.134226: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.134228: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.134229: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.134231: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.134234: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.134236: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.134238: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.384588: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.384594: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.384597: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.384598: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.384600: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.384602: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.384604: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.384605: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.384607: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.384609: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.634936: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.634942: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.634944: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.634946: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.634948: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.634950: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.634952: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.634954: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.634956: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.634958: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.885302: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.885309: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.885311: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.885313: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.885315: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.885317: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.885318: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.885320: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.885322: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   645.885324: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.135660: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.135666: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.135668: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.135670: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.135672: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.135674: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.135676: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.135678: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.135679: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.135681: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.386038: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.386044: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.386046: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.386048: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.386050: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.386052: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.386054: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.386056: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.386058: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.386059: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.636394: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.636400: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.636402: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.636404: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.636406: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.636407: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.636409: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.636411: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.636413: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.636415: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.886757: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.886763: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.886765: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.886767: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.886769: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.886771: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.886772: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.886774: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.886776: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   646.886778: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   647.137115: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   647.137121: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   647.137124: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   647.137125: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   647.137128: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   647.137130: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   647.137132: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   647.137134: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   647.137136: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [003] ....   647.137137: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   647.387460: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   647.387466: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   647.387468: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   647.387470: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   647.387472: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   647.387473: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   647.387475: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   647.387477: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   647.387479: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [000] ....   647.387480: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.637820: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.637826: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.637828: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.637829: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.637831: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.637833: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.637835: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.637837: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.637838: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.637840: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.888197: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.888203: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.888205: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.888207: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.888209: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.888211: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.888212: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.888214: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.888216: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   647.888218: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.138555: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.138560: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.138562: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.138564: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.138566: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.138568: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.138570: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.138571: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.138573: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.138575: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.388911: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.388918: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.388920: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.388922: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.388924: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.388926: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.388928: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.388930: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.388931: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.388933: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.639263: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.639269: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.639271: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.639273: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.639274: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.639276: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.639278: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.639280: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.639282: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.639284: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.889619: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.889625: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.889627: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.889629: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.889631: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.889633: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.889635: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.889637: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.889638: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   648.889640: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.139973: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.139979: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.139981: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.139983: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.139985: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.139987: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.139989: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.139990: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.139992: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.139994: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.390329: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.390334: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.390336: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.390338: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.390340: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.390342: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.390344: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.390346: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.390348: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.390350: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.640699: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.640705: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.640710: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.640712: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.640714: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.640715: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.640717: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.640719: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.640730: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.640732: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.891064: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.891069: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.891072: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.891093: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.891096: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.891098: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.891100: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.891102: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.891103: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [001] ....   649.891105: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.141454: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.141460: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.141462: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.141464: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.141466: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.141468: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.141469: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.141471: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.141473: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.141475: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.391813: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.391818: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.391820: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.391822: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.391823: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.391825: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.391827: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.391829: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.391830: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
 qemu-system-x86-1630  [002] ....   650.391832: kvm_hypercall: nr 0x9 a0 0x3725ff20 a1 0x0 a2 0x6 a3 0xffffb75e40e7bca8
maintainer
#52 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

In reply to Radim Krcmar
Quoted 4 lines · Radim Krcmar
Quoted 4 lines
drivers/ptp/ptp_kvm.c new file +4 -0
+static int __init ptp_kvm_init(void)
+{
+	if (!kvm_para_available())
+		return -ENODEV;
Quoted 8 lines

drivers/ptp/ptp_kvm.c line 163

Quoted 4 lines
drivers/ptp/ptp_kvm.c new file +4 -0
+
+	kvm_ptp_clock.caps = ptp_kvm_caps;
+
+	kvm_ptp_clock.ptp_clock = ptp_clock_register(&kvm_ptp_clock.caps, NULL);

It is a shame that the infrastructure uses polling when the guest could be notified on every host real time change, but this should be good enough.

This comment makes no sense at all. What do you mean by "host real time change"?

Thanks,
Richard

#53 lore raw

Re: [patch 3/3] PTP: add kvm PTP driver

2017-01-14 16:26+0100, Richard Cochran:

Quoted 14 lines · Radim Krcmar
Quoted 4 lines
Quoted 4 lines
drivers/ptp/ptp_kvm.c new file +4 -0
+static int __init ptp_kvm_init(void)
+{
+	if (!kvm_para_available())
+		return -ENODEV;
Quoted 8 lines

drivers/ptp/ptp_kvm.c line 163

Quoted 4 lines
drivers/ptp/ptp_kvm.c new file +4 -0
+
+	kvm_ptp_clock.caps = ptp_kvm_caps;
+
+	kvm_ptp_clock.ptp_clock = ptp_clock_register(&kvm_ptp_clock.caps, NULL);

It is a shame that the infrastructure uses polling when the guest could be notified on every host real time change, but this should be good enough.

Quoted 2 lines

This comment makes no sense at all. What do you mean by "host real time change"?

Real time in the sense of wall clock, as perceived by the host, and the change of that time.

Unlike other PTP drivers, host (source) and guest (destination) share the same hardware clock, so they cannot shift or drift unless one of them changes its "TSC to real time" conversion (the host is most likely using NTP/PTP to keep its own real time).

I meant that the host could notify the guest when a change happens, which would be more efficient.