KVM virtual PTP driver (v5)

Expose the realtime host clock and save the TSC value used for the clock calculation.

Signed-off-by Marcelo Tosatti <[email protected]>

---
 arch/x86/kvm/x86.c |   36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

v2: unify nsec_base (Radim)

Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c +29 -0

@@ -1139,6 +1139,7 @@
 
 	u64		boot_ns;
 	u64		nsec_base;
+	u64		wall_time_sec;
 };
 
 static struct pvclock_gtod_data pvclock_gtod_data;

@@ -1162,6 +1163,8 @@
 	vdata->boot_ns			= boot_ns;
 	vdata->nsec_base		= tk->tkr_mono.xtime_nsec;
 
+	vdata->wall_time_sec            = tk->xtime_sec;
+
 	write_seqcount_end(&vdata->seq);
 }
 #endif

@@ -1623,6 +1626,28 @@
 	return mode;
 }
 
+static int do_realtime(struct timespec *ts, cycle_t *cycle_now)
+{
+	struct pvclock_gtod_data *gtod = &pvclock_gtod_data;
+	unsigned long seq;
+	int mode;
+	u64 ns;
+
+	do {
+		seq = read_seqcount_begin(&gtod->seq);
+		mode = gtod->clock.vclock_mode;
+		ts->tv_sec = gtod->wall_time_sec;
+		ns = gtod->nsec_base;
+		ns += vgettsc(cycle_now);
+		ns >>= gtod->clock.shift;
+	} while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
+
+	ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
+	ts->tv_nsec = ns;
+
+	return mode;
+}
+
 /* returns true if host is using tsc clocksource */
 static bool kvm_get_time_and_clockread(s64 *kernel_ns, cycle_t *cycle_now)
 {

@@ -1632,6 +1657,17 @@
 
 	return do_monotonic_boot(kernel_ns, cycle_now) == VCLOCK_TSC;
 }
+
+/* returns true if host is using tsc clocksource */
+static bool kvm_get_walltime_and_clockread(struct timespec *ts,
+					   cycle_t *cycle_now)

Add a hypercall to retrieve the host realtime clock and the TSC value used to calculate that clock read.

Used to implement clock synchronization between
host and guest.

Signed-off-by Marcelo Tosatti <[email protected]>

---
 Documentation/virtual/kvm/hypercalls.txt |   33 ++++++++++++++++++++++++
 arch/x86/include/uapi/asm/kvm_para.h     |    9 ++++++
 arch/x86/kvm/x86.c                       |   41 +++++++++++++++++++++++++++++++
 include/uapi/linux/kvm_para.h            |    3 ++
 4 files changed, 86 insertions(+)

v2: improve documentation (Radim)
change hypercall name to KVM_HC_CLOCK_PAIRING (Radim) increase padding size

Index: kvm-ptpdriver/arch/x86/include/uapi/asm/kvm_para.h
===================================================================

arch/x86/include/uapi/asm/kvm_para.h +7 -0

+	__s64 sec;
+	__s64 nsec;
+	__u64 tsc;
+	__u32 flags;
+	__u32 pad[9];
+};
+
 #define KVM_STEAL_ALIGNMENT_BITS 5
 #define KVM_STEAL_VALID_BITS ((-1ULL << (KVM_STEAL_ALIGNMENT_BITS + 1)))
 #define KVM_STEAL_RESERVED_MASK (((1 << KVM_STEAL_ALIGNMENT_BITS) - 1 ) << 1)

Index: kvm-ptpdriver/Documentation/virtual/kvm/hypercalls.txt
===================================================================

Documentation/virtual/kvm/hypercalls.txt +6 -0

@@ -81,3 +81,36 @@
 same guest can wakeup the sleeping vcpu by issuing KVM_HC_KICK_CPU hypercall,
 specifying APIC ID (a1) of the vcpu to be woken up. An additional argument (a0)
 is used in the hypercall for future use.
+
+
+6. KVM_HC_CLOCK_PAIRING
+------------------------
+Architecture: x86
+Status: active

Documentation/virtual/kvm/hypercalls.txt +5 -0

+               * flags: flags, unused (0) at the moment.
+
+Returns KVM_EOPNOTSUPP if the host does not use TSC clocksource,
+or if clock type is different than KVM_CLOCK_PAIRING_WALLCLOCK.
+

Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c +3 -0

@@ -6119,6 +6119,44 @@
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
 
+static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
+			       unsigned long clock_type)
+{

arch/x86/kvm/x86.c +26 -0

+	clock_offset = kzalloc(sizeof(struct kvm_clock_offset), GFP_KERNEL);
+	if (clock_offset == NULL)
+		return -KVM_ENOMEM;
+
+	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false) {
+		kfree(clock_offset);
+		return -KVM_EOPNOTSUPP;
+	}
+	clock_offset->sec = ts.tv_sec;
+	clock_offset->nsec = ts.tv_nsec;
+	clock_offset->tsc = kvm_read_l1_tsc(vcpu, cycle);
+	clock_offset->flags = 0;
+
+	ret = 0;
+	if (kvm_write_guest(vcpu->kvm, paddr, clock_offset,
+			      sizeof(struct kvm_clock_offset)))
+		ret = -KVM_EFAULT;
+
+	kfree(clock_offset);
+
+	return ret;
+}
+
 /*
  * kvm_pv_kick_cpu_op:  Kick a vcpu.
  *

@@ -6183,6 +6221,9 @@
 		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
 		ret = 0;
 		break;
+	case KVM_HC_CLOCK_PAIRING:
+		ret = kvm_pv_clock_pairing(vcpu, a0, a1);
+		break;
 	default:
 		ret = -KVM_ENOSYS;
 		break;

Index: kvm-ptpdriver/include/uapi/linux/kvm_para.h
===================================================================

include/uapi/linux/kvm_para.h +1 -0

 #define KVM_HC_VAPIC_POLL_IRQ		1
 #define KVM_HC_MMU_OP			2

@@ -23,6 +25,7 @@
 #define KVM_HC_MIPS_GET_CLOCK_FREQ	6
 #define KVM_HC_MIPS_EXIT_VM		7
 #define KVM_HC_MIPS_CONSOLE_OUTPUT	8
+#define KVM_HC_CLOCK_PAIRING		9
 
 /*
  * hypercalls use architecture specific

Quoted 24 lines · Marcelo Tosatti

Add a hypercall to retrieve the host realtime clock and the TSC value used to calculate that clock read.

Used to implement clock synchronization between
host and guest.

Signed-off-by Marcelo Tosatti <[email protected]>

---
 Documentation/virtual/kvm/hypercalls.txt |   33 ++++++++++++++++++++++++
 arch/x86/include/uapi/asm/kvm_para.h     |    9 ++++++
 arch/x86/kvm/x86.c                       |   41 +++++++++++++++++++++++++++++++
 include/uapi/linux/kvm_para.h            |    3 ++
 4 files changed, 86 insertions(+)

v2: improve documentation (Radim)
change hypercall name to KVM_HC_CLOCK_PAIRING (Radim) increase padding size

Index: kvm-ptpdriver/arch/x86/include/uapi/asm/kvm_para.h
===================================================================

arch/x86/include/uapi/asm/kvm_para.h line 54

Quoted 5 lines

arch/x86/include/uapi/asm/kvm_para.h +2 -0

@@ -50,6 +50,15 @@
 	__u32 pad[11];
 };
 
+#define KVM_CLOCK_PAIRING_WALLCLOCK 0
+struct kvm_clock_offset {

The struct still has the old name.

arch/x86/include/uapi/asm/kvm_para.h line 60

Quoted 6 lines

arch/x86/include/uapi/asm/kvm_para.h +6 -0

+	__s64 sec;
+	__s64 nsec;
+	__u64 tsc;
+	__u32 flags;
+	__u32 pad[9];
+};

Quoted 17 lines

arch/x86/include/uapi/asm/kvm_para.h +0 -0

 #define KVM_STEAL_ALIGNMENT_BITS 5
 #define KVM_STEAL_VALID_BITS ((-1ULL << (KVM_STEAL_ALIGNMENT_BITS + 1)))
 #define KVM_STEAL_RESERVED_MASK (((1 << KVM_STEAL_ALIGNMENT_BITS) - 1 ) << 1)

Index: kvm-ptpdriver/Documentation/virtual/kvm/hypercalls.txt
===================================================================

Documentation/virtual/kvm/hypercalls.txt +6 -0

@@ -81,3 +81,36 @@
 same guest can wakeup the sleeping vcpu by issuing KVM_HC_KICK_CPU hypercall,
 specifying APIC ID (a1) of the vcpu to be woken up. An additional argument (a0)
 is used in the hypercall for future use.
+
+
+6. KVM_HC_CLOCK_PAIRING
+------------------------
+Architecture: x86
+Status: active

Documentation/virtual/kvm/hypercalls.txt line 99

Quoted 10 lines

Documentation/virtual/kvm/hypercalls.txt +10 -0

+Purpose: Hypercall used to synchronize host and guest clocks.
+Usage:
+
+a0: guest physical address where host copies
+"struct kvm_clock_offset" structure.
+
+a1: clock_type, ATM only KVM_CLOCK_PAIRING_WALLCLOCK (0)
+is supported (hosts CLOCK_REALTIME clock).
+
+		struct kvm_clock_offset {

Name of the struct not adjusted.

Documentation/virtual/kvm/hypercalls.txt line 104

Quoted 5 lines

Documentation/virtual/kvm/hypercalls.txt +5 -0

+			__s64 sec;
+			__s64 nsec;
+			__u64 tsc;
+			__u32 flags;
+			__u32 pad[9];

I'd just leave one element of padding and put the struct on the stack, since we have no reason to think we'll need a lot of padding.

Documentation/virtual/kvm/hypercalls.txt line 111

Quoted 7 lines

Documentation/virtual/kvm/hypercalls.txt +7 -0

+		};
+
+       Where:
+               * sec: seconds from clock_type clock.
+               * nsec: nanoseconds from clock_type clock.
+               * tsc: TSC value used to calculate sec/nsec pair
+                 (this hypercall only works when host uses TSC clocksource).

This is already mentioned below, and it's not clear that tsc is a guest TSC value.

Quoted 13 lines

Documentation/virtual/kvm/hypercalls.txt +5 -0

+               * flags: flags, unused (0) at the moment.
+
+Returns KVM_EOPNOTSUPP if the host does not use TSC clocksource,
+or if clock type is different than KVM_CLOCK_PAIRING_WALLCLOCK.
+

Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c +0 -0

@@ -6119,6 +6119,44 @@
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
 

arch/x86/kvm/x86.c line 6134

Quoted 13 lines

arch/x86/kvm/x86.c +13 -0

+static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
+			       unsigned long clock_type)
+{
+	struct kvm_clock_offset *clock_offset;
+	struct timespec ts;
+	cycle_t cycle;
+	int ret;
+
+	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
+		return -KVM_EOPNOTSUPP;
+
+	if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
+		return -KVM_EOPNOTSUPP;

Already checked in kvm_get_walltime_and_clockread.

Quoted 42 lines

arch/x86/kvm/x86.c +26 -0

+	clock_offset = kzalloc(sizeof(struct kvm_clock_offset), GFP_KERNEL);
+	if (clock_offset == NULL)
+		return -KVM_ENOMEM;
+
+	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false) {
+		kfree(clock_offset);
+		return -KVM_EOPNOTSUPP;
+	}
+	clock_offset->sec = ts.tv_sec;
+	clock_offset->nsec = ts.tv_nsec;
+	clock_offset->tsc = kvm_read_l1_tsc(vcpu, cycle);
+	clock_offset->flags = 0;
+
+	ret = 0;
+	if (kvm_write_guest(vcpu->kvm, paddr, clock_offset,
+			      sizeof(struct kvm_clock_offset)))
+		ret = -KVM_EFAULT;
+
+	kfree(clock_offset);
+
+	return ret;
+}
+
 /*
  * kvm_pv_kick_cpu_op:  Kick a vcpu.
  *

@@ -6183,6 +6221,9 @@
 		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
 		ret = 0;
 		break;
+	case KVM_HC_CLOCK_PAIRING:
+		ret = kvm_pv_clock_pairing(vcpu, a0, a1);
+		break;
 	default:
 		ret = -KVM_ENOSYS;
 		break;

Index: kvm-ptpdriver/include/uapi/linux/kvm_para.h
===================================================================

include/uapi/linux/kvm_para.h +0 -0

@@ -14,6 +14,8 @@
 #define KVM_EFAULT		EFAULT

include/uapi/linux/kvm_para.h

Quoted 4 lines

include/uapi/linux/kvm_para.h +2 -0

 #define KVM_E2BIG		E2BIG
 #define KVM_EPERM		EPERM
+#define KVM_EOPNOTSUPP		EOPNOTSUPP
+#define KVM_ENOMEM		ENOMEM

This file could potentially be used across multiple architectures (it's not in arch/x86/include/uapi/asm), but EOPNOTSUPP is not the same on all systems. Better use "95" which is the value it has on x86.

All this gives:

Documentation/virtual/kvm/hypercalls.txt +7 -5

index 55eb3e1ca494..cd35de84999f 100644
@@ -94,9 +94,9 @@ a0: guest physical address where host copies
 "struct kvm_clock_offset" structure.
 
 a1: clock_type, ATM only KVM_CLOCK_PAIRING_WALLCLOCK (0)
-is supported (hosts CLOCK_REALTIME clock).
+is supported (corresponding to the host's CLOCK_REALTIME clock).
 
-		struct kvm_clock_offset {
+		struct kvm_clock_pairing {
 			__s64 sec;
 			__s64 nsec;
 			__u64 tsc;

@@ -107,10 +107,12 @@ is supported (hosts CLOCK_REALTIME clock).
        Where:
                * sec: seconds from clock_type clock.
                * nsec: nanoseconds from clock_type clock.
-               * tsc: TSC value used to calculate sec/nsec pair
-                 (this hypercall only works when host uses TSC clocksource).
+               * tsc: guest TSC value used to calculate sec/nsec pair
                * flags: flags, unused (0) at the moment.
 
+The hypercall lets a guest compute a precise timestamp across
+host and guest.  The guest can use the returned TSC value to
+compute the CLOCK_REALTIME for its clock, at the same instant.
+
 Returns KVM_EOPNOTSUPP if the host does not use TSC clocksource,
 or if clock type is different than KVM_CLOCK_PAIRING_WALLCLOCK.
-

arch/x86/include/uapi/asm/kvm_para.h +2 -2

index ce7b411e28bc..3641d8b7dba9 100644
@@ -51,12 +51,12 @@ struct kvm_steal_time {
 };
 
 #define KVM_CLOCK_PAIRING_WALLCLOCK 0
-struct kvm_clock_offset {
+struct kvm_clock_pairing {
 	__s64 sec;
 	__s64 nsec;
 	__u64 tsc;
 	__u32 flags;
-	__u32 pad[9];
+	__u32 pad[1];
 };
 
 #define KVM_STEAL_ALIGNMENT_BITS 5

arch/x86/kvm/x86.c +9 -20

index 4ecc3145ea4b..09e5d31dac98 100644
@@ -6149,9 +6149,9 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu)
 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
 
 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
-			       unsigned long clock_type)
+			        unsigned long clock_type)
 {
-	struct kvm_clock_offset *clock_offset;
+	struct kvm_clock_pairing clock_pairing;
 	struct timespec ts;
 	cycle_t cycle;
 	int ret;

@@ -6159,30 +6159,19 @@ static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
 	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
 		return -KVM_EOPNOTSUPP;
 
-	if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
-		return -KVM_EOPNOTSUPP;
-
-	clock_offset = kzalloc(sizeof(struct kvm_clock_offset), GFP_KERNEL);
-	if (clock_offset == NULL)
-		return -KVM_ENOMEM;
-
-	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false) {
-		kfree(clock_offset);
+	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false)
 		return -KVM_EOPNOTSUPP;
-	}
 
-	clock_offset->sec = ts.tv_sec;
-	clock_offset->nsec = ts.tv_nsec;
-	clock_offset->tsc = kvm_read_l1_tsc(vcpu, cycle);
-	clock_offset->flags = 0;
+	clock_pairing.sec = ts.tv_sec;
+	clock_pairing.nsec = ts.tv_nsec;
+	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
+	clock_pairing.flags = 0;
 
 	ret = 0;
-	if (kvm_write_guest(vcpu->kvm, paddr, clock_offset,
-			      sizeof(struct kvm_clock_offset)))
+	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
+			    sizeof(struct kvm_clock_pairing)))
 		ret = -KVM_EFAULT;
 
-	kfree(clock_offset);
-
 	return ret;
 }
 

include/uapi/linux/kvm_para.h +1 -1

index 68da60e7f519..c72128527377 100644
@@ -14,7 +14,7 @@
 #define KVM_EFAULT		EFAULT
 #define KVM_E2BIG		E2BIG
 #define KVM_EPERM		EPERM
-#define KVM_EOPNOTSUPP		EOPNOTSUPP
+#define KVM_EOPNOTSUPP		95
 #define KVM_ENOMEM		ENOMEM
 
 #define KVM_HC_VAPIC_POLL_IRQ		1

Okay?

Paolo

Quoted 13 lines

include/uapi/linux/kvm_para.h +1 -0

 #define KVM_HC_VAPIC_POLL_IRQ		1
 #define KVM_HC_MMU_OP			2

@@ -23,6 +25,7 @@
 #define KVM_HC_MIPS_GET_CLOCK_FREQ	6
 #define KVM_HC_MIPS_EXIT_VM		7
 #define KVM_HC_MIPS_CONSOLE_OUTPUT	8
+#define KVM_HC_CLOCK_PAIRING		9
 
 /*
  * hypercalls use architecture specific

Quoted 35 lines · Paolo Bonzini

Quoted 24 lines · Marcelo Tosatti

Add a hypercall to retrieve the host realtime clock and the TSC value used to calculate that clock read.

Used to implement clock synchronization between
host and guest.

Signed-off-by Marcelo Tosatti <[email protected]>

---
 Documentation/virtual/kvm/hypercalls.txt |   33 ++++++++++++++++++++++++
 arch/x86/include/uapi/asm/kvm_para.h     |    9 ++++++
 arch/x86/kvm/x86.c                       |   41 +++++++++++++++++++++++++++++++
 include/uapi/linux/kvm_para.h            |    3 ++
 4 files changed, 86 insertions(+)

v2: improve documentation (Radim)
change hypercall name to KVM_HC_CLOCK_PAIRING (Radim) increase padding size

Index: kvm-ptpdriver/arch/x86/include/uapi/asm/kvm_para.h
===================================================================

arch/x86/include/uapi/asm/kvm_para.h line 54

Quoted 5 lines

arch/x86/include/uapi/asm/kvm_para.h +2 -0

@@ -50,6 +50,15 @@
 	__u32 pad[11];
 };
 
+#define KVM_CLOCK_PAIRING_WALLCLOCK 0
+struct kvm_clock_offset {

The struct still has the old name.

Quoted 6 lines

arch/x86/include/uapi/asm/kvm_para.h line 60

Quoted 6 lines

arch/x86/include/uapi/asm/kvm_para.h +6 -0

+	__s64 sec;
+	__s64 nsec;
+	__u64 tsc;
+	__u32 flags;
+	__u32 pad[9];
+};

That was on purpose: "to pair" is the operation.

"struct kvm_clock_pairing" makes no sense to me
(because pairing is an "action").

(if you'd prefer that name, let me know).

Quoted 29 lines

Quoted 17 lines

arch/x86/include/uapi/asm/kvm_para.h +0 -0

 #define KVM_STEAL_ALIGNMENT_BITS 5
 #define KVM_STEAL_VALID_BITS ((-1ULL << (KVM_STEAL_ALIGNMENT_BITS + 1)))
 #define KVM_STEAL_RESERVED_MASK (((1 << KVM_STEAL_ALIGNMENT_BITS) - 1 ) << 1)

Index: kvm-ptpdriver/Documentation/virtual/kvm/hypercalls.txt
===================================================================

Documentation/virtual/kvm/hypercalls.txt +6 -0

@@ -81,3 +81,36 @@
 same guest can wakeup the sleeping vcpu by issuing KVM_HC_KICK_CPU hypercall,
 specifying APIC ID (a1) of the vcpu to be woken up. An additional argument (a0)
 is used in the hypercall for future use.
+
+
+6. KVM_HC_CLOCK_PAIRING
+------------------------
+Architecture: x86
+Status: active

Documentation/virtual/kvm/hypercalls.txt line 99

Quoted 10 lines

Documentation/virtual/kvm/hypercalls.txt +10 -0

+Purpose: Hypercall used to synchronize host and guest clocks.
+Usage:
+
+a0: guest physical address where host copies
+"struct kvm_clock_offset" structure.
+
+a1: clock_type, ATM only KVM_CLOCK_PAIRING_WALLCLOCK (0)
+is supported (hosts CLOCK_REALTIME clock).
+
+		struct kvm_clock_offset {

Name of the struct not adjusted.

Quoted 8 lines

Documentation/virtual/kvm/hypercalls.txt line 104

Quoted 5 lines

Documentation/virtual/kvm/hypercalls.txt +5 -0

+			__s64 sec;
+			__s64 nsec;
+			__u64 tsc;
+			__u32 flags;
+			__u32 pad[9];

I'd just leave one element of padding and put the struct on the stack, since we have no reason to think we'll need a lot of padding.

Well you don't know: we could implement it in case of HPET clocksource hosts.

Quoted 10 lines

Documentation/virtual/kvm/hypercalls.txt line 111

Quoted 7 lines

Documentation/virtual/kvm/hypercalls.txt +7 -0

+		};
+
+       Where:
+               * sec: seconds from clock_type clock.
+               * nsec: nanoseconds from clock_type clock.
+               * tsc: TSC value used to calculate sec/nsec pair
+                 (this hypercall only works when host uses TSC clocksource).

This is already mentioned below, and it's not clear that tsc is a guest TSC value.

Ok fixed.

Quoted 13 lines

Quoted 13 lines

Documentation/virtual/kvm/hypercalls.txt +5 -0

+               * flags: flags, unused (0) at the moment.
+
+Returns KVM_EOPNOTSUPP if the host does not use TSC clocksource,
+or if clock type is different than KVM_CLOCK_PAIRING_WALLCLOCK.
+

Index: kvm-ptpdriver/arch/x86/kvm/x86.c
===================================================================

arch/x86/kvm/x86.c +0 -0

@@ -6119,6 +6119,44 @@
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
 

Quoted 15 lines

arch/x86/kvm/x86.c line 6134

Quoted 13 lines

arch/x86/kvm/x86.c +13 -0

+static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
+			       unsigned long clock_type)
+{
+	struct kvm_clock_offset *clock_offset;
+	struct timespec ts;
+	cycle_t cycle;
+	int ret;
+
+	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
+		return -KVM_EOPNOTSUPP;
+
+	if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
+		return -KVM_EOPNOTSUPP;

Already checked in kvm_get_walltime_and_clockread.

Fixed.

Quoted 42 lines

Quoted 42 lines

arch/x86/kvm/x86.c +26 -0

+	clock_offset = kzalloc(sizeof(struct kvm_clock_offset), GFP_KERNEL);
+	if (clock_offset == NULL)
+		return -KVM_ENOMEM;
+
+	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false) {
+		kfree(clock_offset);
+		return -KVM_EOPNOTSUPP;
+	}
+	clock_offset->sec = ts.tv_sec;
+	clock_offset->nsec = ts.tv_nsec;
+	clock_offset->tsc = kvm_read_l1_tsc(vcpu, cycle);
+	clock_offset->flags = 0;
+
+	ret = 0;
+	if (kvm_write_guest(vcpu->kvm, paddr, clock_offset,
+			      sizeof(struct kvm_clock_offset)))
+		ret = -KVM_EFAULT;
+
+	kfree(clock_offset);
+
+	return ret;
+}
+
 /*
  * kvm_pv_kick_cpu_op:  Kick a vcpu.
  *

@@ -6183,6 +6221,9 @@
 		kvm_pv_kick_cpu_op(vcpu->kvm, a0, a1);
 		ret = 0;
 		break;
+	case KVM_HC_CLOCK_PAIRING:
+		ret = kvm_pv_clock_pairing(vcpu, a0, a1);
+		break;
 	default:
 		ret = -KVM_ENOSYS;
 		break;

Index: kvm-ptpdriver/include/uapi/linux/kvm_para.h
===================================================================

include/uapi/linux/kvm_para.h +0 -0

@@ -14,6 +14,8 @@
 #define KVM_EFAULT		EFAULT

Quoted 8 lines

include/uapi/linux/kvm_para.h

Quoted 4 lines

include/uapi/linux/kvm_para.h +2 -0

 #define KVM_E2BIG		E2BIG
 #define KVM_EPERM		EPERM
+#define KVM_EOPNOTSUPP		EOPNOTSUPP
+#define KVM_ENOMEM		ENOMEM

This file could potentially be used across multiple architectures (it's not in arch/x86/include/uapi/asm), but EOPNOTSUPP is not the same on all systems. Better use "95" which is the value it has on x86.

Fixed.

Quoted 121 lines

All this gives:

Documentation/virtual/kvm/hypercalls.txt +7 -5

index 55eb3e1ca494..cd35de84999f 100644
@@ -94,9 +94,9 @@ a0: guest physical address where host copies
 "struct kvm_clock_offset" structure.
 
 a1: clock_type, ATM only KVM_CLOCK_PAIRING_WALLCLOCK (0)
-is supported (hosts CLOCK_REALTIME clock).
+is supported (corresponding to the host's CLOCK_REALTIME clock).
 
-		struct kvm_clock_offset {
+		struct kvm_clock_pairing {
 			__s64 sec;
 			__s64 nsec;
 			__u64 tsc;

@@ -107,10 +107,12 @@ is supported (hosts CLOCK_REALTIME clock).
        Where:
                * sec: seconds from clock_type clock.
                * nsec: nanoseconds from clock_type clock.
-               * tsc: TSC value used to calculate sec/nsec pair
-                 (this hypercall only works when host uses TSC clocksource).
+               * tsc: guest TSC value used to calculate sec/nsec pair
                * flags: flags, unused (0) at the moment.
 
+The hypercall lets a guest compute a precise timestamp across
+host and guest.  The guest can use the returned TSC value to
+compute the CLOCK_REALTIME for its clock, at the same instant.
+
 Returns KVM_EOPNOTSUPP if the host does not use TSC clocksource,
 or if clock type is different than KVM_CLOCK_PAIRING_WALLCLOCK.
-

arch/x86/include/uapi/asm/kvm_para.h +2 -2

index ce7b411e28bc..3641d8b7dba9 100644
@@ -51,12 +51,12 @@ struct kvm_steal_time {
 };
 
 #define KVM_CLOCK_PAIRING_WALLCLOCK 0
-struct kvm_clock_offset {
+struct kvm_clock_pairing {
 	__s64 sec;
 	__s64 nsec;
 	__u64 tsc;
 	__u32 flags;
-	__u32 pad[9];
+	__u32 pad[1];
 };
 
 #define KVM_STEAL_ALIGNMENT_BITS 5

arch/x86/kvm/x86.c +9 -20

index 4ecc3145ea4b..09e5d31dac98 100644
@@ -6149,9 +6149,9 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu)
 EXPORT_SYMBOL_GPL(kvm_emulate_halt);
 
 static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
-			       unsigned long clock_type)
+			        unsigned long clock_type)
 {
-	struct kvm_clock_offset *clock_offset;
+	struct kvm_clock_pairing clock_pairing;
 	struct timespec ts;
 	cycle_t cycle;
 	int ret;

@@ -6159,30 +6159,19 @@ static int kvm_pv_clock_pairing(struct kvm_vcpu *vcpu, gpa_t paddr,
 	if (clock_type != KVM_CLOCK_PAIRING_WALLCLOCK)
 		return -KVM_EOPNOTSUPP;
 
-	if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
-		return -KVM_EOPNOTSUPP;
-
-	clock_offset = kzalloc(sizeof(struct kvm_clock_offset), GFP_KERNEL);
-	if (clock_offset == NULL)
-		return -KVM_ENOMEM;
-
-	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false) {
-		kfree(clock_offset);
+	if (kvm_get_walltime_and_clockread(&ts, &cycle) == false)
 		return -KVM_EOPNOTSUPP;
-	}
 
-	clock_offset->sec = ts.tv_sec;
-	clock_offset->nsec = ts.tv_nsec;
-	clock_offset->tsc = kvm_read_l1_tsc(vcpu, cycle);
-	clock_offset->flags = 0;
+	clock_pairing.sec = ts.tv_sec;
+	clock_pairing.nsec = ts.tv_nsec;
+	clock_pairing.tsc = kvm_read_l1_tsc(vcpu, cycle);
+	clock_pairing.flags = 0;
 
 	ret = 0;
-	if (kvm_write_guest(vcpu->kvm, paddr, clock_offset,
-			      sizeof(struct kvm_clock_offset)))
+	if (kvm_write_guest(vcpu->kvm, paddr, &clock_pairing,
+			    sizeof(struct kvm_clock_pairing)))
 		ret = -KVM_EFAULT;
 
-	kfree(clock_offset);
-
 	return ret;
 }
 

include/uapi/linux/kvm_para.h +1 -1

index 68da60e7f519..c72128527377 100644
@@ -14,7 +14,7 @@
 #define KVM_EFAULT		EFAULT
 #define KVM_E2BIG		E2BIG
 #define KVM_EPERM		EPERM
-#define KVM_EOPNOTSUPP		EOPNOTSUPP
+#define KVM_EOPNOTSUPP		95
 #define KVM_ENOMEM		ENOMEM
 
 #define KVM_HC_VAPIC_POLL_IRQ		1

Quoted 1 line

Okay?

Just not the padding please, the rest is fine.

Quoted 16 lines

Paolo

Quoted 13 lines

include/uapi/linux/kvm_para.h +1 -0

 #define KVM_HC_VAPIC_POLL_IRQ		1
 #define KVM_HC_MMU_OP			2

@@ -23,6 +25,7 @@
 #define KVM_HC_MIPS_GET_CLOCK_FREQ	6
 #define KVM_HC_MIPS_EXIT_VM		7
 #define KVM_HC_MIPS_CONSOLE_OUTPUT	8
+#define KVM_HC_CLOCK_PAIRING		9
 
 /*
  * hypercalls use architecture specific

Quoted 45 lines · Marcelo Tosatti

Quoted 35 lines · Paolo Bonzini

Quoted 24 lines · Marcelo Tosatti

Add a hypercall to retrieve the host realtime clock and the TSC value used to calculate that clock read.

Used to implement clock synchronization between
host and guest.

Signed-off-by Marcelo Tosatti <[email protected]>

---
 Documentation/virtual/kvm/hypercalls.txt |   33 ++++++++++++++++++++++++
 arch/x86/include/uapi/asm/kvm_para.h     |    9 ++++++
 arch/x86/kvm/x86.c                       |   41 +++++++++++++++++++++++++++++++
 include/uapi/linux/kvm_para.h            |    3 ++
 4 files changed, 86 insertions(+)

v2: improve documentation (Radim)
change hypercall name to KVM_HC_CLOCK_PAIRING (Radim) increase padding size

Index: kvm-ptpdriver/arch/x86/include/uapi/asm/kvm_para.h
===================================================================

arch/x86/include/uapi/asm/kvm_para.h line 54

Quoted 5 lines

arch/x86/include/uapi/asm/kvm_para.h +2 -0

@@ -50,6 +50,15 @@
 	__u32 pad[11];
 };
 
+#define KVM_CLOCK_PAIRING_WALLCLOCK 0
+struct kvm_clock_offset {

The struct still has the old name.

Quoted 6 lines

arch/x86/include/uapi/asm/kvm_para.h line 60

Quoted 6 lines

arch/x86/include/uapi/asm/kvm_para.h +6 -0

+	__s64 sec;
+	__s64 nsec;
+	__u64 tsc;
+	__u32 flags;
+	__u32 pad[9];
+};

That was on purpose: "to pair" is the operation.

Quoted 4 lines

"struct kvm_clock_pairing" makes no sense to me
(because pairing is an "action").

(if you'd prefer that name, let me know).

A "pairing" is a union of two things, in this case a sec/nsec value and a TSC value. (I had to look it up in the dictionary though :)).

Quoted 38 lines

Quoted 37 lines

Quoted 17 lines

arch/x86/include/uapi/asm/kvm_para.h +0 -0

 #define KVM_STEAL_ALIGNMENT_BITS 5
 #define KVM_STEAL_VALID_BITS ((-1ULL << (KVM_STEAL_ALIGNMENT_BITS + 1)))
 #define KVM_STEAL_RESERVED_MASK (((1 << KVM_STEAL_ALIGNMENT_BITS) - 1 ) << 1)

Index: kvm-ptpdriver/Documentation/virtual/kvm/hypercalls.txt
===================================================================

Documentation/virtual/kvm/hypercalls.txt +6 -0

@@ -81,3 +81,36 @@
 same guest can wakeup the sleeping vcpu by issuing KVM_HC_KICK_CPU hypercall,
 specifying APIC ID (a1) of the vcpu to be woken up. An additional argument (a0)
 is used in the hypercall for future use.
+
+
+6. KVM_HC_CLOCK_PAIRING
+------------------------
+Architecture: x86
+Status: active

Documentation/virtual/kvm/hypercalls.txt line 99

Quoted 10 lines

Documentation/virtual/kvm/hypercalls.txt +10 -0

+Purpose: Hypercall used to synchronize host and guest clocks.
+Usage:
+
+a0: guest physical address where host copies
+"struct kvm_clock_offset" structure.
+
+a1: clock_type, ATM only KVM_CLOCK_PAIRING_WALLCLOCK (0)
+is supported (hosts CLOCK_REALTIME clock).
+
+		struct kvm_clock_offset {

Name of the struct not adjusted.

Documentation/virtual/kvm/hypercalls.txt line 104

Quoted 5 lines

Documentation/virtual/kvm/hypercalls.txt +5 -0

+			__s64 sec;
+			__s64 nsec;
+			__u64 tsc;
+			__u32 flags;
+			__u32 pad[9];

I'd just leave one element of padding and put the struct on the stack, since we have no reason to think we'll need a lot of padding.

Quoted 2 lines

Well you don't know: we could implement it in case of HPET clocksource hosts.

But then the guest wouldn't have a TSC value to use, would it?

Also, how would it use more padding?

Quoted 5 lines

Quoted 2 lines

Well you don't know: we could implement it in case of HPET clocksource hosts.

But then the guest wouldn't have a TSC value to use, would it?

Also, how would it use more padding?

I don't know, maybe pass the TSC and host HPET read in the structure? (after exposing the HPET to the guest).

Or maybe you want to sync CLOCK_MONOTONIC and that requires other data.

Not sure.

Just some space if new features are necessary.

Add a driver with gettime method returning hosts realtime clock. This allows Chrony to synchronize host and guest clocks with high precision (see results below).

chronyc> sources
MS Name/IP address Stratum Poll Reach LastRx Last sample
===============================================================================
#* PHC0 0 3 377 4 +162ns[ -683ns] +/- 11ns

To configure Chronyd to use PHC refclock, add the
following line to its configuration file:

refclock PHC /dev/ptpX poll 3 dpoll -2 offset 0

Where /dev/ptpX is the kvmclock PTP clock.

Signed-off-by Marcelo Tosatti <[email protected]>

Add a driver with gettime method returning hosts realtime clock. This allows Chrony to synchronize host and guest clocks with high precision (see results below).

chronyc> sources
MS Name/IP address Stratum Poll Reach LastRx Last sample
===============================================================================
#* PHC0 0 3 377 4 +162ns[ -683ns] +/- 11ns

To configure Chronyd to use PHC refclock, add the
following line to its configuration file:

refclock PHC /dev/ptpX poll 3 dpoll -2 offset 0

Where /dev/ptpX is the kvmclock PTP clock.

Signed-off-by Marcelo Tosatti <[email protected]>

---
 drivers/ptp/Kconfig   |   12 ++
 drivers/ptp/Makefile  |    1 
 drivers/ptp/ptp_kvm.c |  202 ++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 215 insertions(+)

v2: check for kvmclock (Radim)
initialize global variables before device registration (Radim)
v3: use cross timestamps callback (Paolo, Miroslav, Radim)
v4: remove gettime64 callback (Paolo)
v5: drop useless nsec assignment (Radim)

Index: kvm-ptpdriver/drivers/ptp/Kconfig
===================================================================

drivers/ptp/Kconfig +12 -0

@@ -90,4 +90,16 @@
 	  To compile this driver as a module, choose M here: the module
 	  will be called ptp_pch.
 
+config PTP_1588_CLOCK_KVM
+	tristate "KVM virtual PTP clock"
+	depends on PTP_1588_CLOCK
+	depends on KVM_GUEST
+	default y
+	help
+	  This driver adds support for using kvm infrastructure as a PTP
+	  clock. This clock is only useful if you are using KVM guests.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called ptp_kvm.
+
 endmenu

Index: kvm-ptpdriver/drivers/ptp/Makefile
===================================================================

drivers/ptp/Makefile +1 -0

@@ -6,3 +6,4 @@
 obj-$(CONFIG_PTP_1588_CLOCK)		+= ptp.o
 obj-$(CONFIG_PTP_1588_CLOCK_IXP46X)	+= ptp_ixp46x.o
 obj-$(CONFIG_PTP_1588_CLOCK_PCH)	+= ptp_pch.o
+obj-$(CONFIG_PTP_1588_CLOCK_KVM)	+= ptp_kvm.o

Index: kvm-ptpdriver/drivers/ptp/ptp_kvm.c
===================================================================

drivers/ptp/ptp_kvm.c new file +47 -0

@@ -0,0 +1,202 @@
+/*
+ * Virtual PTP 1588 clock for use with KVM guests
+ *
+ * Copyright (C) 2017 Red Hat Inc.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ */
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <uapi/linux/kvm_para.h>
+#include <asm/kvm_para.h>
+#include <asm/pvclock.h>
+#include <asm/kvmclock.h>
+#include <uapi/asm/kvm_para.h>
+
+#include <linux/ptp_clock_kernel.h>
+
+struct kvm_ptp_clock {
+	struct ptp_clock *ptp_clock;
+	struct ptp_clock_info caps;
+};
+
+DEFINE_SPINLOCK(kvm_ptp_lock);
+
+static struct pvclock_vsyscall_time_info *hv_clock;
+
+static struct kvm_clock_offset clock_off;
+static phys_addr_t clock_off_gpa;
+
+static int ptp_kvm_get_time_fn(ktime_t *device_time,
+			       struct system_counterval_t *system_counter,
+			       void *ctx)
+{
+	unsigned long ret;
+	struct timespec64 tspec;

drivers/ptp/ptp_kvm.c new file +5 -0

+	do {
+		/*
+		 * We are using a TSC value read in the hosts
+		 * kvm_hc_clock_pairing handling.
+		 * So any changes to tsc_to_system_mul

Quoted 117 lines · Marcelo Tosatti

Add a driver with gettime method returning hosts realtime clock. This allows Chrony to synchronize host and guest clocks with high precision (see results below).

chronyc> sources
MS Name/IP address Stratum Poll Reach LastRx Last sample
===============================================================================
#* PHC0 0 3 377 4 +162ns[ -683ns] +/- 11ns

To configure Chronyd to use PHC refclock, add the
following line to its configuration file:

refclock PHC /dev/ptpX poll 3 dpoll -2 offset 0

Where /dev/ptpX is the kvmclock PTP clock.

Signed-off-by Marcelo Tosatti <[email protected]>

---
 drivers/ptp/Kconfig   |   12 ++
 drivers/ptp/Makefile  |    1 
 drivers/ptp/ptp_kvm.c |  202 ++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 215 insertions(+)

v2: check for kvmclock (Radim)
initialize global variables before device registration (Radim)
v3: use cross timestamps callback (Paolo, Miroslav, Radim)
v4: remove gettime64 callback (Paolo)
v5: drop useless nsec assignment (Radim)

Index: kvm-ptpdriver/drivers/ptp/Kconfig
===================================================================

drivers/ptp/Kconfig +12 -0

@@ -90,4 +90,16 @@
 	  To compile this driver as a module, choose M here: the module
 	  will be called ptp_pch.
 
+config PTP_1588_CLOCK_KVM
+	tristate "KVM virtual PTP clock"
+	depends on PTP_1588_CLOCK
+	depends on KVM_GUEST
+	default y
+	help
+	  This driver adds support for using kvm infrastructure as a PTP
+	  clock. This clock is only useful if you are using KVM guests.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called ptp_kvm.
+
 endmenu

Index: kvm-ptpdriver/drivers/ptp/Makefile
===================================================================

drivers/ptp/Makefile +1 -0

@@ -6,3 +6,4 @@
 obj-$(CONFIG_PTP_1588_CLOCK)		+= ptp.o
 obj-$(CONFIG_PTP_1588_CLOCK_IXP46X)	+= ptp_ixp46x.o
 obj-$(CONFIG_PTP_1588_CLOCK_PCH)	+= ptp_pch.o
+obj-$(CONFIG_PTP_1588_CLOCK_KVM)	+= ptp_kvm.o

Index: kvm-ptpdriver/drivers/ptp/ptp_kvm.c
===================================================================

drivers/ptp/ptp_kvm.c new file +52 -0

@@ -0,0 +1,202 @@
+/*
+ * Virtual PTP 1588 clock for use with KVM guests
+ *
+ * Copyright (C) 2017 Red Hat Inc.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ */
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <uapi/linux/kvm_para.h>
+#include <asm/kvm_para.h>
+#include <asm/pvclock.h>
+#include <asm/kvmclock.h>
+#include <uapi/asm/kvm_para.h>
+
+#include <linux/ptp_clock_kernel.h>
+
+struct kvm_ptp_clock {
+	struct ptp_clock *ptp_clock;
+	struct ptp_clock_info caps;
+};
+
+DEFINE_SPINLOCK(kvm_ptp_lock);
+
+static struct pvclock_vsyscall_time_info *hv_clock;
+
+static struct kvm_clock_offset clock_off;
+static phys_addr_t clock_off_gpa;
+
+static int ptp_kvm_get_time_fn(ktime_t *device_time,
+			       struct system_counterval_t *system_counter,
+			       void *ctx)
+{
+	unsigned long ret;
+	struct timespec64 tspec;
+	unsigned version;
+	u8 flags;
+	int cpu;
+	struct pvclock_vcpu_time_info *src;
+

drivers/ptp/ptp_kvm.c line 57

Quoted 5 lines

drivers/ptp/ptp_kvm.c new file +5 -0

+	preempt_disable_notrace();
+	cpu = smp_processor_id();
+	src = &hv_clock[cpu].pvti;
+
+	spin_lock(&kvm_ptp_lock);

Any reason to take the spinlock in the non-preemptable region? This would be worse for RT kernels than taking it outside.

Quoted 11 lines

drivers/ptp/ptp_kvm.c new file +10 -0

+	do {
+		/*
+		 * We are using a TSC value read in the hosts
+		 * kvm_hc_clock_pairing handling.
+		 * So any changes to tsc_to_system_mul
+		 * and tsc_shift or any other pvclock
+		 * data invalidate that measurement.
+		 */
+		version = pvclock_read_begin(src);
+

drivers/ptp/ptp_kvm.c line 73

Quoted 5 lines

drivers/ptp/ptp_kvm.c new file +5 -0

+		ret = kvm_hypercall2(KVM_HC_CLOCK_PAIRING,
+				     clock_off_gpa,
+				     KVM_CLOCK_PAIRING_WALLCLOCK);
+		if (ret != 0) {
+			pr_err("clock offset hypercall ret %lu\n", ret);

Need ratelimiting in case something goes wrong and the host stops using the TSC clocksource?

drivers/ptp/ptp_kvm.c line 82

Quoted 9 lines

drivers/ptp/ptp_kvm.c new file +9 -0

+			spin_unlock(&kvm_ptp_lock);
+			preempt_enable_notrace();
+			return -EOPNOTSUPP;
+		}
+
+		tspec.tv_sec = clock_off.sec;
+		tspec.tv_nsec = clock_off.nsec;
+		ret = __pvclock_read_cycles(src, clock_off.tsc);
+		flags = src->flags;

Flags written and not read.

Also, I think I disagree with Radim, gettime64 should use the TSC correction. It's the right thing to do for the POSIX clock interface. It's a bit arbitrary for PTP_SYS_OFFSET (though you cannot really argue with better precision), but chrony is now using PTP_SYS_OFFSET_PRECISE (and again you cannot really argue with better precision anyway).

I'll test this a bit and then push it to kvm/queue. Please take a look!

Paolo