Skip to contentSkip to conversation

Drivers: hv: util: Implement Time Synchronization using host time sample

PATCH In review Thomas Shao 2 participants 6 messages Sep 26, 2014 Hyper-V/Azure core and drivers

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Drivers: hv: util: Implement Time Synchronization using host time sample

In current hyper-v time sync service,it only gets the initial clock time from the host. It didn't process the following time samples. This change introduced a module parameter called host_time_sync. If it is set to true, the guest will periodically sychronize it's time with the host clock using host time sample. By default it is disabled, because we still recommend user to configure NTP for time synchronization.

---
 drivers/hv/hv_util.c |  133 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 files changed, 125 insertions(+), 8 deletions(-)
drivers/hv/hv_util.c +125 -8
index 3b9c9ef..9b167c6 100644
@@ -51,11 +51,30 @@
 #define HB_WS2008_MAJOR	1
 #define HB_WS2008_VERSION	(HB_WS2008_MAJOR << 16 | HB_MINOR)
 
+#define  TIMESAMPLE_INTERVAL 5000000000L  /* 5s in nanosecond */
+
+/*host sends time sample for every 5s.So the max polling interval
+ *is 128*5 = 640s.
+*/
+#define  TIME_ADJ_MAX_INTERVAL 128 /*Max polling interval */
+
 static int sd_srv_version;
 static int ts_srv_version;
 static int hb_srv_version;
 static int util_fw_version;
 
+/*host sends time sample for every 5s.So the initial polling interval
+ *is 5s.
+*/
+static s32 adj_interval = 1;
+
+/*The host_time_sync module parameter is used to control the time
+  sync between host and guest.
+*/
+static bool host_time_sync;
+module_param(host_time_sync, bool, (S_IRUGO | S_IWUSR));
+MODULE_PARM_DESC(host_time_sync, "If the guest sync time with host");
+
 static void shutdown_onchannelcallback(void *context);
 static struct hv_util_service util_shutdown = {
 	.util_cb = shutdown_onchannelcallback,
@@ -160,18 +179,83 @@ static void shutdown_onchannelcallback(void *context)
 		schedule_work(&shutdown_work);
 }
 
+/* helper function to call adjtimex command in user mode */
+static void run_adjtimex_cmd(s64 tickvalue)
+{
+	char *argv[4], *envp[3];
+	char str_tickvalue[20];
+
+	sprintf(str_tickvalue, "%lld", tickvalue);
+
+	argv[0] = "/sbin/adjtimex";
+	argv[1] = "-t";
+	argv[2] = str_tickvalue;
+	argv[3] = NULL;
+
+	envp[0] = "HOME=/";
+	envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
+	envp[2] = NULL;
+
+	call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC);
+}
+
 /*
  * Set guest time to host UTC time.
  */
-static inline void do_adj_guesttime(u64 hosttime)
+static inline void do_adj_guesttime(u64 hosttime, bool forceSync)
 {
-	s64 host_tns;
-	struct timespec host_ts;
+	s64 host_tns, guest_tns, diff;
+	struct timespec host_ts, guest_ts;
+	s64 tickchg, tickval;
+	int diff_sign;
 
 	host_tns = (hosttime - WLTIMEDELTA) * 100;
 	host_ts = ns_to_timespec(host_tns);
 
-	do_settimeofday(&host_ts);
+	if (forceSync) {
+		do_settimeofday(&host_ts);
+	} else {
+		/* for the first time, reset the tick value */
+		if (adj_interval == 1) {
+			run_adjtimex_cmd(TICK_USEC);
+			adj_interval = adj_interval * 2;
+			do_settimeofday(&host_ts);
+			return;
+		}
+
+		guest_ts = CURRENT_TIME;
+		guest_tns = timespec_to_ns(&guest_ts);
+		diff = host_tns - guest_tns;
+		if (diff >= 0) {
+			diff_sign = 1;
+		} else {
+			diff_sign = -1;
+			diff = -diff;
+		}
+
+		/*1s in nanosecond */
+		if (diff > 1000000000 || diff < -1000000000) {
+			do_settimeofday(&host_ts);
+			return;
+		}
+
+		/*1ms in nanosecond */
+		if (diff > 1000000 || diff < -1000000) {
+			tickchg = diff * TICK_USEC /
+					(TIMESAMPLE_INTERVAL * adj_interval);
+
+			if (tickchg > TICK_USEC/10)
+				tickchg = TICK_USEC/10;
+
+			tickval = TICK_USEC + diff_sign * tickchg;
+			run_adjtimex_cmd(tickval);
+
+		} else {
+			/* double the polling interval*/
+			if (adj_interval < TIME_ADJ_MAX_INTERVAL)
+				adj_interval = adj_interval * 2;
+		}
+	}
 }
 
 /*
@@ -179,8 +263,9 @@ static inline void do_adj_guesttime(u64 hosttime)
  */
 
 struct adj_time_work {
-	struct work_struct work;
+	struct	work_struct work;
 	u64	host_time;
+	bool	forceSync;
 };
 
 static void hv_set_host_time(struct work_struct *work)
@@ -188,7 +273,7 @@ static void hv_set_host_time(struct work_struct *work)
 	struct adj_time_work	*wrk;
 
 	wrk = container_of(work, struct adj_time_work, work);
-	do_adj_guesttime(wrk->host_time);
+	do_adj_guesttime(wrk->host_time, wrk->forceSync);
 	kfree(wrk);
 }
 
@@ -202,11 +287,14 @@ static void hv_set_host_time(struct work_struct *work)
  * thing is, systime is automatically set to emulated hardware clock which may
  * not be UTC time or in the same time zone. So, to override these effects, we
  * use the first 50 time samples for initial system time setting.
+ * If the host_time_sync module parameter is set, we will use the host time
+ * samples to adjust guest time after the first 50 samples.
  */
 static inline void adj_guesttime(u64 hosttime, u8 flags)
 {
 	struct adj_time_work    *wrk;
 	static s32 scnt = 50;
+	static s32 sample_count;
 
 	wrk = kmalloc(sizeof(struct adj_time_work), GFP_ATOMIC);
 	if (wrk == NULL)
@@ -214,6 +302,7 @@ static inline void adj_guesttime(u64 hosttime, u8 flags)
 
 	wrk->host_time = hosttime;
 	if ((flags & ICTIMESYNCFLAG_SYNC) != 0) {
+		wrk->forceSync = true;
 		INIT_WORK(&wrk->work, hv_set_host_time);
 		schedule_work(&wrk->work);
 		return;
@@ -221,10 +310,38 @@ static inline void adj_guesttime(u64 hosttime, u8 flags)
 
 	if ((flags & ICTIMESYNCFLAG_SAMPLE) != 0 && scnt > 0) {
 		scnt--;
+		wrk->forceSync = true;
 		INIT_WORK(&wrk->work, hv_set_host_time);
 		schedule_work(&wrk->work);
-	} else
-		kfree(wrk);
+		return;
+	}
+
+	if (host_time_sync) {
+		/*
+		* Use the Hyper-V time sample to adjust the guest time. The
+		* algorithm is: If the sample offsets exceeds 1 second, we
+		* directly set the clock to the server time. If the offset is
+		* less than 1ms, we ignore the time sample. Otherwise we adjust
+		* the clock.
+		*/
+
+		if ((flags & ICTIMESYNCFLAG_SAMPLE) != 0) {
+			if (sample_count < adj_interval) {
+				sample_count++;
+				goto cleanup;
+			}
+			/* reset the polling interval */
+			sample_count = 0;
+			wrk->forceSync = false;
+			INIT_WORK(&wrk->work, hv_set_host_time);
+			schedule_work(&wrk->work);
+			return;
+		}
+	}
+
+cleanup:
+	kfree(wrk);
+
 }
 
 /*
Signature · 1 line
1.7.1
#1 lore raw
Quoted 9 lines
drivers/hv/hv_util.c +9 -0
+/* helper function to call adjtimex command in user mode */
+static void run_adjtimex_cmd(s64 tickvalue)
+{
+       char *argv[4], *envp[3];
+       char str_tickvalue[20];
+
+       sprintf(str_tickvalue, "%lld", tickvalue);
+
+       argv[0] = "/sbin/adjtimex";

drivers/hv/hv_util.c static void shutdown_onchannelcallback(void *context) · line 200

Quoted 10 lines
drivers/hv/hv_util.c +10 -0
+       argv[1] = "-t";
+       argv[2] = str_tickvalue;
+       argv[3] = NULL;
+
+       envp[0] = "HOME=/";
+       envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
+       envp[2] = NULL;
+
+       call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC);
+}

What happens when adjtimex is not present?
Is there no kernel space function for that?
Does this patch affect "not setting correct time on restore" or "not setting correct time on live migration" bug?

Thanks

Mike

#2 lore raw
Quoted 26 lines

-----Original Message-----
From: Mike Surcouf [mailto:[email protected]]
Sent: Friday, September 26, 2014 7:53 PM
To: Thomas Shao
Cc: [email protected]; [email protected]; [email protected]; Olaf Hering; Robo Bot; [email protected]; KY Srinivasan; Haiyang Zhang
Subject: Re: [PATCH] Drivers: hv: util: Implement Time Synchronization using host time sample

Quoted 15 lines
drivers/hv/hv_util.c +15 -0
+/* helper function to call adjtimex command in user mode */ static
+void run_adjtimex_cmd(s64 tickvalue) {
+       char *argv[4], *envp[3];
+       char str_tickvalue[20];
+
+       sprintf(str_tickvalue, "%lld", tickvalue);
+
+       argv[0] = "/sbin/adjtimex";
+       argv[1] = "-t";
+       argv[2] = str_tickvalue;
+       argv[3] = NULL;
+
+       envp[0] = "HOME=/";
+       envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
+       envp[2] = NULL;
Quoted 9 lines

Review

Quoted 2 lines
(quoted diff) +2 -0
+
+       call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC); }

What happens when adjtimex is not present?
Is there no kernel space function for that?
Does this patch affect "not setting correct time on restore" or "not setting correct time on live migration" bug?

If adjtimex is not present, then the slew time part didn't take effect.
There is a kernel space do_adjtmex, but it's only for sys call and not exposed to module. So I didn't use it in my patch. With this patch, it will step the clock if the time drift is larger than 1 seconds, so it can solve the issue your mentioned. And on the other hand, "setting correct time on restore" is already fixed in other patch, which is already in upstream.

Quoted 3 lines

Thanks

Mike

���{.n�+�������+%�����ݶ��w��{.n�+����{��G�����{ay�ʇڙ�,j��f���h��������z_�(�階�ݢj"���m������G������?����&���~��iO��z��v�^�m���� ������?�I�

#3 lore raw
Quoted 6 lines
Quoted 5 lines

What happens when adjtimex is not present?
Is there no kernel space function for that?
Does this patch affect "not setting correct time on restore" or "not setting correct time on live migration" bug?

Quoted 6 lines

If adjtimex is not present, then the slew time part didn't take effect.
There is a kernel space do_adjtmex, but it's only for sys call and not exposed to module. So I didn't use it in my patch. With this patch, it will step the clock if the time drift is larger than 1 seconds, so it can solve the issue your mentioned. And on the other hand, "setting correct time on restore" is already fixed in other patch, which is already in upstream.

AFAIK CENTOS and RHEL stopped providing adjtimex for a while now (since V5?). Even went as far as removing references from there man page for hwclock. in V6

On CENTOS 6.5

yum provides */adjtimex

Loaded plugins: fastestmirror, security
Loading mirror speeds from cached hostfile
* base: centosc6.centos.org
* extras: centosb5.centos.org
* updates: centosb6.centos.org
No Matches found

So the end user will have to go searching the internet rpmfind or something to install an unofficial package on an enterprise product. So by default you will be stepping time on CENTOS/RHEL 6 and above:
with the result of an unstable clock.

#4 lore raw

In CENTOS and RHEL 6 and upwards the official way would be to use

/usr/sbin/tickadj

which is provided by ntp/chrony package

This dependency on user space packages that may or may not be installed is going to cause a lot of confusion in an already confused space.
Are we sure theres no other way?

#5 lore raw
Quoted 14 lines

-----Original Message-----
From: Mike Surcouf [mailto:[email protected]]
Sent: Friday, September 26, 2014 9:52 PM
To: Thomas Shao
Cc: [email protected]; [email protected]; [email protected]; Olaf Hering; Robo Bot; [email protected]; KY Srinivasan; Haiyang Zhang
Subject: Re: [PATCH] Drivers: hv: util: Implement Time Synchronization using host time sample

In CENTOS and RHEL 6 and upwards the official way would be to use

Quoted 7 lines

/usr/sbin/tickadj

which is provided by ntp/chrony package

This dependency on user space packages that may or may not be installed is going to cause a lot of confusion in an already confused space. Are we sure theres no other way?

Thanks Mike.
I think another way is to expose the do_adjtimex to kernel module. I'm not sure if there is any concerns. I'll investigate that. ���{.n�+�������+%�����ݶ��w��{.n�+����{��G�����{ay�ʇڙ�,j��f���h��������z_�(�階�ݢj"���m������G������?����&���~��iO��z��v�^�m���� ������?�I�