Skip to contentSkip to conversation

Drivers: hv: kvp,vss: improve kernel-userspace communication in failure case

PATCH New Vitaly Kuznetsov 1 participant 3 messages Nov 6, 2014 Hyper-V/Azure core and drivers

lore
Share & tools
lore
#0 lore raw

cover Drivers: hv: kvp,vss: improve kernel-userspace communication in failure case

This series addresses two issues:
- There is no timeout when communicating with userspace VSS daemon.
- In case we fail to send a message to VSS or KVP userspace daemons we can report the failure to the host right away avoiding the timeout.

Newly introduced 10 second timeout is something worth discussing. In theory freeze/thaw ioctls should be fast. In case someone thinks 10 seconds is not enough we can easily increase it as we cover the most common failure scenario (when the daemon was stopped) with the second patch of this series.

Vitaly Kuznetsov (2):
Drivers: hv: vss: Introduce timeout for communication with userspace
Drivers: hv: kvp,vss: Fast propagation of userspace communication
failure

 drivers/hv/hv_kvp.c      |  9 ++++++++-
 drivers/hv/hv_snapshot.c | 28 +++++++++++++++++++++++++---
 2 files changed, 33 insertions(+), 4 deletions(-)
Signature · 1 line
1.9.3
#1 lore raw

PATCH 1/2 Drivers: hv: vss: Introduce timeout for communication with userspace

In contrast with KVP there is no timeout when communicating with userspace VSS daemon. In case it gets stuck performing freeze/thaw operation no message will be sent to the host so it will take very long (around 10 minutes) before backup fails. Introduce 10 second timeout using schedule_delayed_work().

---
 drivers/hv/hv_snapshot.c | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)
drivers/hv/hv_snapshot.c +18 -2
index 34f14fd..21e51be 100644
@@ -28,7 +28,7 @@
 #define VSS_MINOR  0
 #define VSS_VERSION    (VSS_MAJOR << 16 | VSS_MINOR)
 
-
+#define VSS_USERSPACE_TIMEOUT (msecs_to_jiffies(10 * 1000))
 
 /*
  * Global state maintained for transaction that is being processed.
@@ -55,12 +55,24 @@ static const char vss_name[] = "vss_kernel_module";
 static __u8 *recv_buffer;
 
 static void vss_send_op(struct work_struct *dummy);
+static void vss_timeout_func(struct work_struct *dummy);
+
+static DECLARE_DELAYED_WORK(vss_timeout_work, vss_timeout_func);
 static DECLARE_WORK(vss_send_op_work, vss_send_op);
 
 /*
  * Callback when data is received from user mode.
  */
 
+static void vss_timeout_func(struct work_struct *dummy)
+{
+	/*
+	 * Timeout waiting for userspace component to reply happened.
+	 */
+	pr_warn("VSS: timeout waiting for daemon to reply\n");
+	vss_respond_to_host(HV_E_FAIL);
+}
+
 static void
 vss_cn_callback(struct cn_msg *msg, struct netlink_skb_parms *nsp)
 {
@@ -76,7 +88,8 @@ vss_cn_callback(struct cn_msg *msg, struct netlink_skb_parms *nsp)
 		return;
 
 	}
-	vss_respond_to_host(vss_msg->error);
+	if (cancel_delayed_work_sync(&vss_timeout_work))
+		vss_respond_to_host(vss_msg->error);
 }
 
 
@@ -223,6 +236,8 @@ void hv_vss_onchannelcallback(void *context)
 			case VSS_OP_FREEZE:
 			case VSS_OP_THAW:
 				schedule_work(&vss_send_op_work);
+				schedule_delayed_work(&vss_timeout_work,
+						      VSS_USERSPACE_TIMEOUT);
 				return;
 
 			case VSS_OP_HOT_BACKUP:
@@ -277,5 +292,6 @@ hv_vss_init(struct hv_util_service *srv)
 void hv_vss_deinit(void)
 {
 	cn_del_callback(&vss_id);
+	cancel_delayed_work_sync(&vss_timeout_work);
 	cancel_work_sync(&vss_send_op_work);
 }
Signature · 1 line
1.9.3
#2 lore raw

PATCH 2/2 Drivers: hv: kvp,vss: Fast propagation of userspace communication failure

In reply to Vitaly Kuznetsov

If we fail to send a message to userspace daemon with cn_netlink_send() there is no need to wait for userspace to reply as it is not going to happen. This happens when kvp or vss daemon is stopped after a successful handshake. Report HV_E_FAIL immediately and cancel the timeout job so host won't receive two failures.
Use pr_warn() for VSS and pr_debug() for KVP deliberately as VSS request are rare and result in a failed backup. KVP requests are much more frequent after a successful handshake so avoid flooding logs. It would be nice to have an ability to de-negotiate with the host in case userspace daemon gets disconnected so we won't receive new requests. But I'm not sure it is possible.

---
 drivers/hv/hv_kvp.c      | 9 ++++++++-
 drivers/hv/hv_snapshot.c | 8 +++++++-
 2 files changed, 15 insertions(+), 2 deletions(-)
drivers/hv/hv_kvp.c +8 -1
index 521c146..beb8105 100644
@@ -350,6 +350,7 @@ kvp_send_key(struct work_struct *dummy)
 	__u8 pool = kvp_transaction.kvp_msg->kvp_hdr.pool;
 	__u32 val32;
 	__u64 val64;
+	int rc;
 
 	msg = kzalloc(sizeof(*msg) + sizeof(struct hv_kvp_msg) , GFP_ATOMIC);
 	if (!msg)
@@ -446,7 +447,13 @@ kvp_send_key(struct work_struct *dummy)
 	}
 
 	msg->len = sizeof(struct hv_kvp_msg);
-	cn_netlink_send(msg, 0, 0, GFP_ATOMIC);
+	rc = cn_netlink_send(msg, 0, 0, GFP_ATOMIC);
+	if (rc) {
+		pr_debug("KVP: failed to communicate to the daemon: %d\n", rc);
+		if (cancel_delayed_work_sync(&kvp_work))
+			kvp_respond_to_host(message, HV_E_FAIL);
+	}
+
 	kfree(msg);
 
 	return;
drivers/hv/hv_snapshot.c +7 -1
index 21e51be..9d5e0d1 100644
@@ -96,6 +96,7 @@ vss_cn_callback(struct cn_msg *msg, struct netlink_skb_parms *nsp)
 static void vss_send_op(struct work_struct *dummy)
 {
 	int op = vss_transaction.msg->vss_hdr.operation;
+	int rc;
 	struct cn_msg *msg;
 	struct hv_vss_msg *vss_msg;
 
@@ -111,7 +112,12 @@ static void vss_send_op(struct work_struct *dummy)
 	vss_msg->vss_hdr.operation = op;
 	msg->len = sizeof(struct hv_vss_msg);
 
-	cn_netlink_send(msg, 0, 0, GFP_ATOMIC);
+	rc = cn_netlink_send(msg, 0, 0, GFP_ATOMIC);
+	if (rc) {
+		pr_warn("VSS: failed to communicate to the daemon: %d\n", rc);
+		if (cancel_delayed_work_sync(&vss_timeout_work))
+			vss_respond_to_host(HV_E_FAIL);
+	}
 	kfree(msg);
 
 	return;
Signature · 1 line
1.9.3