Skip to contentSkip to conversation

nbd: local daemon restart support

PATCH In review Mike Christie 1 participant 4 messages Nov 16, 2019 – Nov 19, 2019

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover nbd: local daemon restart support

The following patches made over Linus's tree allow setups that are using AF_UNIX sockets with a local daemon to recover from crashes or to upgrade the daemon while IO is running without having to disrupt the application (no need to reopen the device or handle IO errors). They basically just use the existing failover infrastructure, but to failover to a new socket from a non-dead socket.

#1 lore raw

PATCH 1/2 nbd: move send_disconnects

Just move send_disconnects so we can call it earlier in the file in the next patch.

---
 drivers/block/nbd.c | 48 ++++++++++++++++++++++-----------------------
 1 file changed, 24 insertions(+), 24 deletions(-)
drivers/block/nbd.c +24 -24
index a94ee45440b3..06b63c11ae50 100644
@@ -980,6 +980,30 @@ static blk_status_t nbd_queue_rq(struct blk_mq_hw_ctx *hctx,
 	return ret;
 }
 
+static void send_disconnects(struct nbd_device *nbd)
+{
+	struct nbd_config *config = nbd->config;
+	struct nbd_request request = {
+		.magic = htonl(NBD_REQUEST_MAGIC),
+		.type = htonl(NBD_CMD_DISC),
+	};
+	struct kvec iov = {.iov_base = &request, .iov_len = sizeof(request)};
+	struct iov_iter from;
+	int i, ret;
+
+	for (i = 0; i < config->num_connections; i++) {
+		struct nbd_sock *nsock = config->socks[i];
+
+		iov_iter_kvec(&from, WRITE, &iov, 1, sizeof(request));
+		mutex_lock(&nsock->tx_lock);
+		ret = sock_xmit(nbd, i, 1, &from, 0, NULL);
+		if (ret <= 0)
+			dev_err(disk_to_dev(nbd->disk),
+				"Send disconnect failed %d\n", ret);
+		mutex_unlock(&nsock->tx_lock);
+	}
+}
+
 static struct socket *nbd_get_socket(struct nbd_device *nbd, unsigned long fd,
 				     int *err)
 {
@@ -1139,30 +1163,6 @@ static void nbd_parse_flags(struct nbd_device *nbd)
 		blk_queue_write_cache(nbd->disk->queue, false, false);
 }
 
-static void send_disconnects(struct nbd_device *nbd)
-{
-	struct nbd_config *config = nbd->config;
-	struct nbd_request request = {
-		.magic = htonl(NBD_REQUEST_MAGIC),
-		.type = htonl(NBD_CMD_DISC),
-	};
-	struct kvec iov = {.iov_base = &request, .iov_len = sizeof(request)};
-	struct iov_iter from;
-	int i, ret;
-
-	for (i = 0; i < config->num_connections; i++) {
-		struct nbd_sock *nsock = config->socks[i];
-
-		iov_iter_kvec(&from, WRITE, &iov, 1, sizeof(request));
-		mutex_lock(&nsock->tx_lock);
-		ret = sock_xmit(nbd, i, 1, &from, 0, NULL);
-		if (ret <= 0)
-			dev_err(disk_to_dev(nbd->disk),
-				"Send disconnect failed %d\n", ret);
-		mutex_unlock(&nsock->tx_lock);
-	}
-}
-
 static int nbd_disconnect(struct nbd_device *nbd)
 {
 	struct nbd_config *config = nbd->config;
Signature · 1 line
2.21.0
#2 lore raw

PATCH 2/2 nbd: add support to replace running daemon

In reply to Mike Christie

This patch allows userspace to switch daemons while IO is running or to recovery from a crashed daemon. With the patch when using AF_UNIX sockets and a local running daemon, the new daemon can perform a NBD_CMD_RECONFIGURE nl command with the NBD_ATTR_SWAP_SOCKETS attr set to 0. The kernel will then switch over to using the new socket similar to how we handle failing over from a dead socket.

---
 drivers/block/nbd.c              | 44 +++++++++++++++++++++++++++++---
 include/uapi/linux/nbd-netlink.h |  1 +
 2 files changed, 41 insertions(+), 4 deletions(-)
drivers/block/nbd.c +40 -4
index 06b63c11ae50..6aa396bea401 100644
@@ -1076,7 +1076,14 @@ static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg,
 	return 0;
 }
 
-static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)
+static bool nbd_requeue_req(struct request *req, void *data, bool reserved)
+{
+	nbd_requeue_cmd(blk_mq_rq_to_pdu(req));
+	return true;
+}
+
+static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg,
+				u8 swap_socks)
 {
 	struct nbd_config *config = nbd->config;
 	struct socket *sock, *old;
@@ -1094,6 +1101,16 @@ static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)
 		return -ENOMEM;
 	}
 
+	if (swap_socks) {
+		dev_dbg(disk_to_dev(nbd->disk), "stopping queue\n");
+
+		blk_mq_quiesce_queue(nbd->disk->queue);
+		send_disconnects(nbd);
+		sock_shutdown(nbd);
+		flush_workqueue(nbd->recv_workq);
+		blk_mq_tagset_busy_iter(&nbd->tag_set, nbd_requeue_req, NULL);
+	}
+
 	for (i = 0; i < config->num_connections; i++) {
 		struct nbd_sock *nsock = config->socks[i];
 
@@ -1130,11 +1147,20 @@ static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)
 
 		atomic_inc(&config->live_connections);
 		wake_up(&config->conn_wait);
-		return 0;
+		err = 0;
+		goto start_queue;
 	}
 	sockfd_put(sock);
 	kfree(args);
-	return -ENOSPC;
+	err = -ENOSPC;
+
+start_queue:
+	if (swap_socks) {
+		nbd->task_recv = current;
+		blk_mq_unquiesce_queue(nbd->disk->queue);
+		dev_dbg(disk_to_dev(nbd->disk), "queue restarted\n");
+	}
+	return err;
 }
 
 static void nbd_bdev_reset(struct block_device *bdev)
@@ -2027,6 +2053,7 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
 {
 	struct nbd_device *nbd = NULL;
 	struct nbd_config *config;
+	u8 swap_socks = 0;
 	int index;
 	int ret = 0;
 	bool put_dev = false;
@@ -2107,6 +2134,15 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
 		}
 	}
 
+	if (info->attrs[NBD_ATTR_SWAP_SOCKETS]) {
+		swap_socks = nla_get_u8(info->attrs[NBD_ATTR_SWAP_SOCKETS]);
+		if (swap_socks != 1 && swap_socks != 0) {
+			printk(KERN_ERR "nbd: NBD_SOCK_SWAP must be 0 or 1\n");
+			ret = -EINVAL;
+			goto out;
+		}
+	}
+
 	if (info->attrs[NBD_ATTR_SOCKETS]) {
 		struct nlattr *attr;
 		int rem, fd;
@@ -2132,7 +2168,7 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
 			if (!socks[NBD_SOCK_FD])
 				continue;
 			fd = (int)nla_get_u32(socks[NBD_SOCK_FD]);
-			ret = nbd_reconnect_socket(nbd, fd);
+			ret = nbd_reconnect_socket(nbd, fd, swap_socks);
 			if (ret) {
 				if (ret == -ENOSPC)
 					ret = 0;
include/uapi/linux/nbd-netlink.h +1 -0
index c5d0ef7aa7d5..0e406445f06c 100644
@@ -35,6 +35,7 @@ enum {
 	NBD_ATTR_SOCKETS,
 	NBD_ATTR_DEAD_CONN_TIMEOUT,
 	NBD_ATTR_DEVICE_LIST,
+	NBD_ATTR_SWAP_SOCKETS,
 	__NBD_ATTR_MAX,
 };
 #define NBD_ATTR_MAX (__NBD_ATTR_MAX - 1)
Signature · 1 line
2.21.0
#3 lore raw
In reply to Mike Christie
Quoted 7 lines · Mike Christie

The following patches made over Linus's tree allow setups that are using AF_UNIX sockets with a local daemon to recover from crashes or to upgrade the daemon while IO is running without having to disrupt the application (no need to reopen the device or handle IO errors). They basically just use the existing failover infrastructure, but to failover to a new socket from a non-dead socket.

Josef and Jens,

I am dropping this patchset. Do not bother reviewing it. I will post something else later.