nbd: local daemon restart support
Subsystems (3)
cover nbd: local daemon restart support
The following patches made over Linus's tree allow setups that are using AF_UNIX sockets with a local daemon to recover from crashes or to upgrade the daemon while IO is running without having to disrupt the application (no need to reopen the device or handle IO errors). They basically just use the existing failover infrastructure, but to failover to a new socket from a non-dead socket.
PATCH 1/2 nbd: move send_disconnects
Just move send_disconnects so we can call it earlier in the file in the next patch.
--- drivers/block/nbd.c | 48 ++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-)
drivers/block/nbd.c +24 -24
@@ -980,6 +980,30 @@ static blk_status_t nbd_queue_rq(struct blk_mq_hw_ctx *hctx,return ret; } +static void send_disconnects (struct nbd_device *nbd) +{ +struct nbd_config *config = nbd->config; +struct nbd_request request = { + .magic =htonl (NBD_REQUEST_MAGIC ), + .type =htonl (NBD_CMD_DISC ), + }; +struct kvec iov = {.iov_base = &request, .iov_len =sizeof (request)}; +struct iov_iter from; +int i, ret; + +for (i =0 ; i < config->num_connections; i++) { +struct nbd_sock *nsock = config->socks[i]; + +iov_iter_kvec (&from,WRITE , &iov,1 ,sizeof (request)); +mutex_lock (&nsock->tx_lock); + ret =sock_xmit (nbd, i,1 , &from,0 ,NULL ); +if (ret <=0 ) +dev_err (disk_to_dev (nbd->disk), +"Send disconnect failed %d\n" , ret); +mutex_unlock (&nsock->tx_lock); + } +} +static struct socket *nbd_get_socket (struct nbd_device *nbd,unsigned long fd,int *err) {
@@ -1139,30 +1163,6 @@ static void nbd_parse_flags(struct nbd_device *nbd)blk_queue_write_cache (nbd->disk->queue, false, false); } -static void send_disconnects (struct nbd_device *nbd) -{ -struct nbd_config *config = nbd->config; -struct nbd_request request = { - .magic =htonl (NBD_REQUEST_MAGIC ), - .type =htonl (NBD_CMD_DISC ), - }; -struct kvec iov = {.iov_base = &request, .iov_len =sizeof (request)}; -struct iov_iter from; -int i, ret; - -for (i =0 ; i < config->num_connections; i++) { -struct nbd_sock *nsock = config->socks[i]; - -iov_iter_kvec (&from,WRITE , &iov,1 ,sizeof (request)); -mutex_lock (&nsock->tx_lock); - ret =sock_xmit (nbd, i,1 , &from,0 ,NULL ); -if (ret <=0 ) -dev_err (disk_to_dev (nbd->disk), -"Send disconnect failed %d\n" , ret); -mutex_unlock (&nsock->tx_lock); - } -} -static int nbd_disconnect (struct nbd_device *nbd) {struct nbd_config *config = nbd->config;
Signature · 1 line
2.21.0
PATCH 2/2 nbd: add support to replace running daemon
In reply to Mike ChristieThis patch allows userspace to switch daemons while IO is running or to recovery from a crashed daemon. With the patch when using AF_UNIX sockets and a local running daemon, the new daemon can perform a NBD_CMD_RECONFIGURE nl command with the NBD_ATTR_SWAP_SOCKETS attr set to 0. The kernel will then switch over to using the new socket similar to how we handle failing over from a dead socket.
--- drivers/block/nbd.c | 44 +++++++++++++++++++++++++++++--- include/uapi/linux/nbd-netlink.h | 1 + 2 files changed, 41 insertions(+), 4 deletions(-)
drivers/block/nbd.c +40 -4
@@ -1076,7 +1076,14 @@ static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg,return 0 ; } -static int nbd_reconnect_socket (struct nbd_device *nbd,unsigned long arg) +static bool nbd_requeue_req (struct request *req,void *data,bool reserved) +{ +nbd_requeue_cmd (blk_mq_rq_to_pdu (req)); +return true; +} + +static int nbd_reconnect_socket (struct nbd_device *nbd,unsigned long arg, +u8 swap_socks) {struct nbd_config *config = nbd->config;struct socket *sock, *old;
@@ -1094,6 +1101,16 @@ static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)return -ENOMEM ; } +if (swap_socks) { +dev_dbg (disk_to_dev (nbd->disk),"stopping queue\n" ); + +blk_mq_quiesce_queue (nbd->disk->queue); +send_disconnects (nbd); +sock_shutdown (nbd); +flush_workqueue (nbd->recv_workq); +blk_mq_tagset_busy_iter (&nbd->tag_set, nbd_requeue_req,NULL ); + } +for (i =0 ; i < config->num_connections; i++) {struct nbd_sock *nsock = config->socks[i];
@@ -1130,11 +1147,20 @@ static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)atomic_inc (&config->live_connections);wake_up (&config->conn_wait); -return 0 ; + err =0 ; + goto start_queue; }sockfd_put (sock);kfree (args); -return -ENOSPC ; + err = -ENOSPC ; + +start_queue: +if (swap_socks) { + nbd->task_recv = current; +blk_mq_unquiesce_queue (nbd->disk->queue); +dev_dbg (disk_to_dev (nbd->disk),"queue restarted\n" ); + } +return err; }static void nbd_bdev_reset (struct block_device *bdev)
@@ -2027,6 +2053,7 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info) {struct nbd_device *nbd =NULL ;struct nbd_config *config; +u8 swap_socks =0 ;int index;int ret =0 ;bool put_dev = false;
@@ -2107,6 +2134,15 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info) } } +if (info->attrs[NBD_ATTR_SWAP_SOCKETS ]) { + swap_socks =nla_get_u8 (info->attrs[NBD_ATTR_SWAP_SOCKETS ]); +if (swap_socks !=1 && swap_socks !=0 ) { +printk (KERN_ERR "nbd: NBD_SOCK_SWAP must be 0 or 1\n" ); + ret = -EINVAL ; + goto out; + } + } +if (info->attrs[NBD_ATTR_SOCKETS ]) {struct nlattr *attr;int rem, fd;
@@ -2132,7 +2168,7 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)if (!socks[NBD_SOCK_FD ])continue ; fd = (int )nla_get_u32 (socks[NBD_SOCK_FD ]); - ret =nbd_reconnect_socket (nbd, fd); + ret =nbd_reconnect_socket (nbd, fd, swap_socks);if (ret) {if (ret == -ENOSPC ) ret =0 ;
include/uapi/linux/nbd-netlink.h +1 -0
@@ -35,6 +35,7 @@ enum {NBD_ATTR_SOCKETS ,NBD_ATTR_DEAD_CONN_TIMEOUT ,NBD_ATTR_DEVICE_LIST , +NBD_ATTR_SWAP_SOCKETS , __NBD_ATTR_MAX, };#define NBD_ATTR_MAX (__NBD_ATTR_MAX - 1)
Signature · 1 line
2.21.0
Quoted 7 lines · Mike Christie
The following patches made over Linus's tree allow setups that are using
AF_UNIXsockets with a local daemon to recover from crashes or to upgrade the daemon while IO is running without having to disrupt the application (no need to reopen the device or handle IO errors). They basically just use the existing failover infrastructure, but to failover to a new socket from a non-dead socket.
Josef and Jens,
I am dropping this patchset. Do not bother reviewing it. I will post something else later.