Skip to contentSkip to conversation

net/virtio-user: do not close tap when disabling queue pairs

PATCH Superseded Tiwei Bie 3 participants 6 messages Nov 19, 2019 – Jan 15, 2020 Network drivers

lore
Share & tools
lore

Tiwei Bie

author
#0 lore raw

PATCH 1/1 net/virtio-user: do not close tap when disabling queue pairs

Do not close the tap fds when disabling queue pairs, instead, we just need to unbind the backend. Otherwise, tap port can be destroyed unexpectedly.

Fixes e3b434818bbb ("net/virtio-user: support kernel vhost")
---
 drivers/net/virtio/virtio_user/vhost_kernel.c | 34 +++++++++++---
 .../net/virtio/virtio_user/vhost_kernel_tap.c | 44 +++++++++++++------
 .../net/virtio/virtio_user/vhost_kernel_tap.h |  3 ++
 drivers/net/virtio/virtio_user/vhost_user.c   |  4 ++
 .../net/virtio/virtio_user/virtio_user_dev.c  |  5 ++-
 .../net/virtio/virtio_user/virtio_user_dev.h  |  1 +
 6 files changed, 70 insertions(+), 21 deletions(-)
drivers/net/virtio/virtio_user/vhost_kernel.c +27 -7
index 5c81e8dd9..d48d284d4 100644
@@ -330,16 +330,34 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
 
 	vhostfd = dev->vhostfds[pair_idx];
 
+	if (dev->qp_enabled[pair_idx] == enable)
+		return 0;
+
 	if (!enable) {
-		if (dev->tapfds[pair_idx] >= 0) {
-			close(dev->tapfds[pair_idx]);
-			dev->tapfds[pair_idx] = -1;
+		tapfd = dev->tapfds[pair_idx];
+		if (vhost_kernel_set_backend(vhostfd, -1) < 0) {
+			PMD_DRV_LOG(ERR, "fail to set backend for vhost kernel");
+			return -1;
 		}
-		return vhost_kernel_set_backend(vhostfd, -1);
-	} else if (dev->tapfds[pair_idx] >= 0) {
+		if (req_mq && vhost_kernel_tap_detach_queue(tapfd) < 0) {
+			PMD_DRV_LOG(ERR, "fail to disable tap for vhost kernel");
+			return -1;
+		}
+		dev->qp_enabled[pair_idx] = false;
 		return 0;
 	}
 
+	if (dev->tapfds[pair_idx] >= 0) {
+		tapfd = dev->tapfds[pair_idx];
+		if (vhost_kernel_tap_set_offload(tapfd, dev->features) == -1)
+			return -1;
+		if (req_mq && vhost_kernel_tap_attach_queue(tapfd) < 0) {
+			PMD_DRV_LOG(ERR, "fail to enable tap for vhost kernel");
+			return -1;
+		}
+		goto set_backend;
+	}
+
 	if ((dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF)) ||
 	    (dev->features & (1ULL << VIRTIO_F_VERSION_1)))
 		hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
@@ -353,13 +371,15 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
 		return -1;
 	}
 
+	dev->tapfds[pair_idx] = tapfd;
+
+set_backend:
 	if (vhost_kernel_set_backend(vhostfd, tapfd) < 0) {
 		PMD_DRV_LOG(ERR, "fail to set backend for vhost kernel");
-		close(tapfd);
 		return -1;
 	}
 
-	dev->tapfds[pair_idx] = tapfd;
+	dev->qp_enabled[pair_idx] = true;
 	return 0;
 }
 
drivers/net/virtio/virtio_user/vhost_kernel_tap.c +31 -13

58 lines not on this page · raw message

drivers/net/virtio/virtio_user/vhost_kernel_tap.h +3 -0
index e0e95b4f5..1202932f7 100644
@@ -37,3 +37,6 @@
 
 int vhost_kernel_open_tap(char **p_ifname, int hdr_size, int req_mq,
 			 const char *mac, uint64_t features);
+int vhost_kernel_tap_attach_queue(int fd);
+int vhost_kernel_tap_detach_queue(int fd);
+int vhost_kernel_tap_set_offload(int fd, uint64_t features);
drivers/net/virtio/virtio_user/vhost_user.c +4 -0
index a4b5c25cd..d8e083ba8 100644
@@ -456,6 +456,9 @@ vhost_user_enable_queue_pair(struct virtio_user_dev *dev,
 {
 	int i;
 
+	if (dev->qp_enabled[pair_idx] == enable)
+		return 0;
+
 	for (i = 0; i < 2; ++i) {
 		struct vhost_vring_state state = {
 			.index = pair_idx * 2 + i,
@@ -466,6 +469,7 @@ vhost_user_enable_queue_pair(struct virtio_user_dev *dev,
 			return -1;
 	}
 
+	dev->qp_enabled[pair_idx] = enable;
 	return 0;
 }
 
drivers/net/virtio/virtio_user/virtio_user_dev.c +4 -1
index a4400e772..177e829c9 100644
@@ -545,8 +545,11 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
 	}
 
 	if (dev->vhostfds) {
-		for (i = 0; i < dev->max_queue_pairs; ++i)
+		for (i = 0; i < dev->max_queue_pairs; ++i) {
 			close(dev->vhostfds[i]);
+			if (dev->tapfds[i] >= 0)
+				close(dev->tapfds[i]);
+		}
 		free(dev->vhostfds);
 		free(dev->tapfds);
 	}
drivers/net/virtio/virtio_user/virtio_user_dev.h +1 -0
index ad8683771..3b6b6065a 100644
@@ -49,6 +49,7 @@ struct virtio_user_dev {
 		struct vring_packed	packed_vrings[VIRTIO_MAX_VIRTQUEUES];
 	};
 	struct virtio_user_queue packed_queues[VIRTIO_MAX_VIRTQUEUES];
+	bool		qp_enabled[VIRTIO_MAX_VIRTQUEUE_PAIRS];
 
 	struct virtio_user_backend_ops *ops;
 	pthread_mutex_t	mutex;
Signature · 1 line
2.23.0
#1 lore raw
Quoted 11 lines · On Tue, 19 Nov 2019 14:29:48 +0800 Tiwei Bie <[email protected]> wrote:
drivers/net/virtio/virtio_user/vhost_kernel_tap.c +10 -1
-static int
+int
+vhost_kernel_tap_attach_queue(int fd)
+{
+	struct ifreq ifr;
+
+	memset(&ifr, 0, sizeof(ifr));
+	ifr.ifr_flags = IFF_ATTACH_QUEUE;
+	return ioctl(fd, TUNSETQUEUE, (void *)&ifr);
+}
+

drivers/net/virtio/virtio_user/vhost_kernel_tap.c line 40

Quoted 10 lines
drivers/net/virtio/virtio_user/vhost_kernel_tap.c +10 -0
+int
+vhost_kernel_tap_detach_queue(int fd)
+{
+	struct ifreq ifr;
+
+	memset(&ifr, 0, sizeof(ifr));
+	ifr.ifr_flags = IFF_DETACH_QUEUE;
+	return ioctl(fd, TUNSETQUEUE, (void *)&ifr);
+}
+

These both could be squashed into one routine.

int
vhost_kernel_tap_set_queue(int fd, bool attach)

{
	struct ifreq ifr = {
		.ifr_flags = attach ? IFF_ATTACH_QUEUE : IFF_DETACH_QUEUE
	};

	return ioctl(fd, TUNSETQUEUE, &ifr);
}

1. Use initializer instead of memset
2. Cast to void * is unnecessary

author
#2 lore raw
Quoted 37 lines · Stephen Hemminger
Quoted 11 lines · On Tue, 19 Nov 2019 14:29:48 +0800 Tiwei Bie <[email protected]> wrote:
drivers/net/virtio/virtio_user/vhost_kernel_tap.c +10 -1
-static int
+int
+vhost_kernel_tap_attach_queue(int fd)
+{
+	struct ifreq ifr;
+
+	memset(&ifr, 0, sizeof(ifr));
+	ifr.ifr_flags = IFF_ATTACH_QUEUE;
+	return ioctl(fd, TUNSETQUEUE, (void *)&ifr);
+}
+

drivers/net/virtio/virtio_user/vhost_kernel_tap.c line 40

Quoted 10 lines
drivers/net/virtio/virtio_user/vhost_kernel_tap.c +10 -0
+int
+vhost_kernel_tap_detach_queue(int fd)
+{
+	struct ifreq ifr;
+
+	memset(&ifr, 0, sizeof(ifr));
+	ifr.ifr_flags = IFF_DETACH_QUEUE;
+	return ioctl(fd, TUNSETQUEUE, (void *)&ifr);
+}
+

These both could be squashed into one routine.

int
vhost_kernel_tap_set_queue(int fd, bool attach)

{
	struct ifreq ifr = {
		.ifr_flags = attach ? IFF_ATTACH_QUEUE : IFF_DETACH_QUEUE
	};

	return ioctl(fd, TUNSETQUEUE, &ifr);
}
Quoted 2 lines

1. Use initializer instead of memset
2. Cast to void * is unnecessary

Thanks! Will do it in v2.

Regards,
Tiwei

author
#3 lore raw

PATCH v2 1/1 net/virtio-user: do not close tap when disabling queue pairs

In reply to Tiwei Bie

Do not close the tap fds when disabling queue pairs, instead, we just need to unbind the backend. Otherwise, tap port can be destroyed unexpectedly.

Fixes e3b434818bbb ("net/virtio-user: support kernel vhost")

---
v2:
- Squash attach_queue/detach_queue into one routine (Stephen);
- Use initializer instead of memset (Stephen);
- Drop the cast to void * (Stephen);

 drivers/net/virtio/virtio_user/vhost_kernel.c | 34 +++++++++++++++----
 .../net/virtio/virtio_user/vhost_kernel_tap.c | 34 ++++++++++++-------
 .../net/virtio/virtio_user/vhost_kernel_tap.h |  8 +++++
 drivers/net/virtio/virtio_user/vhost_user.c   |  4 +++
 .../net/virtio/virtio_user/virtio_user_dev.c  |  5 ++-
 .../net/virtio/virtio_user/virtio_user_dev.h  |  1 +
 6 files changed, 65 insertions(+), 21 deletions(-)
drivers/net/virtio/virtio_user/vhost_kernel.c +27 -7
index 5c81e8dd9..2c805077a 100644
@@ -330,16 +330,34 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
 
 	vhostfd = dev->vhostfds[pair_idx];
 
+	if (dev->qp_enabled[pair_idx] == enable)
+		return 0;
+
 	if (!enable) {
-		if (dev->tapfds[pair_idx] >= 0) {
-			close(dev->tapfds[pair_idx]);
-			dev->tapfds[pair_idx] = -1;
+		tapfd = dev->tapfds[pair_idx];
+		if (vhost_kernel_set_backend(vhostfd, -1) < 0) {
+			PMD_DRV_LOG(ERR, "fail to set backend for vhost kernel");
+			return -1;
 		}
-		return vhost_kernel_set_backend(vhostfd, -1);
-	} else if (dev->tapfds[pair_idx] >= 0) {
+		if (req_mq && vhost_kernel_tap_set_queue(tapfd, false) < 0) {
+			PMD_DRV_LOG(ERR, "fail to disable tap for vhost kernel");
+			return -1;
+		}
+		dev->qp_enabled[pair_idx] = false;
 		return 0;
 	}
 
+	if (dev->tapfds[pair_idx] >= 0) {
+		tapfd = dev->tapfds[pair_idx];
+		if (vhost_kernel_tap_set_offload(tapfd, dev->features) == -1)
+			return -1;
+		if (req_mq && vhost_kernel_tap_set_queue(tapfd, true) < 0) {
+			PMD_DRV_LOG(ERR, "fail to enable tap for vhost kernel");
+			return -1;
+		}
+		goto set_backend;
+	}
+
 	if ((dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF)) ||
 	    (dev->features & (1ULL << VIRTIO_F_VERSION_1)))
 		hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
@@ -353,13 +371,15 @@ vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
 		return -1;
 	}
 
+	dev->tapfds[pair_idx] = tapfd;
+
+set_backend:
 	if (vhost_kernel_set_backend(vhostfd, tapfd) < 0) {
 		PMD_DRV_LOG(ERR, "fail to set backend for vhost kernel");
-		close(tapfd);
 		return -1;
 	}
 
-	dev->tapfds[pair_idx] = tapfd;
+	dev->qp_enabled[pair_idx] = true;
 	return 0;
 }
 
drivers/net/virtio/virtio_user/vhost_kernel_tap.c +21 -13
index 76bf75423..2fbfecba1 100644
@@ -18,7 +18,7 @@
 #include "../virtio_logs.h"
 #include "../virtio_pci.h"
 
-static int
+int
 vhost_kernel_tap_set_offload(int fd, uint64_t features)
 {
 	unsigned int offload = 0;
@@ -37,26 +37,34 @@ vhost_kernel_tap_set_offload(int fd, uint64_t features)
 			offload |= TUN_F_UFO;
 	}
 
-	if (offload != 0) {
-		/* Check if our kernel supports TUNSETOFFLOAD */
-		if (ioctl(fd, TUNSETOFFLOAD, 0) != 0 && errno == EINVAL) {
-			PMD_DRV_LOG(ERR, "Kernel does't support TUNSETOFFLOAD\n");
-			return -ENOTSUP;
-		}
+	/* Check if our kernel supports TUNSETOFFLOAD */
+	if (ioctl(fd, TUNSETOFFLOAD, 0) != 0 && errno == EINVAL) {
+		PMD_DRV_LOG(ERR, "Kernel does't support TUNSETOFFLOAD\n");
+		return -ENOTSUP;
+	}
 
+	if (ioctl(fd, TUNSETOFFLOAD, offload) != 0) {
+		offload &= ~TUN_F_UFO;
 		if (ioctl(fd, TUNSETOFFLOAD, offload) != 0) {
-			offload &= ~TUN_F_UFO;
-			if (ioctl(fd, TUNSETOFFLOAD, offload) != 0) {
-				PMD_DRV_LOG(ERR, "TUNSETOFFLOAD ioctl() failed: %s\n",
-					strerror(errno));
-				return -1;
-			}
+			PMD_DRV_LOG(ERR, "TUNSETOFFLOAD ioctl() failed: %s\n",
+				strerror(errno));
+			return -1;
 		}
 	}
 
 	return 0;
 }
 
+int
+vhost_kernel_tap_set_queue(int fd, bool attach)
+{
+	struct ifreq ifr = {
+		.ifr_flags = attach ? IFF_ATTACH_QUEUE : IFF_DETACH_QUEUE,
+	};
+
+	return ioctl(fd, TUNSETQUEUE, &ifr);
+}
+
 int
 vhost_kernel_open_tap(char **p_ifname, int hdr_size, int req_mq,
 			 const char *mac, uint64_t features)
drivers/net/virtio/virtio_user/vhost_kernel_tap.h +8 -0
index e0e95b4f5..5c4447b29 100644
@@ -2,6 +2,10 @@
  * Copyright(c) 2016 Intel Corporation
  */
 
+#ifndef _VHOST_KERNEL_TAP_H
+#define _VHOST_KERNEL_TAP_H
+
+#include <stdbool.h>
 #include <sys/ioctl.h>
 
 /* TUN ioctls */
@@ -37,3 +41,7 @@
 
 int vhost_kernel_open_tap(char **p_ifname, int hdr_size, int req_mq,
 			 const char *mac, uint64_t features);
+int vhost_kernel_tap_set_offload(int fd, uint64_t features);
+int vhost_kernel_tap_set_queue(int fd, bool attach);
+
+#endif
drivers/net/virtio/virtio_user/vhost_user.c +4 -0
index a4b5c25cd..d8e083ba8 100644
@@ -456,6 +456,9 @@ vhost_user_enable_queue_pair(struct virtio_user_dev *dev,
 {
 	int i;
 
+	if (dev->qp_enabled[pair_idx] == enable)
+		return 0;
+
 	for (i = 0; i < 2; ++i) {
 		struct vhost_vring_state state = {
 			.index = pair_idx * 2 + i,
@@ -466,6 +469,7 @@ vhost_user_enable_queue_pair(struct virtio_user_dev *dev,
 			return -1;
 	}
 
+	dev->qp_enabled[pair_idx] = enable;
 	return 0;
 }
 
drivers/net/virtio/virtio_user/virtio_user_dev.c +4 -1
index a4400e772..177e829c9 100644
@@ -545,8 +545,11 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
 	}
 
 	if (dev->vhostfds) {
-		for (i = 0; i < dev->max_queue_pairs; ++i)
+		for (i = 0; i < dev->max_queue_pairs; ++i) {
 			close(dev->vhostfds[i]);
+			if (dev->tapfds[i] >= 0)
+				close(dev->tapfds[i]);
+		}
 		free(dev->vhostfds);
 		free(dev->tapfds);
 	}
drivers/net/virtio/virtio_user/virtio_user_dev.h +1 -0
index ad8683771..3b6b6065a 100644
@@ -49,6 +49,7 @@ struct virtio_user_dev {
 		struct vring_packed	packed_vrings[VIRTIO_MAX_VIRTQUEUES];
 	};
 	struct virtio_user_queue packed_queues[VIRTIO_MAX_VIRTQUEUES];
+	bool		qp_enabled[VIRTIO_MAX_VIRTQUEUE_PAIRS];
 
 	struct virtio_user_backend_ops *ops;
 	pthread_mutex_t	mutex;
Signature · 1 line
2.24.0
Reviewed-by on v2 1/1
#4 lore raw
Quoted 15 lines · Tiwei Bie

Do not close the tap fds when disabling queue pairs, instead, we just need to unbind the backend. Otherwise, tap port can be destroyed unexpectedly.

Fixes e3b434818bbb ("net/virtio-user: support kernel vhost")

---
v2:
- Squash attach_queue/detach_queue into one routine (Stephen);
- Use initializer instead of memset (Stephen);
- Drop the cast to void * (Stephen);

Quoted 8 lines
 drivers/net/virtio/virtio_user/vhost_kernel.c | 34 +++++++++++++++----
 .../net/virtio/virtio_user/vhost_kernel_tap.c | 34 ++++++++++++-------
 .../net/virtio/virtio_user/vhost_kernel_tap.h |  8 +++++
 drivers/net/virtio/virtio_user/vhost_user.c   |  4 +++
 .../net/virtio/virtio_user/virtio_user_dev.c  |  5 ++-
 .../net/virtio/virtio_user/virtio_user_dev.h  |  1 +
 6 files changed, 65 insertions(+), 21 deletions(-)

Thanks,
Maxime

#5 lore raw
In reply to Tiwei Bie
Quoted 15 lines · Tiwei Bie

Do not close the tap fds when disabling queue pairs, instead, we just need to unbind the backend. Otherwise, tap port can be destroyed unexpectedly.

Fixes e3b434818bbb ("net/virtio-user: support kernel vhost")

---
v2:
- Squash attach_queue/detach_queue into one routine (Stephen);
- Use initializer instead of memset (Stephen);
- Drop the cast to void * (Stephen);

Quoted 7 lines
 drivers/net/virtio/virtio_user/vhost_kernel.c | 34 +++++++++++++++----
 .../net/virtio/virtio_user/vhost_kernel_tap.c | 34 ++++++++++++-------
 .../net/virtio/virtio_user/vhost_kernel_tap.h |  8 +++++
 drivers/net/virtio/virtio_user/vhost_user.c   |  4 +++
 .../net/virtio/virtio_user/virtio_user_dev.c  |  5 ++-
 .../net/virtio/virtio_user/virtio_user_dev.h  |  1 +
 6 files changed, 65 insertions(+), 21 deletions(-)

Applied to dpdk-next-virtio/master

Thanks,
Maxime