Skip to contentSkip to conversation

net/vhost: add flag to control wait queuing

PATCH In review Yuan Wang 4 participants 5 messages Jun 1, 2022 – Jul 1, 2022 Network drivers

lore
Share & tools
lore

Yuan Wang

author
#0 lore raw

PATCH 1/1 net/vhost: add flag to control wait queuing

update_queuing_status prevents PMD queue operations from affecting the data plane by waiting for rx/tx_pkt_burst to stops accessing the vhost device.
In fact, it is only necessary to wait when destroy/stop the device, new/start device and vring_state_changed cases do not need.

Since vring is locked when vring state changes, unconditional waiting may also cause deadlocks.

To avoid deadlocks and unnecessary waiting, this patch adds a flag to control whether waiting is required.

Fixes 9dc6bb0682 (net/vhost: fix access to freed memory)
Fixes 1ce3c7fe14 (net/vhost: emulate device start/stop behavior)
---
 drivers/net/vhost/rte_eth_vhost.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)
drivers/net/vhost/rte_eth_vhost.c +8 -8
index a248a65df4..a280e788fb 100644
@@ -716,7 +716,7 @@ eth_vhost_install_intr(struct rte_eth_dev *dev)
 }
 
 static void
-update_queuing_status(struct rte_eth_dev *dev)
+update_queuing_status(struct rte_eth_dev *dev, bool wait_queuing)
 {
 	struct pmd_internal *internal = dev->data->dev_private;
 	struct vhost_queue *vq;
@@ -742,7 +742,7 @@ update_queuing_status(struct rte_eth_dev *dev)
 			rte_atomic32_set(&vq->allow_queuing, 1);
 		else
 			rte_atomic32_set(&vq->allow_queuing, 0);
-		while (rte_atomic32_read(&vq->while_queuing))
+		while (wait_queuing && rte_atomic32_read(&vq->while_queuing))
 			rte_pause();
 	}
 
@@ -754,7 +754,7 @@ update_queuing_status(struct rte_eth_dev *dev)
 			rte_atomic32_set(&vq->allow_queuing, 1);
 		else
 			rte_atomic32_set(&vq->allow_queuing, 0);
-		while (rte_atomic32_read(&vq->while_queuing))
+		while (wait_queuing && rte_atomic32_read(&vq->while_queuing))
 			rte_pause();
 	}
 }
@@ -836,7 +836,7 @@ new_device(int vid)
 	eth_dev->data->dev_link.link_status = RTE_ETH_LINK_UP;
 
 	rte_atomic32_set(&internal->dev_attached, 1);
-	update_queuing_status(eth_dev);
+	update_queuing_status(eth_dev, false);
 
 	VHOST_LOG(INFO, "Vhost device %d created\n", vid);
 
@@ -866,7 +866,7 @@ destroy_device(int vid)
 	internal = eth_dev->data->dev_private;
 
 	rte_atomic32_set(&internal->dev_attached, 0);
-	update_queuing_status(eth_dev);
+	update_queuing_status(eth_dev, true);
 
 	eth_dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN;
 
@@ -976,7 +976,7 @@ vring_state_changed(int vid, uint16_t vring, int enable)
 	state->max_vring = RTE_MAX(vring, state->max_vring);
 	rte_spinlock_unlock(&state->lock);
 
-	update_queuing_status(eth_dev);
+	update_queuing_status(eth_dev, false);
 
 	VHOST_LOG(INFO, "vring%u is %s\n",
 			vring, enable ? "enabled" : "disabled");
@@ -1163,7 +1163,7 @@ eth_dev_start(struct rte_eth_dev *eth_dev)
 	}
 
 	rte_atomic32_set(&internal->started, 1);
-	update_queuing_status(eth_dev);
+	update_queuing_status(eth_dev, false);
 
 	return 0;
 }
@@ -1175,7 +1175,7 @@ eth_dev_stop(struct rte_eth_dev *dev)
 
 	dev->data->dev_started = 0;
 	rte_atomic32_set(&internal->started, 0);
-	update_queuing_status(dev);
+	update_queuing_status(dev, true);
 
 	return 0;
 }
Signature · 1 line
2.25.1
Tested-by on v1 1/1
#1 lore raw
Quoted 21 lines

-----Original Message-----
From: Yuan Wang <[email protected]>
Sent: Wednesday, June 1, 2022 10:26 PM
To: [email protected]; Xia, Chenbo <[email protected]>; [email protected]
Cc: Hu, Jiayu <[email protected]>; He, Xingguang <[email protected]>; Wang, YuanX <[email protected]>; [email protected]
Subject: [PATCH] net/vhost: add flag to control wait queuing

update_queuing_status prevents PMD queue operations from affecting the data plane by waiting for rx/tx_pkt_burst to stops accessing the vhost device. In fact, it is only necessary to wait when destroy/stop the device, new/start device and vring_state_changed cases do not need.

Since vring is locked when vring state changes, unconditional waiting may also cause deadlocks.

To avoid deadlocks and unnecessary waiting, this patch adds a flag to control whether waiting is required.

Quoted 6 lines
Fixes 9dc6bb0682 (net/vhost: fix access to freed memory)
Fixes 1ce3c7fe14 (net/vhost: emulate device start/stop behavior)

---

author
#2 lore raw

PATCH v2 1/1 net/vhost: fix deadlock on vring state change

In reply to Yuan Wang

If vring state changes after pmd starts working, the locked vring notifies pmd, thus calling update_queuing_status(), the latter will wait for pmd to finish accessing vring, while pmd is also waiting for vring to be unlocked, thus causing deadlock.

Actually, update_queuing_status() only needs to wait while destroy/stopping the device, but not in other cases.

This patch adds a flag for whether or not to wait to fix this issue.

Fixes 1ce3c7fe149f ("net/vhost: emulate device start/stop behavior")

---
V2: rewrite the commit log.

---
 drivers/net/vhost/rte_eth_vhost.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)
drivers/net/vhost/rte_eth_vhost.c +8 -8
index d75d256040..7e512d94bf 100644
@@ -741,7 +741,7 @@ eth_vhost_install_intr(struct rte_eth_dev *dev)
 }
 
 static void
-update_queuing_status(struct rte_eth_dev *dev)
+update_queuing_status(struct rte_eth_dev *dev, bool wait_queuing)
 {
 	struct pmd_internal *internal = dev->data->dev_private;
 	struct vhost_queue *vq;
@@ -767,7 +767,7 @@ update_queuing_status(struct rte_eth_dev *dev)
 			rte_atomic32_set(&vq->allow_queuing, 1);
 		else
 			rte_atomic32_set(&vq->allow_queuing, 0);
-		while (rte_atomic32_read(&vq->while_queuing))
+		while (wait_queuing && rte_atomic32_read(&vq->while_queuing))
 			rte_pause();
 	}
 
@@ -779,7 +779,7 @@ update_queuing_status(struct rte_eth_dev *dev)
 			rte_atomic32_set(&vq->allow_queuing, 1);
 		else
 			rte_atomic32_set(&vq->allow_queuing, 0);
-		while (rte_atomic32_read(&vq->while_queuing))
+		while (wait_queuing && rte_atomic32_read(&vq->while_queuing))
 			rte_pause();
 	}
 }
@@ -868,7 +868,7 @@ new_device(int vid)
 	vhost_dev_csum_configure(eth_dev);
 
 	rte_atomic32_set(&internal->dev_attached, 1);
-	update_queuing_status(eth_dev);
+	update_queuing_status(eth_dev, false);
 
 	VHOST_LOG(INFO, "Vhost device %d created\n", vid);
 
@@ -898,7 +898,7 @@ destroy_device(int vid)
 	internal = eth_dev->data->dev_private;
 
 	rte_atomic32_set(&internal->dev_attached, 0);
-	update_queuing_status(eth_dev);
+	update_queuing_status(eth_dev, true);
 
 	eth_dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN;
 
@@ -1008,7 +1008,7 @@ vring_state_changed(int vid, uint16_t vring, int enable)
 	state->max_vring = RTE_MAX(vring, state->max_vring);
 	rte_spinlock_unlock(&state->lock);
 
-	update_queuing_status(eth_dev);
+	update_queuing_status(eth_dev, false);
 
 	VHOST_LOG(INFO, "vring%u is %s\n",
 			vring, enable ? "enabled" : "disabled");
@@ -1197,7 +1197,7 @@ eth_dev_start(struct rte_eth_dev *eth_dev)
 	}
 
 	rte_atomic32_set(&internal->started, 1);
-	update_queuing_status(eth_dev);
+	update_queuing_status(eth_dev, false);
 
 	return 0;
 }
@@ -1209,7 +1209,7 @@ eth_dev_stop(struct rte_eth_dev *dev)
 
 	dev->data->dev_started = 0;
 	rte_atomic32_set(&internal->started, 0);
-	update_queuing_status(dev);
+	update_queuing_status(dev, true);
 
 	return 0;
 }
Signature · 1 line
2.25.1
Reviewed-by on v2 1/1
#3 lore raw

RE: [PATCH v2] net/vhost: fix deadlock on vring state change

Quoted 109 lines

-----Original Message-----
From: Wang, YuanX <[email protected]>
Sent: Monday, June 27, 2022 1:51 PM
To: [email protected]; Xia, Chenbo <[email protected]>; [email protected]
Cc: Hu, Jiayu <[email protected]>; He, Xingguang <[email protected]>; Jiang, Cheng1 <[email protected]>; Ling, WeiX <[email protected]>; Wang, YuanX <[email protected]>; [email protected]
Subject: [PATCH v2] net/vhost: fix deadlock on vring state change

If vring state changes after pmd starts working, the locked vring notifies pmd, thus calling update_queuing_status(), the latter will wait for pmd to finish accessing vring, while pmd is also waiting for vring to be unlocked, thus causing deadlock.

Actually, update_queuing_status() only needs to wait while destroy/stopping the device, but not in other cases.

This patch adds a flag for whether or not to wait to fix this issue.

Fixes 1ce3c7fe149f ("net/vhost: emulate device start/stop behavior")

---
V2: rewrite the commit log.

---
 drivers/net/vhost/rte_eth_vhost.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)
drivers/net/vhost/rte_eth_vhost.c +6 -6
index d75d256040..7e512d94bf 100644
@@ -741,7 +741,7 @@ eth_vhost_install_intr(struct rte_eth_dev *dev)
 }
 
 static void
-update_queuing_status(struct rte_eth_dev *dev)
+update_queuing_status(struct rte_eth_dev *dev, bool wait_queuing)
 {
 	struct pmd_internal *internal = dev->data->dev_private;
 	struct vhost_queue *vq;
@@ -767,7 +767,7 @@ update_queuing_status(struct rte_eth_dev *dev)
 			rte_atomic32_set(&vq->allow_queuing, 1);
 		else
 			rte_atomic32_set(&vq->allow_queuing, 0);
-		while (rte_atomic32_read(&vq->while_queuing))
+		while (wait_queuing && rte_atomic32_read(&vq->while_queuing))
 			rte_pause();
 	}
 
@@ -779,7 +779,7 @@ update_queuing_status(struct rte_eth_dev *dev)
 			rte_atomic32_set(&vq->allow_queuing, 1);
 		else
 			rte_atomic32_set(&vq->allow_queuing, 0);
-		while (rte_atomic32_read(&vq->while_queuing))
+		while (wait_queuing && rte_atomic32_read(&vq->while_queuing))
 			rte_pause();
 	}
 }
@@ -868,7 +868,7 @@ new_device(int vid)
 	vhost_dev_csum_configure(eth_dev);
 
 	rte_atomic32_set(&internal->dev_attached, 1);
-	update_queuing_status(eth_dev);
+	update_queuing_status(eth_dev, false);
 
 	VHOST_LOG(INFO, "Vhost device %d created\n", vid);
 
@@ -898,7 +898,7 @@ destroy_device(int vid)
 	internal = eth_dev->data->dev_private;
 
 	rte_atomic32_set(&internal->dev_attached, 0);
-	update_queuing_status(eth_dev);
+	update_queuing_status(eth_dev, true);
 
 	eth_dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN;
 
@@ -1008,7 +1008,7 @@ vring_state_changed(int vid, uint16_t vring, int
 enable)
 	state->max_vring = RTE_MAX(vring, state->max_vring);
 	rte_spinlock_unlock(&state->lock);
 
-	update_queuing_status(eth_dev);
+	update_queuing_status(eth_dev, false);
 
 	VHOST_LOG(INFO, "vring%u is %s\n",
drivers/net/vhost/rte_eth_vhost.c +2 -3
 			vring, enable ? "enabled" : "disabled");
@@ -1197,7 +1197,7 @@ eth_dev_start(struct rte_eth_dev *eth_dev)
 	}
 
 	rte_atomic32_set(&internal->started, 1);
-	update_queuing_status(eth_dev);
+	update_queuing_status(eth_dev, false);
 
 	return 0;
 }
@@ -1209,7 +1209,7 @@ eth_dev_stop(struct rte_eth_dev *dev)
 
 	dev->data->dev_started = 0;
 	rte_atomic32_set(&internal->started, 0);
-	update_queuing_status(dev);
+	update_queuing_status(dev, true);
 
 	return 0;
 }
--
Quoted 1 line

2.25.1

#4 lore raw

Re: [PATCH v2] net/vhost: fix deadlock on vring state change

In reply to Yuan Wang
Quoted 13 lines · Yuan Wang

If vring state changes after pmd starts working, the locked vring notifies pmd, thus calling update_queuing_status(), the latter will wait for pmd to finish accessing vring, while pmd is also waiting for vring to be unlocked, thus causing deadlock.

Actually, update_queuing_status() only needs to wait while destroy/stopping the device, but not in other cases.

This patch adds a flag for whether or not to wait to fix this issue.

Fixes 1ce3c7fe149f ("net/vhost: emulate device start/stop behavior")
Quoted 7 lines

---
V2: rewrite the commit log.

---
  drivers/net/vhost/rte_eth_vhost.c | 16 ++++++++--------
  1 file changed, 8 insertions(+), 8 deletions(-)

Applied to dpdk-next-virtio/main.

Thanks,
Maxime