Skip to contentSkip to conversation

vhost: fix device IOTLB feature lifecycle

PATCH Superseded Jia Jia 2 participants 7 messages Aug 18 – Aug 19

Subsystems (6)

v6 of 9 Superseded Latest is v9 →

lore
Share & tools
lore

Jia Jia

author
#0 lore raw

v6 cover vhost: fix device IOTLB feature lifecycle

Both vhost-vsock and vhost-net can leave the device IOTLB attached when userspace clears VIRTIO_F_ACCESS_PLATFORM. They can also replace an existing IOTLB with a new empty table when a later feature update keeps ACCESS_PLATFORM enabled, for example when updating logging.

When the IOTLB mode changes, the vring addresses previously supplied by userspace no longer have the same address-space meaning. Leaving those addresses installed would allow an old IOVA to be used as a direct userspace address after the IOTLB is detached.

This series invalidates the vring access state during IOTLB transitions, makes IOTLB initialization idempotent, and uses a common teardown helper for vhost-vsock and vhost-net. IOTLB mode changes are applied even while a virtqueue backend is attached. The device-wide IOTLB is dropped first, each virtqueue then clears its IOTLB pointer and cached ring access under its own mutex, and the old table is freed only after every virtqueue has completed the handoff.

A successful live mode change leaves the backend attached but invalidates the cached vring addresses. Userspace must configure the vring addresses for the new address mode before data processing can resume. When ACCESS_PLATFORM is enabled, the usual IOTLB miss/update protocol repopulates the new table.

Changes since v5:
- invalidate desc, avail, used, logging state, and metadata on IOTLB transitions;
- apply IOTLB mode changes while a backend is attached, following the per-virtqueue handoff suggested in review;
- apply the IOTLB teardown to vhost-net as well as vhost-vsock.

Jia Jia (3):
vhost: invalidate vring access on IOTLB transitions vhost/vsock: discard IOTLB when ACCESS_PLATFORM is cleared vhost/net: discard IOTLB when ACCESS_PLATFORM is cleared

 drivers/vhost/net.c   |  8 ++++++--
 drivers/vhost/vhost.c | 53 ++++++++++++++++++++++++++++++++++++++++-
 drivers/vhost/vhost.h |  1 +
 drivers/vhost/vsock.c | 10 ++++++---
 4 files changed, 66 insertions(+), 6 deletions(-)
author
#1 lore raw

PATCH v6 1/3 vhost: invalidate vring access on IOTLB transitions

When ACCESS_PLATFORM changes, the addresses cached in desc, avail, and used change meaning with the address space. Clear the cached vring access state when the device IOTLB is installed or removed so stale IOVAs cannot be reused as direct userspace addresses.

Keep device IOTLB initialization idempotent and apply the mode change even when a virtqueue backend is attached. Drop the device-wide IOTLB first, then clear each VQ state under its own mutex, and keep the old table alive until every VQ has completed the handoff.

A successful live mode change leaves the backend attached but invalidates the cached vring addresses. Userspace must configure the vring addresses for the new address mode before data processing can resume.

Fixes 6b1e6cc7855b ("vhost: new device IOTLB API")
---
 drivers/vhost/vhost.c | 53 ++++++++++++++++++++++++++++++++++++++++++-
 drivers/vhost/vhost.h |  1 +
 2 files changed, 53 insertions(+), 1 deletion(-)
drivers/vhost/vhost.c +52 -1
index 4c525b3e16ea..9f74537b1c1c 100644
@@ -344,6 +344,17 @@ static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq)
 		vq->meta_iotlb[j] = NULL;
 }
 
+/* Caller must hold the virtqueue mutex. */
+static void vhost_vq_invalidate_access(struct vhost_virtqueue *vq)
+{
+	vq->desc = NULL;
+	vq->avail = NULL;
+	vq->used = NULL;
+	vq->log_used = false;
+	vq->log_addr = -1ull;
+	__vhost_vq_meta_reset(vq);
+}
+
 static void vhost_vq_meta_reset(struct vhost_dev *d)
 {
 	int i;
@@ -1911,6 +1922,9 @@ int vq_meta_prefetch(struct vhost_virtqueue *vq)
 {
 	unsigned int num = vq->num;
 
+	if (!vq->desc || !vq->avail || !vq->used)
+		return 0;
+
 	if (!vq->iotlb)
 		return 1;
 
@@ -2270,11 +2284,48 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg
 }
 EXPORT_SYMBOL_GPL(vhost_vring_ioctl);
 
+/* Caller must hold the device mutex. */
+void vhost_clear_device_iotlb(struct vhost_dev *d)
+{
+	struct vhost_iotlb *iotlb;
+	int i;
+
+	iotlb = d->iotlb;
+	if (!iotlb)
+		return;
+
+	/*
+	 * Drop the device-wide view first.  Each VQ then drops its
+	 * per-VQ view and its cached ring access under its own mutex.
+	 * Keep the old table alive until every VQ has completed this
+	 * handoff, since a worker may still be using it while waiting
+	 * for its VQ mutex.
+	 */
+	d->iotlb = NULL;
+
+	for (i = 0; i < d->nvqs; ++i) {
+		struct vhost_virtqueue *vq = d->vqs[i];
+
+		mutex_lock(&vq->mutex);
+		vq->iotlb = NULL;
+		vhost_vq_invalidate_access(vq);
+		mutex_unlock(&vq->mutex);
+	}
+
+	vhost_clear_msg(d);
+	vhost_iotlb_free(iotlb);
+	wake_up_interruptible_poll(&d->wait, EPOLLIN | EPOLLRDNORM);
+}
+EXPORT_SYMBOL_GPL(vhost_clear_device_iotlb);
+
 int vhost_init_device_iotlb(struct vhost_dev *d)
 {
 	struct vhost_iotlb *niotlb, *oiotlb;
 	int i;
 
+	if (d->iotlb)
+		return 0;
+
 	niotlb = iotlb_alloc();
 	if (!niotlb)
 		return -ENOMEM;
@@ -2287,7 +2338,7 @@ int vhost_init_device_iotlb(struct vhost_dev *d)
 
 		mutex_lock(&vq->mutex);
 		vq->iotlb = niotlb;
-		__vhost_vq_meta_reset(vq);
+		vhost_vq_invalidate_access(vq);
 		mutex_unlock(&vq->mutex);
 	}
 
drivers/vhost/vhost.h +1 -0
index 0192ade6e749..3c75e8089373 100644
@@ -277,6 +277,7 @@ ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to,
 			    int noblock);
 ssize_t vhost_chr_write_iter(struct vhost_dev *dev,
 			     struct iov_iter *from);
+void vhost_clear_device_iotlb(struct vhost_dev *d);
 int vhost_init_device_iotlb(struct vhost_dev *d);
 
 void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
author
#4 lore raw

PATCH v6 2/3 vhost/vsock: discard IOTLB when ACCESS_PLATFORM is cleared

In reply to Jia Jia

Clear the device IOTLB when userspace clears VIRTIO_F_ACCESS_PLATFORM. Otherwise descriptor translation can continue to use mappings installed before the feature change.

The common helper invalidates cached vring access and applies the transition even while a backend is attached. The backend remains attached, but userspace must configure the vring addresses for the new address mode after a successful live transition.

Fixes e13a6915a03f ("vhost/vsock: add IOTLB API support")
---
 drivers/vhost/vsock.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)
drivers/vhost/vsock.c +7 -3
index 9aaab6bb8061..77a5a6775c86 100644
@@ -854,6 +854,7 @@ static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid)
 static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)
 {
 	struct vhost_virtqueue *vq;
+	int ret = -EFAULT;
 	int i;
 
 	if (features & ~VHOST_VSOCK_FEATURES)
@@ -865,9 +866,12 @@ static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)
 		goto err;
 	}
 
-	if ((features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) {
-		if (vhost_init_device_iotlb(&vsock->dev))
+	if (features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)) {
+		ret = vhost_init_device_iotlb(&vsock->dev);
+		if (ret)
 			goto err;
+	} else {
+		vhost_clear_device_iotlb(&vsock->dev);
 	}
 
 	vsock->seqpacket_allow = features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET);
@@ -883,7 +887,7 @@ static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)
 
 err:
 	mutex_unlock(&vsock->dev.mutex);
-	return -EFAULT;
+	return ret;
 }
 
 static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl,
author
#6 lore raw

PATCH v6 3/3 vhost/net: discard IOTLB when ACCESS_PLATFORM is cleared

In reply to Jia Jia

Apply the common device IOTLB teardown when userspace clears VIRTIO_F_ACCESS_PLATFORM. This drops stale translations and avoids rebuilding an existing IOTLB during feature updates that keep ACCESS_PLATFORM enabled.

The transition invalidates cached vring access even with an attached backend. The backend remains attached, but userspace must configure the vring addresses for the new address mode after a successful live transition.

Fixes 6b1e6cc7855b ("vhost: new device IOTLB API")
---
 drivers/vhost/net.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
drivers/vhost/net.c +6 -2
index 77b59f49bddb..64671cf77a35 100644
@@ -1673,6 +1673,7 @@ static long vhost_net_reset_owner(struct vhost_net *n)
 static int vhost_net_set_features(struct vhost_net *n, const u64 *features)
 {
 	size_t vhost_hlen, sock_hlen, hdr_len;
+	int ret = -EFAULT;
 	int i;
 
 	hdr_len = virtio_features_test_bit(features, VIRTIO_NET_F_MRG_RXBUF) ||
@@ -1701,8 +1702,11 @@ static int vhost_net_set_features(struct vhost_net *n, const u64 *features)
 		goto out_unlock;
 
 	if (virtio_features_test_bit(features, VIRTIO_F_ACCESS_PLATFORM)) {
-		if (vhost_init_device_iotlb(&n->dev))
+		ret = vhost_init_device_iotlb(&n->dev);
+		if (ret)
 			goto out_unlock;
+	} else {
+		vhost_clear_device_iotlb(&n->dev);
 	}
 
 	for (i = 0; i < VHOST_NET_VQ_MAX; ++i) {
@@ -1718,7 +1722,7 @@ static int vhost_net_set_features(struct vhost_net *n, const u64 *features)
 
 out_unlock:
 	mutex_unlock(&n->dev.mutex);
-	return -EFAULT;
+	return ret;
 }
 
 static long vhost_net_set_owner(struct vhost_net *n)
#2 lore raw

Re: [PATCH v6 1/3] vhost: invalidate vring access on IOTLB transitions

In reply to Jia Jia
Quoted 10 lines · Jia Jia

When ACCESS_PLATFORM changes, the addresses cached in desc, avail, and used change meaning with the address space. Clear the cached vring access state when the device IOTLB is installed or removed so stale IOVAs cannot be reused as direct userspace addresses.

Keep device IOTLB initialization idempotent and apply the mode change even when a virtqueue backend is attached. Drop the device-wide IOTLB first, then clear each VQ state under its own mutex, and keep the old table alive until every VQ has completed the handoff.

Quoted 6 lines

A successful live mode change leaves the backend attached but invalidates the cached vring addresses. Userspace must configure the vring addresses for the new address mode before data processing can resume.

Fixes 6b1e6cc7855b ("vhost: new device IOTLB API")

Please don't leave spaces here!

Quoted 5 lines

---
drivers/vhost/vhost.c | 53 ++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/vhost.h | 1 +

2 files changed, 53 insertions(+), 1 deletion(-)

This patch doesn't apply, I tried vhost tree, net tree, and master. On which tree is this based?

Stefano

Quoted 104 lines
drivers/vhost/vhost.c +52 -1
index 4c525b3e16ea..9f74537b1c1c 100644
@@ -344,6 +344,17 @@ static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq)
 		vq->meta_iotlb[j] = NULL;
 }
 
+/* Caller must hold the virtqueue mutex. */
+static void vhost_vq_invalidate_access(struct vhost_virtqueue *vq)
+{
+	vq->desc = NULL;
+	vq->avail = NULL;
+	vq->used = NULL;
+	vq->log_used = false;
+	vq->log_addr = -1ull;
+	__vhost_vq_meta_reset(vq);
+}
+
 static void vhost_vq_meta_reset(struct vhost_dev *d)
 {
 	int i;
@@ -1911,6 +1922,9 @@ int vq_meta_prefetch(struct vhost_virtqueue *vq)
 {
 	unsigned int num = vq->num;
 
+	if (!vq->desc || !vq->avail || !vq->used)
+		return 0;
+
 	if (!vq->iotlb)
 		return 1;
 
@@ -2270,11 +2284,48 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg
 }
 EXPORT_SYMBOL_GPL(vhost_vring_ioctl);
 
+/* Caller must hold the device mutex. */
+void vhost_clear_device_iotlb(struct vhost_dev *d)
+{
+	struct vhost_iotlb *iotlb;
+	int i;
+
+	iotlb = d->iotlb;
+	if (!iotlb)
+		return;
+
+	/*
+	 * Drop the device-wide view first.  Each VQ then drops its
+	 * per-VQ view and its cached ring access under its own mutex.
+	 * Keep the old table alive until every VQ has completed this
+	 * handoff, since a worker may still be using it while waiting
+	 * for its VQ mutex.
+	 */
+	d->iotlb = NULL;
+
+	for (i = 0; i < d->nvqs; ++i) {
+		struct vhost_virtqueue *vq = d->vqs[i];
+
+		mutex_lock(&vq->mutex);
+		vq->iotlb = NULL;
+		vhost_vq_invalidate_access(vq);
+		mutex_unlock(&vq->mutex);
+	}
+
+	vhost_clear_msg(d);
+	vhost_iotlb_free(iotlb);
+	wake_up_interruptible_poll(&d->wait, EPOLLIN | EPOLLRDNORM);
+}
+EXPORT_SYMBOL_GPL(vhost_clear_device_iotlb);
+
 int vhost_init_device_iotlb(struct vhost_dev *d)
 {
 	struct vhost_iotlb *niotlb, *oiotlb;
 	int i;
 
+	if (d->iotlb)
+		return 0;
+
 	niotlb = iotlb_alloc();
 	if (!niotlb)
 		return -ENOMEM;
@@ -2287,7 +2338,7 @@ int vhost_init_device_iotlb(struct vhost_dev *d)
 
 		mutex_lock(&vq->mutex);
 		vq->iotlb = niotlb;
-		__vhost_vq_meta_reset(vq);
+		vhost_vq_invalidate_access(vq);
 		mutex_unlock(&vq->mutex);
 	}
 
drivers/vhost/vhost.h +1 -0
index 0192ade6e749..3c75e8089373 100644
@@ -277,6 +277,7 @@ ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to,
 			    int noblock);
 ssize_t vhost_chr_write_iter(struct vhost_dev *dev,
 			     struct iov_iter *from);
+void vhost_clear_device_iotlb(struct vhost_dev *d);
 int vhost_init_device_iotlb(struct vhost_dev *d);
 
 void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
#5 lore raw

Re: [PATCH v6 2/3] vhost/vsock: discard IOTLB when ACCESS_PLATFORM is cleared

In reply to Jia Jia
Quoted 4 lines · Jia Jia

Clear the device IOTLB when userspace clears VIRTIO_F_ACCESS_PLATFORM. Otherwise descriptor translation can continue to use mappings installed before the feature change.

Quoted 8 lines

The common helper invalidates cached vring access and applies the transition even while a backend is attached. The backend remains attached, but userspace must configure the vring addresses for the new address mode after a successful live transition.

Fixes e13a6915a03f ("vhost/vsock: add IOTLB API support")

In v5 there weren't any blank lines there, why are you adding them? Please remove them.

Quoted 11 lines

---
drivers/vhost/vsock.c | 10 +++++++---

1 file changed, 7 insertions(+), 3 deletions(-)
drivers/vhost/vsock.c +0 -0
index 9aaab6bb8061..77a5a6775c86 100644

drivers/vhost/vsock.c static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid) · line 857

Quoted 4 lines
drivers/vhost/vsock.c +1 -0
@@ -854,6 +854,7 @@ static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid)
 static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)
 {
 	struct vhost_virtqueue *vq;
+	int ret = -EFAULT;

This is a return value used only in the error path IIUC, better to call it err or something like that IMO.

drivers/vhost/vsock.c static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features) · line 871

Quoted 12 lines
drivers/vhost/vsock.c +3 -2
 	int i;
 
 	if (features & ~VHOST_VSOCK_FEATURES)
@@ -865,9 +866,12 @@ static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)
 		goto err;
 	}
 
-	if ((features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) {
-		if (vhost_init_device_iotlb(&vsock->dev))
+	if (features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)) {
+		ret = vhost_init_device_iotlb(&vsock->dev);
+		if (ret)

Is this change related to this fix?

Stefano

Quoted 16 lines
drivers/vhost/vsock.c +2 -0
 			goto err;
+	} else {
+		vhost_clear_device_iotlb(&vsock->dev);
 	}
 
 	vsock->seqpacket_allow = features & (1ULL << VIRTIO_VSOCK_F_SEQPACKET);
@@ -883,7 +887,7 @@ static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)
 

err:

	mutex_unlock(&vsock->dev.mutex);
drivers/vhost/vsock.c +1 -1
-	return -EFAULT;
+	return ret;

}

static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl,

author
#3 lore raw

Re: [PATCH v6 1/3] vhost: invalidate vring access on IOTLB transitions

In reply to Stefano Garzarella
Quoted 28 lines · Jia Jia
Quoted 10 lines

When ACCESS_PLATFORM changes, the addresses cached in desc, avail, and used change meaning with the address space. Clear the cached vring access state when the device IOTLB is installed or removed so stale IOVAs cannot be reused as direct userspace addresses.

Keep device IOTLB initialization idempotent and apply the mode change even when a virtqueue backend is attached. Drop the device-wide IOTLB first, then clear each VQ state under its own mutex, and keep the old table alive until every VQ has completed the handoff.

Quoted 6 lines

A successful live mode change leaves the backend attached but invalidates the cached vring addresses. Userspace must configure the vring addresses for the new address mode before data processing can resume.

Fixes 6b1e6cc7855b ("vhost: new device IOTLB API")

Please don't leave spaces here!

Quoted 5 lines

---
drivers/vhost/vhost.c | 53 ++++++++++++++++++++++++++++++++++++++++++- drivers/vhost/vhost.h | 1 +

2 files changed, 53 insertions(+), 1 deletion(-)
Quoted 5 lines

This patch doesn't apply, I tried vhost tree, net tree, and master. On which tree is this based?

Stefano

Thanks for checking. I confirmed that v6 was generated from an older local linux-next snapshot with local changes on top,
so it does not apply cleanly to the current tree. Sorry for taking up your time. I will fix this first. Thanks again.

Quoted 105 lines
Quoted 104 lines
drivers/vhost/vhost.c +52 -1
index 4c525b3e16ea..9f74537b1c1c 100644
@@ -344,6 +344,17 @@ static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq)
              vq->meta_iotlb[j] = NULL;
 }
 
+/* Caller must hold the virtqueue mutex. */
+static void vhost_vq_invalidate_access(struct vhost_virtqueue *vq)
+{
+      vq->desc = NULL;
+      vq->avail = NULL;
+      vq->used = NULL;
+      vq->log_used = false;
+      vq->log_addr = -1ull;
+      __vhost_vq_meta_reset(vq);
+}
+
 static void vhost_vq_meta_reset(struct vhost_dev *d)
 {
      int i;
@@ -1911,6 +1922,9 @@ int vq_meta_prefetch(struct vhost_virtqueue *vq)
 {
      unsigned int num = vq->num;
 
+      if (!vq->desc || !vq->avail || !vq->used)
+              return 0;
+
      if (!vq->iotlb)
              return 1;
 
@@ -2270,11 +2284,48 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *arg
 }
 EXPORT_SYMBOL_GPL(vhost_vring_ioctl);
 
+/* Caller must hold the device mutex. */
+void vhost_clear_device_iotlb(struct vhost_dev *d)
+{
+      struct vhost_iotlb *iotlb;
+      int i;
+
+      iotlb = d->iotlb;
+      if (!iotlb)
+              return;
+
+      /*
+       * Drop the device-wide view first.  Each VQ then drops its
+       * per-VQ view and its cached ring access under its own mutex.
+       * Keep the old table alive until every VQ has completed this
+       * handoff, since a worker may still be using it while waiting
+       * for its VQ mutex.
+       */
+      d->iotlb = NULL;
+
+      for (i = 0; i < d->nvqs; ++i) {
+              struct vhost_virtqueue *vq = d->vqs[i];
+
+              mutex_lock(&vq->mutex);
+              vq->iotlb = NULL;
+              vhost_vq_invalidate_access(vq);
+              mutex_unlock(&vq->mutex);
+      }
+
+      vhost_clear_msg(d);
+      vhost_iotlb_free(iotlb);
+      wake_up_interruptible_poll(&d->wait, EPOLLIN | EPOLLRDNORM);
+}
+EXPORT_SYMBOL_GPL(vhost_clear_device_iotlb);
+
 int vhost_init_device_iotlb(struct vhost_dev *d)
 {
      struct vhost_iotlb *niotlb, *oiotlb;
      int i;
 
+      if (d->iotlb)
+              return 0;
+
      niotlb = iotlb_alloc();
      if (!niotlb)
              return -ENOMEM;
@@ -2287,7 +2338,7 @@ int vhost_init_device_iotlb(struct vhost_dev *d)
 
              mutex_lock(&vq->mutex);
              vq->iotlb = niotlb;
-              __vhost_vq_meta_reset(vq);
+              vhost_vq_invalidate_access(vq);
              mutex_unlock(&vq->mutex);
      }
 
drivers/vhost/vhost.h +1 -0
index 0192ade6e749..3c75e8089373 100644
@@ -277,6 +277,7 @@ ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to,
                          int noblock);
 ssize_t vhost_chr_write_iter(struct vhost_dev *dev,
                           struct iov_iter *from);
+void vhost_clear_device_iotlb(struct vhost_dev *d);
 int vhost_init_device_iotlb(struct vhost_dev *d);
 
 void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,