Skip to contentSkip to conversation

virtio_pci: fix virtio spec compliance on restore

RFC PATCH In review Michael S. Tsirkin 4 participants 7 messages Sep 23, 2014 – Sep 24, 2014

Subsystems (2)
lore
Share & tools
lore

Michael S. Tsirkin

author maintainer
#0 lore raw

RFC PATCH 1/1 virtio_pci: fix virtio spec compliance on restore

On restore, virtio pci does the following:
+ set features
+ init vqs etc - device can be used at this point!
+ set ACKNOWLEDGE,DRIVER and DRIVER_OK status bits

This is in violation of the virtio spec, which
requires the following order:
- ACKNOWLEDGE
- DRIVER
- init vqs
- DRIVER_OK

---

Lightly tested.
Will repost as non-RFC once testing is done, sending out now for early flames/comments.

Thanks!

 drivers/virtio/virtio_pci.c | 36 ++++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)
drivers/virtio/virtio_pci.c +32 -4
index 58f7e45..58cbf6e 100644
@@ -785,6 +785,7 @@ static int virtio_pci_restore(struct device *dev)
 	struct pci_dev *pci_dev = to_pci_dev(dev);
 	struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 	struct virtio_driver *drv;
+	unsigned status = 0;
 	int ret;
 
 	drv = container_of(vp_dev->vdev.dev.driver,
@@ -795,14 +796,41 @@ static int virtio_pci_restore(struct device *dev)
 		return ret;
 
 	pci_set_master(pci_dev);
+	/* We always start by resetting the device, in case a previous
+	 * driver messed it up. */
+	vp_reset(&vp_dev->vdev);
+
+	/* Acknowledge that we've seen the device. */
+	status |= VIRTIO_CONFIG_S_ACKNOWLEDGE;
+	vp_set_status(&vp_dev->vdev, status);
+
+	/* Maybe driver failed before freeze.
+	 * Restore the failed status, for debugging. */
+	status |= vp_dev->saved_status & VIRTIO_CONFIG_S_FAILED;
+	vp_set_status(&vp_dev->vdev, status);
+
+	if (!drv)
+		return 0;
+
+	/* We have a driver! */
+	status |= VIRTIO_CONFIG_S_DRIVER;
+	vp_set_status(&vp_dev->vdev, status);
+
 	vp_finalize_features(&vp_dev->vdev);
 
-	if (drv && drv->restore)
-		ret = drv->restore(&vp_dev->vdev);
+	if (!drv->restore)
+		return 0;
+
+	ret = drv->restore(&vp_dev->vdev);
+	if (ret) {
+		status |= VIRTIO_CONFIG_S_FAILED;
+		vp_set_status(&vp_dev->vdev, status);
+		return ret;
+	}
 
 	/* Finally, tell the device we're all set */
-	if (!ret)
-		vp_set_status(&vp_dev->vdev, vp_dev->saved_status);
+	status |= VIRTIO_CONFIG_S_DRIVER_OK;
+	vp_set_status(&vp_dev->vdev, status);
 
 	return ret;
 }
Signature · 1 line
MST
#1 lore raw
Quoted 39 lines · Michael S. Tsirkin

On restore, virtio pci does the following:
+ set features
+ init vqs etc - device can be used at this point!
+ set ACKNOWLEDGE,DRIVER and DRIVER_OK status bits

This is in violation of the virtio spec, which
requires the following order:
- ACKNOWLEDGE
- DRIVER
- init vqs
- DRIVER_OK

---

Lightly tested.
Will repost as non-RFC once testing is done, sending out now for early flames/comments.

Thanks!

 drivers/virtio/virtio_pci.c | 36 ++++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)
drivers/virtio/virtio_pci.c +1 -0
index 58f7e45..58cbf6e 100644
@@ -785,6 +785,7 @@ static int virtio_pci_restore(struct device *dev)
        struct pci_dev *pci_dev = to_pci_dev(dev);
        struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
        struct virtio_driver *drv;
+       unsigned status = 0;
        int ret;
 
        drv = container_of(vp_dev->vdev.dev.driver,

drivers/virtio/virtio_pci.c static int virtio_pci_restore(struct device *dev) · line 801

Quoted 6 lines
drivers/virtio/virtio_pci.c +3 -0
@@ -795,14 +796,41 @@ static int virtio_pci_restore(struct device *dev)
                return ret;
 
        pci_set_master(pci_dev);
+       /* We always start by resetting the device, in case a previous
+        * driver messed it up. */
+       vp_reset(&vp_dev->vdev);

This should happen before enabling PCI bus mastering.

Quoted 45 lines
drivers/virtio/virtio_pci.c +28 -5
+
+       /* Acknowledge that we've seen the device. */
+       status |= VIRTIO_CONFIG_S_ACKNOWLEDGE;
+       vp_set_status(&vp_dev->vdev, status);
+
+       /* Maybe driver failed before freeze.
+        * Restore the failed status, for debugging. */
+       status |= vp_dev->saved_status & VIRTIO_CONFIG_S_FAILED;
+       vp_set_status(&vp_dev->vdev, status);
+
+       if (!drv)
+               return 0;
+
+       /* We have a driver! */
+       status |= VIRTIO_CONFIG_S_DRIVER;
+       vp_set_status(&vp_dev->vdev, status);
+
        vp_finalize_features(&vp_dev->vdev);
 
-       if (drv && drv->restore)
-               ret = drv->restore(&vp_dev->vdev);
+       if (!drv->restore)
+               return 0;
+
+       ret = drv->restore(&vp_dev->vdev);
+       if (ret) {
+               status |= VIRTIO_CONFIG_S_FAILED;
+               vp_set_status(&vp_dev->vdev, status);
+               return ret;
+       }
 
        /* Finally, tell the device we're all set */
-       if (!ret)
-               vp_set_status(&vp_dev->vdev, vp_dev->saved_status);
+       status |= VIRTIO_CONFIG_S_DRIVER_OK;
+       vp_set_status(&vp_dev->vdev, status);
 
        return ret;
 }
--

MST
_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

author maintainer
#2 lore raw
Quoted 42 lines · Eric Northup
Quoted 41 lines · Michael S. Tsirkin

On restore, virtio pci does the following:
+ set features
+ init vqs etc - device can be used at this point!
+ set ACKNOWLEDGE,DRIVER and DRIVER_OK status bits

This is in violation of the virtio spec, which
requires the following order:
- ACKNOWLEDGE
- DRIVER
- init vqs
- DRIVER_OK

---

Lightly tested.
Will repost as non-RFC once testing is done, sending out now for early flames/comments.

Thanks!

 drivers/virtio/virtio_pci.c | 36 ++++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)
drivers/virtio/virtio_pci.c +1 -0
index 58f7e45..58cbf6e 100644
@@ -785,6 +785,7 @@ static int virtio_pci_restore(struct device *dev)
        struct pci_dev *pci_dev = to_pci_dev(dev);
        struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
        struct virtio_driver *drv;
+       unsigned status = 0;
        int ret;
 
        drv = container_of(vp_dev->vdev.dev.driver,
@@ -795,14 +796,41 @@ static int virtio_pci_restore(struct device *dev)
                return ret;
 
Quoted 6 lines

drivers/virtio/virtio_pci.c static int virtio_pci_restore(struct device *dev) · line 801

Quoted 4 lines
drivers/virtio/virtio_pci.c +3 -0
        pci_set_master(pci_dev);
+       /* We always start by resetting the device, in case a previous
+        * driver messed it up. */
+       vp_reset(&vp_dev->vdev);

This should happen before enabling PCI bus mastering.

this is the order of events in initialization,
it seems better to be consistent.

Quoted 45 lines
Quoted 45 lines
drivers/virtio/virtio_pci.c +28 -5
+
+       /* Acknowledge that we've seen the device. */
+       status |= VIRTIO_CONFIG_S_ACKNOWLEDGE;
+       vp_set_status(&vp_dev->vdev, status);
+
+       /* Maybe driver failed before freeze.
+        * Restore the failed status, for debugging. */
+       status |= vp_dev->saved_status & VIRTIO_CONFIG_S_FAILED;
+       vp_set_status(&vp_dev->vdev, status);
+
+       if (!drv)
+               return 0;
+
+       /* We have a driver! */
+       status |= VIRTIO_CONFIG_S_DRIVER;
+       vp_set_status(&vp_dev->vdev, status);
+
        vp_finalize_features(&vp_dev->vdev);
 
-       if (drv && drv->restore)
-               ret = drv->restore(&vp_dev->vdev);
+       if (!drv->restore)
+               return 0;
+
+       ret = drv->restore(&vp_dev->vdev);
+       if (ret) {
+               status |= VIRTIO_CONFIG_S_FAILED;
+               vp_set_status(&vp_dev->vdev, status);
+               return ret;
+       }
 
        /* Finally, tell the device we're all set */
-       if (!ret)
-               vp_set_status(&vp_dev->vdev, vp_dev->saved_status);
+       status |= VIRTIO_CONFIG_S_DRIVER_OK;
+       vp_set_status(&vp_dev->vdev, status);
 
        return ret;
 }
--

MST
_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#3 lore raw
In reply to Michael S. Tsirkin
Quoted 12 lines · Michael S. Tsirkin

On restore, virtio pci does the following:
+ set features
+ init vqs etc - device can be used at this point!
+ set ACKNOWLEDGE,DRIVER and DRIVER_OK status bits

This is in violation of the virtio spec, which
requires the following order:
- ACKNOWLEDGE
- DRIVER
- init vqs
- DRIVER_OK

Quoted 3 lines

[...]

What concrete problem does this fix, such that it should be applied to stable branches?

Ben.

Signature · 3 lines
Ben Hutchings
Everything should be made as simple as possible, but not simpler.
                                                           - Albert Einstein
author maintainer
#4 lore raw
Quoted 16 lines · Ben Hutchings
Quoted 12 lines · Michael S. Tsirkin

On restore, virtio pci does the following:
+ set features
+ init vqs etc - device can be used at this point!
+ set ACKNOWLEDGE,DRIVER and DRIVER_OK status bits

This is in violation of the virtio spec, which
requires the following order:
- ACKNOWLEDGE
- DRIVER
- init vqs
- DRIVER_OK

Quoted 3 lines
Quoted 6 lines

[...]

What concrete problem does this fix, such that it should be applied to stable branches?

Ben.

It will break with hypervisors that assume spec compliant behaviour. I would like this applied to stable branches so hypervisors don't need to support broken behaviour forever.

Signature · 1 line
MST
#5 lore raw
In reply to Michael S. Tsirkin
Quoted 12 lines · Michael S. Tsirkin

On restore, virtio pci does the following:
+ set features
+ init vqs etc - device can be used at this point!
+ set ACKNOWLEDGE,DRIVER and DRIVER_OK status bits

This is in violation of the virtio spec, which
requires the following order:
- ACKNOWLEDGE
- DRIVER
- init vqs
- DRIVER_OK

Quoted 3 lines

[...]

What concrete problem does this fix, such that it should be applied to stable branches?

Ben.

Signature · 3 lines
Ben Hutchings
Everything should be made as simple as possible, but not simpler.
                                                           - Albert Einstein

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#6 lore raw
In reply to Michael S. Tsirkin
Quoted 12 lines · Michael S. Tsirkin

On restore, virtio pci does the following:
+ set features
+ init vqs etc - device can be used at this point!
+ set ACKNOWLEDGE,DRIVER and DRIVER_OK status bits

This is in violation of the virtio spec, which
requires the following order:
- ACKNOWLEDGE
- DRIVER
- init vqs
- DRIVER_OK

Quoted 8 lines

---

Lightly tested.
Will repost as non-RFC once testing is done, sending out now for early flames/comments.

What tests are you running?

Quoted 37 lines

Thanks!

 drivers/virtio/virtio_pci.c | 36 ++++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)
drivers/virtio/virtio_pci.c +17 -0
index 58f7e45..58cbf6e 100644
@@ -785,6 +785,7 @@ static int virtio_pci_restore(struct device *dev)
 	struct pci_dev *pci_dev = to_pci_dev(dev);
 	struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 	struct virtio_driver *drv;
+	unsigned status = 0;
 	int ret;
 
 	drv = container_of(vp_dev->vdev.dev.driver,
@@ -795,14 +796,41 @@ static int virtio_pci_restore(struct device *dev)
 		return ret;
 
 	pci_set_master(pci_dev);
+	/* We always start by resetting the device, in case a previous
+	 * driver messed it up. */
+	vp_reset(&vp_dev->vdev);
+
+	/* Acknowledge that we've seen the device. */
+	status |= VIRTIO_CONFIG_S_ACKNOWLEDGE;
+	vp_set_status(&vp_dev->vdev, status);
+
+	/* Maybe driver failed before freeze.
+	 * Restore the failed status, for debugging. */
+	status |= vp_dev->saved_status & VIRTIO_CONFIG_S_FAILED;
+	vp_set_status(&vp_dev->vdev, status);
+
+	if (!drv)
+		return 0;
+

drivers/virtio/virtio_pci.c static int virtio_pci_restore(struct device *dev) · line 822

Quoted 10 lines
drivers/virtio/virtio_pci.c +6 -2
+	/* We have a driver! */
+	status |= VIRTIO_CONFIG_S_DRIVER;
+	vp_set_status(&vp_dev->vdev, status);
+
 	vp_finalize_features(&vp_dev->vdev);
 
-	if (drv && drv->restore)
-		ret = drv->restore(&vp_dev->vdev);
+	if (!drv->restore)
+		return 0;

So in this case DRIVER_OK will never be set?

Quoted 6 lines
drivers/virtio/virtio_pci.c +6 -0
+
+	ret = drv->restore(&vp_dev->vdev);
+	if (ret) {
+		status |= VIRTIO_CONFIG_S_FAILED;
+		vp_set_status(&vp_dev->vdev, status);
+		return ret;

drivers/virtio/virtio_pci.c static int virtio_pci_restore(struct device *dev) · line 836

Quoted 10 lines
drivers/virtio/virtio_pci.c +3 -2
+	}
 
 	/* Finally, tell the device we're all set */
-	if (!ret)
-		vp_set_status(&vp_dev->vdev, vp_dev->saved_status);
+	status |= VIRTIO_CONFIG_S_DRIVER_OK;
+	vp_set_status(&vp_dev->vdev, status);
 
 	return ret;
 }

Amit