Skip to contentSkip to conversation

vhost_vdpa: no need to fetch vring base when poweroff

PATCH In review Zhu Lingshan 3 participants 10 messages Jul 10, 2023 – Jul 12, 2023

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 vhost_vdpa: no need to fetch vring base when poweroff

In the poweroff routine, no need to fetch last available index.

This commit also provides a better debug message in the vhost caller vhost_virtqueue_stop, because if vhost does not fetch the last avail idx successfully, maybe the device does not suspend, vhost will sync last avail idx to vring used idx as a work around, not a failure.

---
 hw/virtio/vhost-vdpa.c | 10 ++++++++++
 hw/virtio/vhost.c      |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)
hw/virtio/vhost-vdpa.c +10 -0
index 3c575a9a6e..10b445f64e 100644
@@ -26,6 +26,7 @@
 #include "cpu.h"
 #include "trace.h"
 #include "qapi/error.h"
+#include "sysemu/runstate.h"
 
 /*
  * Return one past the end of the end of section. Be careful with uint64_t
@@ -1391,6 +1392,15 @@ static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
     struct vhost_vdpa *v = dev->opaque;
     int ret;
 
+    if (runstate_check(RUN_STATE_SHUTDOWN)) {
+        /*
+         * Some devices do not support this call properly,
+         * and we don't need to retrieve the indexes
+         * if it is shutting down
+         */
+        return 0;
+    }
+
     if (v->shadow_vqs_enabled) {
         ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index);
         return 0;
hw/virtio/vhost.c +1 -1
index 82394331bf..7dd90cff3a 100644
@@ -1262,7 +1262,7 @@ void vhost_virtqueue_stop(struct vhost_dev *dev,
 
     r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
     if (r < 0) {
-        VHOST_OPS_DEBUG(r, "vhost VQ %u ring restore failed: %d", idx, r);
+        VHOST_OPS_DEBUG(r, "sync last avail idx to the guest used idx for vhost VQ %u", idx);
         /* Connection to the backend is broken, so let's sync internal
          * last avail idx to the device used idx.
          */
Signature · 1 line
2.39.3
Acked-by on v2 1/1
#1 lore raw
Quoted 10 lines · Zhu Lingshan

In the poweroff routine, no need to fetch last available index.

This commit also provides a better debug message in the vhost caller vhost_virtqueue_stop, because if vhost does not fetch the last avail idx successfully, maybe the device does not suspend, vhost will sync last avail idx to vring used idx as a work around, not a failure.

Thanks!

Quoted 49 lines
---
 hw/virtio/vhost-vdpa.c | 10 ++++++++++
 hw/virtio/vhost.c      |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)
hw/virtio/vhost-vdpa.c +10 -0
index 3c575a9a6e..10b445f64e 100644
@@ -26,6 +26,7 @@
 #include "cpu.h"
 #include "trace.h"
 #include "qapi/error.h"
+#include "sysemu/runstate.h"
 
 /*
  * Return one past the end of the end of section. Be careful with uint64_t
@@ -1391,6 +1392,15 @@ static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
     struct vhost_vdpa *v = dev->opaque;
     int ret;
 
+    if (runstate_check(RUN_STATE_SHUTDOWN)) {
+        /*
+         * Some devices do not support this call properly,
+         * and we don't need to retrieve the indexes
+         * if it is shutting down
+         */
+        return 0;
+    }
+
     if (v->shadow_vqs_enabled) {
         ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index);
         return 0;
hw/virtio/vhost.c +1 -2
index 82394331bf..7dd90cff3a 100644
@@ -1262,7 +1262,7 @@ void vhost_virtqueue_stop(struct vhost_dev *dev,
 
     r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
     if (r < 0) {
-        VHOST_OPS_DEBUG(r, "vhost VQ %u ring restore failed: %d", idx, r);
+        VHOST_OPS_DEBUG(r, "sync last avail idx to the guest used idx for vhost VQ %u", idx);
         /* Connection to the backend is broken, so let's sync internal
          * last avail idx to the device used idx.
          */
--

2.39.3

#2 lore raw
In reply to Zhu Lingshan
Quoted 3 lines · Zhu Lingshan

In the poweroff routine, no need to fetch last available index.

This is because there's no concept of shutdown in the vhost layer, it only knows start and stop.

Quoted 2 lines

This commit also provides a better debug message in the vhost caller vhost_virtqueue_stop,

A separate patch is better.

Quoted 4 lines

because if vhost does not fetch
the last avail idx successfully, maybe the device does not suspend, vhost will sync last avail idx to vring used idx as a work around, not a failure.

This only happens if we return a negative value?

Quoted 20 lines
---
 hw/virtio/vhost-vdpa.c | 10 ++++++++++
 hw/virtio/vhost.c      |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)
hw/virtio/vhost-vdpa.c +1 -0
index 3c575a9a6e..10b445f64e 100644
@@ -26,6 +26,7 @@
 #include "cpu.h"
 #include "trace.h"
 #include "qapi/error.h"
+#include "sysemu/runstate.h"
 
 /*
  * Return one past the end of the end of section. Be careful with uint64_t

hw/virtio/vhost-vdpa.c static int vhost_vdpa_get_vring_base(struct vhost_dev *dev, · line 1401

Quoted 10 lines
hw/virtio/vhost-vdpa.c +7 -0
@@ -1391,6 +1392,15 @@ static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
     struct vhost_vdpa *v = dev->opaque;
     int ret;
 
+    if (runstate_check(RUN_STATE_SHUTDOWN)) {
+        /*
+         * Some devices do not support this call properly,
+         * and we don't need to retrieve the indexes
+         * if it is shutting down
+         */
+        return 0;

Checking runstate in the vhost code seems like a layer violation.

What happens without this patch?

Thanks

Quoted 21 lines
hw/virtio/vhost-vdpa.c +2 -0
+    }
+
     if (v->shadow_vqs_enabled) {
         ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index);
         return 0;
hw/virtio/vhost.c +1 -2
index 82394331bf..7dd90cff3a 100644
@@ -1262,7 +1262,7 @@ void vhost_virtqueue_stop(struct vhost_dev *dev,
 
     r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
     if (r < 0) {
-        VHOST_OPS_DEBUG(r, "vhost VQ %u ring restore failed: %d", idx, r);
+        VHOST_OPS_DEBUG(r, "sync last avail idx to the guest used idx for vhost VQ %u", idx);
         /* Connection to the backend is broken, so let's sync internal
          * last avail idx to the device used idx.
          */
--

2.39.3

#3 lore raw
Quoted 9 lines · Jason Wang
Quoted 2 lines · Zhu Lingshan

In the poweroff routine, no need to fetch last available index.

This is because there's no concept of shutdown in the vhost layer, it only knows start and stop.

Quoted 2 lines

This commit also provides a better debug message in the vhost caller vhost_virtqueue_stop,

A separate patch is better.

OK

Quoted 6 lines
Quoted 5 lines

because if vhost does not fetch
the last avail idx successfully, maybe the device does not suspend, vhost will sync last avail idx to vring used idx as a work around, not a failure.

This only happens if we return a negative value?

Yes

Quoted 32 lines
Quoted 20 lines
---
  hw/virtio/vhost-vdpa.c | 10 ++++++++++
  hw/virtio/vhost.c      |  2 +-
  2 files changed, 11 insertions(+), 1 deletion(-)
hw/virtio/vhost-vdpa.c +1 -0
index 3c575a9a6e..10b445f64e 100644
@@ -26,6 +26,7 @@
  #include "cpu.h"
  #include "trace.h"
  #include "qapi/error.h"
+#include "sysemu/runstate.h"
 
  /*
   * Return one past the end of the end of section. Be careful with uint64_t

hw/virtio/vhost-vdpa.c static int vhost_vdpa_get_vring_base(struct vhost_dev *dev, · line 1401

Quoted 10 lines
hw/virtio/vhost-vdpa.c +7 -0
@@ -1391,6 +1392,15 @@ static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
      struct vhost_vdpa *v = dev->opaque;
      int ret;
 
+    if (runstate_check(RUN_STATE_SHUTDOWN)) {
+        /*
+         * Some devices do not support this call properly,
+         * and we don't need to retrieve the indexes
+         * if it is shutting down
+         */
+        return 0;

Checking runstate in the vhost code seems like a layer violation.

Quoted 1 line

What happens without this patch?

vhost tries to fetch vring base,
vhost_vdpa needs suspend the device before retrieving last_avail_idx. However not all devices can support .suspend properly so this call may fail. Then vhost will print an error shows something failed.

The error msg is confused, as stated in the commit log, restoring last_avail_idx with guest used idx is a workaround rather than a failure. And no needs to fetch last_avail_idx
when power off.

Thanks

Quoted 24 lines

Thanks

Quoted 21 lines
hw/virtio/vhost-vdpa.c +2 -0
+    }
+
      if (v->shadow_vqs_enabled) {
          ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index);
          return 0;
hw/virtio/vhost.c +1 -2
index 82394331bf..7dd90cff3a 100644
@@ -1262,7 +1262,7 @@ void vhost_virtqueue_stop(struct vhost_dev *dev,
 
      r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
      if (r < 0) {
-        VHOST_OPS_DEBUG(r, "vhost VQ %u ring restore failed: %d", idx, r);
+        VHOST_OPS_DEBUG(r, "sync last avail idx to the guest used idx for vhost VQ %u", idx);
          /* Connection to the backend is broken, so let's sync internal
           * last avail idx to the device used idx.
           */
--

2.39.3

#4 lore raw
Quoted 53 lines · Lingshan
Quoted 9 lines · Jason Wang
Quoted 2 lines · Zhu Lingshan

In the poweroff routine, no need to fetch last available index.

This is because there's no concept of shutdown in the vhost layer, it only knows start and stop.

Quoted 2 lines

This commit also provides a better debug message in the vhost caller vhost_virtqueue_stop,

A separate patch is better.

OK

Quoted 6 lines
Quoted 5 lines

because if vhost does not fetch
the last avail idx successfully, maybe the device does not suspend, vhost will sync last avail idx to vring used idx as a work around, not a failure.

This only happens if we return a negative value?

Yes

Quoted 32 lines
Quoted 20 lines
---
  hw/virtio/vhost-vdpa.c | 10 ++++++++++
  hw/virtio/vhost.c      |  2 +-
  2 files changed, 11 insertions(+), 1 deletion(-)
hw/virtio/vhost-vdpa.c +1 -0
index 3c575a9a6e..10b445f64e 100644
@@ -26,6 +26,7 @@
  #include "cpu.h"
  #include "trace.h"
  #include "qapi/error.h"
+#include "sysemu/runstate.h"
 
  /*
   * Return one past the end of the end of section. Be careful with uint64_t

hw/virtio/vhost-vdpa.c static int vhost_vdpa_get_vring_base(struct vhost_dev *dev, · line 1401

Quoted 10 lines
hw/virtio/vhost-vdpa.c +7 -0
@@ -1391,6 +1392,15 @@ static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
      struct vhost_vdpa *v = dev->opaque;
      int ret;
 
+    if (runstate_check(RUN_STATE_SHUTDOWN)) {
+        /*
+         * Some devices do not support this call properly,
+         * and we don't need to retrieve the indexes
+         * if it is shutting down
+         */
+        return 0;

Checking runstate in the vhost code seems like a layer violation.

Quoted 5 lines
Quoted 1 line

What happens without this patch?

vhost tries to fetch vring base,
vhost_vdpa needs suspend the device before retrieving last_avail_idx. However not all devices can support .suspend properly so this call may fail.

I think this is where I'm lost. If the device doesn't support suspending, any reason we only try to fix the case of shutdown?

Btw, the fail is intended:

    if (!v->suspended) {
        /*
         * Cannot trust in value returned by device, let vhost recover used
         * idx from guest.
         */
        return -1;
    }

And if we return to success here, will we go to set an uninitialized last avail idx?

    r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
    if (r < 0) {
    ...
    }.else {
        virtio_queue_set_last_avail_idx(vdev, idx, state.num);
    }

Thanks

Quoted 33 lines

Then vhost will print an error shows something failed.

The error msg is confused, as stated in the commit log, restoring last_avail_idx with guest used idx
is a workaround rather than a failure. And no needs to fetch last_avail_idx when power off.

Thanks

Quoted 24 lines

Thanks

Quoted 21 lines
hw/virtio/vhost-vdpa.c +2 -0
+    }
+
      if (v->shadow_vqs_enabled) {
          ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index);
          return 0;
hw/virtio/vhost.c +1 -2
index 82394331bf..7dd90cff3a 100644
@@ -1262,7 +1262,7 @@ void vhost_virtqueue_stop(struct vhost_dev *dev,
 
      r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
      if (r < 0) {
-        VHOST_OPS_DEBUG(r, "vhost VQ %u ring restore failed: %d", idx, r);
+        VHOST_OPS_DEBUG(r, "sync last avail idx to the guest used idx for vhost VQ %u", idx);
          /* Connection to the backend is broken, so let's sync internal
           * last avail idx to the device used idx.
           */
--

2.39.3

#5 lore raw
Quoted 66 lines · Jason Wang
Quoted 53 lines · Lingshan
Quoted 9 lines · Jason Wang
Quoted 2 lines · Zhu Lingshan

In the poweroff routine, no need to fetch last available index.

This is because there's no concept of shutdown in the vhost layer, it only knows start and stop.

Quoted 2 lines

This commit also provides a better debug message in the vhost caller vhost_virtqueue_stop,

A separate patch is better.

OK

Quoted 6 lines
Quoted 5 lines

because if vhost does not fetch
the last avail idx successfully, maybe the device does not suspend, vhost will sync last avail idx to vring used idx as a work around, not a failure.

This only happens if we return a negative value?

Yes

Quoted 32 lines
Quoted 20 lines
---
  hw/virtio/vhost-vdpa.c | 10 ++++++++++
  hw/virtio/vhost.c      |  2 +-
  2 files changed, 11 insertions(+), 1 deletion(-)
hw/virtio/vhost-vdpa.c +1 -0
index 3c575a9a6e..10b445f64e 100644
@@ -26,6 +26,7 @@
  #include "cpu.h"
  #include "trace.h"
  #include "qapi/error.h"
+#include "sysemu/runstate.h"
 
  /*
   * Return one past the end of the end of section. Be careful with uint64_t

hw/virtio/vhost-vdpa.c static int vhost_vdpa_get_vring_base(struct vhost_dev *dev, · line 1401

Quoted 10 lines
hw/virtio/vhost-vdpa.c +7 -0
@@ -1391,6 +1392,15 @@ static int vhost_vdpa_get_vring_base(struct vhost_dev *dev,
      struct vhost_vdpa *v = dev->opaque;
      int ret;
 
+    if (runstate_check(RUN_STATE_SHUTDOWN)) {
+        /*
+         * Some devices do not support this call properly,
+         * and we don't need to retrieve the indexes
+         * if it is shutting down
+         */
+        return 0;

Checking runstate in the vhost code seems like a layer violation.

Quoted 5 lines
Quoted 1 line

What happens without this patch?

vhost tries to fetch vring base,
vhost_vdpa needs suspend the device before retrieving last_avail_idx. However not all devices can support .suspend properly so this call may fail.

I think this is where I'm lost. If the device doesn't support suspending, any reason we only try to fix the case of shutdown?

Btw, the fail is intended:

Quoted 8 lines
    if (!v->suspended) {
        /*
         * Cannot trust in value returned by device, let vhost recover used
         * idx from guest.
         */
        return -1;
    }

The fail is intended, but to recover the last used idx, either from device or from the guest, is only useful in the case of migration.

I think the main problem is the error message, actually. But I think it has no use either to recover last_avail_idx or print a debug message if we're not migrating. Another solution would be to recover it from the guest at vhost_vdpa_get_vring_base, but I don't like the duplication.

Quoted 3 lines

And if we return to success here, will we go to set an uninitialized last avail idx?

It will be either the default value (is set to 0 at __virtio_queue_reset) or the one received from a migration (at virtio_load).

Thanks!

Quoted 43 lines
    r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
    if (r < 0) {
    ...
    }.else {
        virtio_queue_set_last_avail_idx(vdev, idx, state.num);
    }

Thanks

Quoted 33 lines

Then vhost will print an error shows something failed.

The error msg is confused, as stated in the commit log, restoring last_avail_idx with guest used idx
is a workaround rather than a failure. And no needs to fetch last_avail_idx when power off.

Thanks

Quoted 24 lines

Thanks

Quoted 21 lines
hw/virtio/vhost-vdpa.c +2 -0
+    }
+
      if (v->shadow_vqs_enabled) {
          ring->num = virtio_queue_get_last_avail_idx(dev->vdev, ring->index);
          return 0;
hw/virtio/vhost.c +1 -2
index 82394331bf..7dd90cff3a 100644
@@ -1262,7 +1262,7 @@ void vhost_virtqueue_stop(struct vhost_dev *dev,
 
      r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
      if (r < 0) {
-        VHOST_OPS_DEBUG(r, "vhost VQ %u ring restore failed: %d", idx, r);
+        VHOST_OPS_DEBUG(r, "sync last avail idx to the guest used idx for vhost VQ %u", idx);
          /* Connection to the backend is broken, so let's sync internal
           * last avail idx to the device used idx.
           */
--

2.39.3

#6 lore raw
Quoted 75 lines · Eugenio Perez Martin
Quoted 2 lines · Jason Wang

On Tue, Jul 11, 2023 at 12:09 PM Zhu, Lingshan <[email protected]>

Quoted 5 lines · wrote:
Quoted 5 lines · Jason Wang
Quoted 1 line

On Mon, Jul 10, 2023 at 4:53 PM Zhu Lingshan <[email protected]>

Quoted 33 lines · wrote:
Quoted 33 lines
Quoted 8 lines
Quoted 2 lines

In the poweroff routine, no need to fetch last available index.

This is because there's no concept of shutdown in the vhost layer, it only knows start and stop.

Quoted 2 lines

This commit also provides a better debug message in the vhost caller vhost_virtqueue_stop,

A separate patch is better.

OK

Quoted 6 lines
Quoted 5 lines

because if vhost does not fetch
the last avail idx successfully, maybe the device does not suspend, vhost will sync last avail idx to vring used idx as a work around, not a failure.

This only happens if we return a negative value?

Yes

Quoted 17 lines
Quoted 17 lines
---
  hw/virtio/vhost-vdpa.c | 10 ++++++++++
  hw/virtio/vhost.c      |  2 +-
  2 files changed, 11 insertions(+), 1 deletion(-)
hw/virtio/vhost-vdpa.c +1 -0
index 3c575a9a6e..10b445f64e 100644
@@ -26,6 +26,7 @@
  #include "cpu.h"
  #include "trace.h"
  #include "qapi/error.h"
+#include "sysemu/runstate.h"
 
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

hw/virtio/vhost-vdpa.c line 32

Quoted 2 lines
hw/virtio/vhost-vdpa.c +0 -0
  /*
   * Return one past the end of the end of section. Be careful with

uint64_t

Quoted 1 line
Quoted 1 line
Quoted 1 line

hw/virtio/vhost-vdpa.c static int vhost_vdpa_get_vring_base(struct

Quoted 1 line
hw/virtio/vhost-vdpa.c +0 -0
@@ -1391,6 +1392,15 @@ static int vhost_vdpa_get_vring_base(struct

vhost_dev *dev,

Quoted 18 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
hw/virtio/vhost-vdpa.c +7 -0
      struct vhost_vdpa *v = dev->opaque;
      int ret;
 
+    if (runstate_check(RUN_STATE_SHUTDOWN)) {
+        /*
+         * Some devices do not support this call properly,
+         * and we don't need to retrieve the indexes
+         * if it is shutting down
+         */
+        return 0;
Quoted 7 lines
Quoted 3 lines

Checking runstate in the vhost code seems like a layer violation.

What happens without this patch?

vhost tries to fetch vring base,
vhost_vdpa needs suspend the device before retrieving last_avail_idx. However not all devices can support .suspend properly so this call may fail.

Quoted 8 lines

I think this is where I'm lost. If the device doesn't support suspending, any reason we only try to fix the case of shutdown?

Btw, the fail is intended:

    if (!v->suspended) {
        /*
         * Cannot trust in value returned by device, let vhost recover

used

Quoted 9 lines
Quoted 5 lines
         * idx from guest.
         */
        return -1;
    }

The fail is intended, but to recover the last used idx, either from device or from the guest, is only useful in the case of migration.

Note that we had userspace devices like VDUSE now, so it could be useful in the case of a VDUSE daemon crash or reconnect.

Quoted 6 lines

I think the main problem is the error message, actually. But I think it has no use either to recover last_avail_idx or print a debug message if we're not migrating. Another solution would be to recover it from the guest at vhost_vdpa_get_vring_base, but I don't like the duplication.

Quoted 8 lines
Quoted 3 lines

And if we return to success here, will we go to set an uninitialized last avail idx?

It will be either the default value (is set to 0 at __virtio_queue_reset) or the one received from a migration (at virtio_load).

0 is even sub-optimal than the index used. Anyhow, VHOST_DEBUG should not be enabled for production environments.

Thanks

Quoted 54 lines

Thanks!

Quoted 7 lines
    r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
    if (r < 0) {
    ...
    }.else {
        virtio_queue_set_last_avail_idx(vdev, idx, state.num);
    }
Quoted 7 lines

Thanks

Quoted 5 lines

Then vhost will print an error shows something failed.

The error msg is confused, as stated in the commit log, restoring last_avail_idx with guest used idx
is a workaround rather than a failure. And no needs to fetch

last_avail_idx

Quoted 10 lines
Quoted 10 lines

when power off.

Thanks

Quoted 7 lines

Thanks

Quoted 4 lines
hw/virtio/vhost-vdpa.c +2 -0
+    }
+
      if (v->shadow_vqs_enabled) {
          ring->num = virtio_queue_get_last_avail_idx(dev->vdev,

ring->index);

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

hw/virtio/vhost.c void vhost_virtqueue_stop(struct vhost_dev

Quoted 6 lines
hw/virtio/vhost-vdpa.c +0 -0
          return 0;
hw/virtio/vhost.c +0 -0
index 82394331bf..7dd90cff3a 100644
@@ -1262,7 +1262,7 @@ void vhost_virtqueue_stop(struct vhost_dev

*dev,

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

hw/virtio/vhost.c void vhost_virtqueue_stop(struct vhost_dev *dev,

Quoted 4 lines
hw/virtio/vhost.c +0 -1
      r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
      if (r < 0) {
-        VHOST_OPS_DEBUG(r, "vhost VQ %u ring restore failed: %d",

idx, r);

Quoted 1 line
Quoted 1 line
Quoted 1 line

hw/virtio/vhost.c void vhost_virtqueue_stop(struct vhost_dev · line 1264

Quoted 1 line
hw/virtio/vhost.c +1 -0
+        VHOST_OPS_DEBUG(r, "sync last avail idx to the guest used

idx for vhost VQ %u", idx);

Quoted 1 line
Quoted 1 line
Quoted 1 line

hw/virtio/vhost.c void vhost_virtqueue_stop(struct vhost_dev · line 1265

Quoted 1 line
hw/virtio/vhost.c +0 -0
          /* Connection to the backend is broken, so let's sync

internal

Quoted 7 lines
Quoted 6 lines
Quoted 5 lines
Quoted 5 lines
hw/virtio/vhost.c +0 -1
           * last avail idx to the device used idx.
           */
--

2.39.3

#7 lore raw
Quoted 93 lines · Jason Wang

On Tue, Jul 11, 2023 at 3:25 PM Eugenio Perez Martin <[email protected]> wrote:

    On Tue, Jul 11, 2023 at 9:05 AM Jason Wang <[email protected]>
    wrote:
    >
    > On Tue, Jul 11, 2023 at 12:09 PM Zhu, Lingshan
    <[email protected]> wrote:
    > >
    > >
    > >
    > > On 7/11/2023 10:50 AM, Jason Wang wrote:
    > > > On Mon, Jul 10, 2023 at 4:53 PM Zhu Lingshan
    <[email protected]> wrote:
    > > >> In the poweroff routine, no need to fetch last available index.
    > > >>
    > > > This is because there's no concept of shutdown in the vhost
    layer, it
    > > > only knows start and stop.
    > > >
    > > >> This commit also provides a better debug message in the vhost
    > > >> caller vhost_virtqueue_stop,
    > > > A separate patch is better.
    > > OK
    > > >
    > > >> because if vhost does not fetch
    > > >> the last avail idx successfully, maybe the device does not
    > > >> suspend, vhost will sync last avail idx to vring used idx as a
    > > >> work around, not a failure.
    > > > This only happens if we return a negative value?
    > > Yes
    > > >
    > > >> Signed-off-by: Zhu Lingshan <[email protected]>
    > > >> ---
    > > >>   hw/virtio/vhost-vdpa.c | 10 ++++++++++
    > > >>   hw/virtio/vhost.c      |  2 +-
    > > >>   2 files changed, 11 insertions(+), 1 deletion(-)
    > > >>
    > > >> diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c
    > > >> index 3c575a9a6e..10b445f64e 100644
    > > >> --- a/hw/virtio/vhost-vdpa.c
    > > >> +++ b/hw/virtio/vhost-vdpa.c
    > > >> @@ -26,6 +26,7 @@
    > > >>   #include "cpu.h"
    > > >>   #include "trace.h"
    > > >>   #include "qapi/error.h"
    > > >> +#include "sysemu/runstate.h"
    > > >>
    > > >>   /*
    > > >>    * Return one past the end of the end of section. Be
    careful with uint64_t
    > > >> @@ -1391,6 +1392,15 @@ static int
    vhost_vdpa_get_vring_base(struct vhost_dev *dev,
    > > >>       struct vhost_vdpa *v = dev->opaque;
    > > >>       int ret;
    > > >>
    > > >> +    if (runstate_check(RUN_STATE_SHUTDOWN)) {
    > > >> +        /*
    > > >> +         * Some devices do not support this call properly,
    > > >> +         * and we don't need to retrieve the indexes
    > > >> +         * if it is shutting down
    > > >> +         */
    > > >> +        return 0;
    > > > Checking runstate in the vhost code seems like a layer
    violation.
    > > >
    > > > What happens without this patch?
    > > vhost tries to fetch vring base,
    > > vhost_vdpa needs suspend the device before retrieving
    last_avail_idx.
    > > However not all devices can support .suspend properly so this call
    > > may fail.
    >
    > I think this is where I'm lost. If the device doesn't support
    > suspending, any reason we only try to fix the case of shutdown?
    >
    > Btw, the fail is intended:
    >
    >     if (!v->suspended) {
    >         /*
    >          * Cannot trust in value returned by device, let vhost
    recover used
    >          * idx from guest.
    >          */
    >         return -1;
    >     }
    >

    The fail is intended, but to recover the last used idx, either from
    device or from the guest, is only useful in the case of migration.
Quoted 1 line

Note that we had userspace devices like VDUSE now, so it could be useful in the case of a VDUSE daemon crash or reconnect.

This code blcok is for vhost_vdpa backend, and I think vduse is another code path. Return a guest used idx may be a good idea but as Eugenio pointed out that may duplicate the code.

Quoted 12 lines

I think the main problem is the error message, actually. But I think it has no use either to recover last_avail_idx or print a debug message if we're not migrating. Another solution would be to recover it from the guest at vhost_vdpa_get_vring_base, but I don't like the duplication.

> And if we return to success here, will we go to set an uninitialized > last avail idx?
>

Quoted 6 lines

It will be either the default value (is set to 0 at __virtio_queue_reset) or the one received from a migration (at virtio_load).

0 is even sub-optimal than the index used. Anyhow, VHOST_DEBUG should not be enabled for production environments.

Returning 0 sounds like a queue reset, yes we can reset a queue if failed to suspend it, I am not sure whther 0 is better than guest used idx.

I think we are not able to disable VHOST_DEBUG because customers can build QEMU by their own.

Thanks

Quoted 56 lines

Thanks

    Thanks!

    >     r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
    >     if (r < 0) {
    >     ...
    >     }.else {
    >         virtio_queue_set_last_avail_idx(vdev, idx, state.num);
    >     }
    >
    > Thanks
    >
    > > Then vhost will print an error shows something failed.
    > >
    > > The error msg is confused, as stated in the commit log, restoring
    > > last_avail_idx with guest used idx
    > > is a workaround rather than a failure. And no needs to fetch
    last_avail_idx
    > > when power off.
    > >
    > > Thanks
    > > >
    > > > Thanks
    > > >
    > > >> +    }
    > > >> +
    > > >>       if (v->shadow_vqs_enabled) {
    > > >>           ring->num =
    virtio_queue_get_last_avail_idx(dev->vdev, ring->index);
    > > >>           return 0;
    > > >> diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c
    > > >> index 82394331bf..7dd90cff3a 100644
    > > >> --- a/hw/virtio/vhost.c
    > > >> +++ b/hw/virtio/vhost.c
    > > >> @@ -1262,7 +1262,7 @@ void vhost_virtqueue_stop(struct
    vhost_dev *dev,
    > > >>
    > > >>       r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
    > > >>       if (r < 0) {
    > > >> -        VHOST_OPS_DEBUG(r, "vhost VQ %u ring restore
    failed: %d", idx, r);
    > > >> +        VHOST_OPS_DEBUG(r, "sync last avail idx to the
    guest used idx for vhost VQ %u", idx);
    > > >>           /* Connection to the backend is broken, so let's
    sync internal
    > > >>            * last avail idx to the device used idx.
    > > >>            */
    > > >> --
    > > >> 2.39.3
    > > >>
    > >
    >
#8 lore raw
Quoted 95 lines · Lingshan

On 7/11/2023 3:34 PM, Jason Wang wrote:

Quoted 76 lines · Eugenio Perez Martin
Quoted 2 lines · Jason Wang

On Tue, Jul 11, 2023 at 12:09 PM Zhu, Lingshan <[email protected]>

Quoted 5 lines · wrote:
Quoted 5 lines · Jason Wang
Quoted 1 line

On Mon, Jul 10, 2023 at 4:53 PM Zhu Lingshan <

Quoted 3 lines · [email protected]> wrote:
Quoted 3 lines
Quoted 3 lines
Quoted 2 lines

In the poweroff routine, no need to fetch last available index.

This is because there's no concept of shutdown in the vhost layer,

it

Quoted 30 lines
Quoted 30 lines
Quoted 5 lines

only knows start and stop.

Quoted 2 lines

This commit also provides a better debug message in the vhost caller vhost_virtqueue_stop,

A separate patch is better.

OK

Quoted 6 lines
Quoted 5 lines

because if vhost does not fetch
the last avail idx successfully, maybe the device does not suspend, vhost will sync last avail idx to vring used idx as a work around, not a failure.

This only happens if we return a negative value?

Yes

Quoted 17 lines
Quoted 17 lines
---
  hw/virtio/vhost-vdpa.c | 10 ++++++++++
  hw/virtio/vhost.c      |  2 +-
  2 files changed, 11 insertions(+), 1 deletion(-)
hw/virtio/vhost-vdpa.c +1 -0
index 3c575a9a6e..10b445f64e 100644
@@ -26,6 +26,7 @@
  #include "cpu.h"
  #include "trace.h"
  #include "qapi/error.h"
+#include "sysemu/runstate.h"
 
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

hw/virtio/vhost-vdpa.c line 32

Quoted 2 lines
hw/virtio/vhost-vdpa.c +0 -0
  /*
   * Return one past the end of the end of section. Be careful

with uint64_t

Quoted 1 line
Quoted 1 line
Quoted 1 line

hw/virtio/vhost-vdpa.c static int vhost_vdpa_get_vring_base(struct

Quoted 1 line
hw/virtio/vhost-vdpa.c +0 -0
@@ -1391,6 +1392,15 @@ static int vhost_vdpa_get_vring_base(struct

vhost_dev *dev,

Quoted 18 lines
Quoted 10 lines
Quoted 10 lines
Quoted 10 lines
hw/virtio/vhost-vdpa.c +7 -0
      struct vhost_vdpa *v = dev->opaque;
      int ret;
 
+    if (runstate_check(RUN_STATE_SHUTDOWN)) {
+        /*
+         * Some devices do not support this call properly,
+         * and we don't need to retrieve the indexes
+         * if it is shutting down
+         */
+        return 0;
Quoted 7 lines
Quoted 3 lines

Checking runstate in the vhost code seems like a layer violation.

What happens without this patch?

vhost tries to fetch vring base,
vhost_vdpa needs suspend the device before retrieving last_avail_idx. However not all devices can support .suspend properly so this call may fail.

Quoted 8 lines

I think this is where I'm lost. If the device doesn't support suspending, any reason we only try to fix the case of shutdown?

Btw, the fail is intended:

    if (!v->suspended) {
        /*
         * Cannot trust in value returned by device, let vhost recover

used

Quoted 9 lines
Quoted 5 lines
         * idx from guest.
         */
        return -1;
    }

The fail is intended, but to recover the last used idx, either from device or from the guest, is only useful in the case of migration.

Quoted 6 lines

Note that we had userspace devices like VDUSE now, so it could be useful in the case of a VDUSE daemon crash or reconnect.

This code blcok is for vhost_vdpa backend, and I think vduse is another code path.

I'm not sure I understand here, I meant vhost_vdpa + vduse. It works similar to vhost-user.

Quoted 23 lines

Return a guest used idx may be a good idea but as Eugenio pointed out that may duplicate the code.

Quoted 6 lines

I think the main problem is the error message, actually. But I think it has no use either to recover last_avail_idx or print a debug message if we're not migrating. Another solution would be to recover it from the guest at vhost_vdpa_get_vring_base, but I don't like the duplication.

Quoted 8 lines
Quoted 3 lines

And if we return to success here, will we go to set an uninitialized last avail idx?

It will be either the default value (is set to 0 at __virtio_queue_reset) or the one received from a migration (at virtio_load).

0 is even sub-optimal than the index used. Anyhow, VHOST_DEBUG should not be enabled for production environments.

Quoted 7 lines

Returning 0 sounds like a queue reset, yes we can reset a queue if failed to suspend it, I am not sure whther
0 is better than guest used idx.

I think we are not able to disable VHOST_DEBUG because customers can build QEMU by their own.

Well, disabling debug information is a common practice in any distribution.

Or if you worry about the default, let's have a patch to undef VHOST_DEBUG by defualt.

Thanks

Quoted 62 lines

Thanks

Thanks

Quoted 54 lines

Thanks!

Quoted 7 lines
    r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
    if (r < 0) {
    ...
    }.else {
        virtio_queue_set_last_avail_idx(vdev, idx, state.num);
    }
Quoted 7 lines

Thanks

Quoted 5 lines

Then vhost will print an error shows something failed.

The error msg is confused, as stated in the commit log, restoring last_avail_idx with guest used idx
is a workaround rather than a failure. And no needs to fetch

last_avail_idx

Quoted 10 lines
Quoted 10 lines

when power off.

Thanks

Quoted 7 lines

Thanks

Quoted 4 lines
hw/virtio/vhost-vdpa.c +2 -0
+    }
+
      if (v->shadow_vqs_enabled) {
          ring->num = virtio_queue_get_last_avail_idx(dev->vdev,

ring->index);

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

hw/virtio/vhost.c void vhost_virtqueue_stop(struct vhost_dev

Quoted 6 lines
hw/virtio/vhost-vdpa.c +0 -0
          return 0;
hw/virtio/vhost.c +0 -0
index 82394331bf..7dd90cff3a 100644
@@ -1262,7 +1262,7 @@ void vhost_virtqueue_stop(struct vhost_dev

*dev,

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

hw/virtio/vhost.c void vhost_virtqueue_stop(struct vhost_dev · line 1264

Quoted 4 lines
hw/virtio/vhost.c +0 -1
      r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
      if (r < 0) {
-        VHOST_OPS_DEBUG(r, "vhost VQ %u ring restore failed: %d",

idx, r);

Quoted 1 line
Quoted 1 line
Quoted 1 line

hw/virtio/vhost.c void vhost_virtqueue_stop(struct vhost_dev · line 1264

Quoted 1 line
hw/virtio/vhost.c +1 -0
+        VHOST_OPS_DEBUG(r, "sync last avail idx to the guest used

idx for vhost VQ %u", idx);

Quoted 1 line
Quoted 1 line
Quoted 1 line

hw/virtio/vhost.c void vhost_virtqueue_stop(struct vhost_dev · line 1265

Quoted 1 line
hw/virtio/vhost.c +0 -0
          /* Connection to the backend is broken, so let's sync

internal

Quoted 7 lines
Quoted 6 lines
Quoted 5 lines
Quoted 5 lines
hw/virtio/vhost.c +0 -1
           * last avail idx to the device used idx.
           */
--

2.39.3

#9 lore raw
Quoted 109 lines · Jason Wang

On Wed, Jul 12, 2023 at 2:54 PM Zhu, Lingshan <[email protected]> wrote:

    On 7/11/2023 3:34 PM, Jason Wang wrote:
Quoted 102 lines

    On Tue, Jul 11, 2023 at 3:25 PM Eugenio Perez Martin
    <[email protected]> wrote:

        On Tue, Jul 11, 2023 at 9:05 AM Jason Wang
        <[email protected]> wrote:
        >
        > On Tue, Jul 11, 2023 at 12:09 PM Zhu, Lingshan
        <[email protected]> wrote:
        > >
        > >
        > >
        > > On 7/11/2023 10:50 AM, Jason Wang wrote:
        > > > On Mon, Jul 10, 2023 at 4:53 PM Zhu Lingshan
        <[email protected]> wrote:
        > > >> In the poweroff routine, no need to fetch last
        available index.
        > > >>
        > > > This is because there's no concept of shutdown in the
        vhost layer, it
        > > > only knows start and stop.
        > > >
        > > >> This commit also provides a better debug message in
        the vhost
        > > >> caller vhost_virtqueue_stop,
        > > > A separate patch is better.
        > > OK
        > > >
        > > >> because if vhost does not fetch
        > > >> the last avail idx successfully, maybe the device does not
        > > >> suspend, vhost will sync last avail idx to vring used
        idx as a
        > > >> work around, not a failure.
        > > > This only happens if we return a negative value?
        > > Yes
        > > >
        > > >> Signed-off-by: Zhu Lingshan <[email protected]>
        > > >> ---
        > > >>   hw/virtio/vhost-vdpa.c | 10 ++++++++++
        > > >>   hw/virtio/vhost.c      |  2 +-
        > > >>   2 files changed, 11 insertions(+), 1 deletion(-)
        > > >>
        > > >> diff --git a/hw/virtio/vhost-vdpa.c
        b/hw/virtio/vhost-vdpa.c
        > > >> index 3c575a9a6e..10b445f64e 100644
        > > >> --- a/hw/virtio/vhost-vdpa.c
        > > >> +++ b/hw/virtio/vhost-vdpa.c
        > > >> @@ -26,6 +26,7 @@
        > > >>   #include "cpu.h"
        > > >>   #include "trace.h"
        > > >>   #include "qapi/error.h"
        > > >> +#include "sysemu/runstate.h"
        > > >>
        > > >>   /*
        > > >>    * Return one past the end of the end of section. Be
        careful with uint64_t
        > > >> @@ -1391,6 +1392,15 @@ static int
        vhost_vdpa_get_vring_base(struct vhost_dev *dev,
        > > >>       struct vhost_vdpa *v = dev->opaque;
        > > >>       int ret;
        > > >>
        > > >> +    if (runstate_check(RUN_STATE_SHUTDOWN)) {
        > > >> +        /*
        > > >> +         * Some devices do not support this call
        properly,
        > > >> +         * and we don't need to retrieve the indexes
        > > >> +         * if it is shutting down
        > > >> +         */
        > > >> +        return 0;
        > > > Checking runstate in the vhost code seems like a layer
        violation.
        > > >
        > > > What happens without this patch?
        > > vhost tries to fetch vring base,
        > > vhost_vdpa needs suspend the device before retrieving
        last_avail_idx.
        > > However not all devices can support .suspend properly so
        this call
        > > may fail.
        >
        > I think this is where I'm lost. If the device doesn't support
        > suspending, any reason we only try to fix the case of shutdown?
        >
        > Btw, the fail is intended:
        >
        >     if (!v->suspended) {
        >         /*
        >          * Cannot trust in value returned by device, let
        vhost recover used
        >          * idx from guest.
        >          */
        >         return -1;
        >     }
        >

        The fail is intended, but to recover the last used idx,
        either from
        device or from the guest, is only useful in the case of
        migration.
Quoted 7 lines
Quoted 2 lines

Note that we had userspace devices like VDUSE now, so it could be useful in the case of a VDUSE daemon crash or reconnect.

This code blcok is for vhost_vdpa backend, and I think vduse is another code path.

I'm not sure I understand here, I meant vhost_vdpa + vduse. It works similar to vhost-user.

OK, so do you suggest we set vring state == 0 and return 0 if failed to suspend the device? No matter shutdown or other cases.

Quoted 30 lines

Return a guest used idx may be a good idea but as Eugenio pointed out that may duplicate the code.

Quoted 16 lines

I think the main problem is the error message, actually. But I think
it has no use either to recover last_avail_idx or print a debug message if we're not migrating. Another solution would be to recover
it from the guest at vhost_vdpa_get_vring_base, but I don't like the
duplication.

> And if we return to success here, will we go to set an uninitialized
> last avail idx?
>

Quoted 7 lines

It will be either the default value (is set to 0 at __virtio_queue_reset) or the one received from a migration (at virtio_load).

0 is even sub-optimal than the index used. Anyhow, VHOST_DEBUG should not be enabled for production environments.

Returning 0 sounds like a queue reset, yes we can reset a queue if failed to suspend it, I am not sure whther
0 is better than guest used idx.

Quoted 8 lines

I think we are not able to disable VHOST_DEBUG because customers can build QEMU by their own.

Well, disabling debug information is a common practice in any distribution.

Or if you worry about the default, let's have a patch to undef VHOST_DEBUG by defualt.

I can do this in the next version

Thanks

Quoted 63 lines

Thanks

Thanks

Quoted 58 lines
    Thanks


        Thanks!

        >     r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
        >     if (r < 0) {
        >     ...
        >     }.else {
        >  virtio_queue_set_last_avail_idx(vdev, idx, state.num);
        >     }
        >
        > Thanks
        >
        > > Then vhost will print an error shows something failed.
        > >
        > > The error msg is confused, as stated in the commit log,
        restoring
        > > last_avail_idx with guest used idx
        > > is a workaround rather than a failure. And no needs to
        fetch last_avail_idx
        > > when power off.
        > >
        > > Thanks
        > > >
        > > > Thanks
        > > >
        > > >> +    }
        > > >> +
        > > >>       if (v->shadow_vqs_enabled) {
        > > >>           ring->num =
        virtio_queue_get_last_avail_idx(dev->vdev, ring->index);
        > > >>           return 0;
        > > >> diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c
        > > >> index 82394331bf..7dd90cff3a 100644
        > > >> --- a/hw/virtio/vhost.c
        > > >> +++ b/hw/virtio/vhost.c
        > > >> @@ -1262,7 +1262,7 @@ void vhost_virtqueue_stop(struct
        vhost_dev *dev,
        > > >>
        > > >>       r = dev->vhost_ops->vhost_get_vring_base(dev,
        &state);
        > > >>       if (r < 0) {
        > > >> -        VHOST_OPS_DEBUG(r, "vhost VQ %u ring restore
        failed: %d", idx, r);
        > > >> +        VHOST_OPS_DEBUG(r, "sync last avail idx to
        the guest used idx for vhost VQ %u", idx);
        > > >>           /* Connection to the backend is broken, so
        let's sync internal
        > > >>            * last avail idx to the device used idx.
        > > >>            */
        > > >> --
        > > >> 2.39.3
        > > >>
        > >
        >