Skip to contentSkip to conversation

mlx5_vdpa: offer VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK

PATCH In review Eugenio Pérez 4 participants 12 messages Jul 3, 2023 – Oct 4, 2023

Subsystems (2)
lore
Share & tools
lore

Eugenio Pérez

author maintainer
#0 lore raw

PATCH 1/1 mlx5_vdpa: offer VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK

Offer this backend feature as mlx5 is compatible with it. It allows it to do live migration with CVQ, dynamically switching between passthrough and shadow virtqueue.

---
 drivers/vdpa/mlx5/net/mlx5_vnet.c | 7 +++++++
 1 file changed, 7 insertions(+)
drivers/vdpa/mlx5/net/mlx5_vnet.c +7 -0
index 9138ef2fb2c8..5f309a16b9dc 100644
@@ -7,6 +7,7 @@
 #include <uapi/linux/virtio_net.h>
 #include <uapi/linux/virtio_ids.h>
 #include <uapi/linux/vdpa.h>
+#include <uapi/linux/vhost_types.h>
 #include <linux/virtio_config.h>
 #include <linux/auxiliary_bus.h>
 #include <linux/mlx5/cq.h>
@@ -2499,6 +2500,11 @@ static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
 		flush_workqueue(ndev->mvdev.wq);
 }
 
+static u64 mlx5_vdpa_get_backend_features(const struct vdpa_device *vdpa)
+{
+	return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
+}
+
 static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
 {
 	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -3140,6 +3146,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = {
 	.get_vq_align = mlx5_vdpa_get_vq_align,
 	.get_vq_group = mlx5_vdpa_get_vq_group,
 	.get_device_features = mlx5_vdpa_get_device_features,
+	.get_backend_features = mlx5_vdpa_get_backend_features,
 	.set_driver_features = mlx5_vdpa_set_driver_features,
 	.get_driver_features = mlx5_vdpa_get_driver_features,
 	.set_config_cb = mlx5_vdpa_set_config_cb,
Signature · 1 line
2.39.3
#1 lore raw
Quoted 5 lines · Eugenio Pérez

Offer this backend feature as mlx5 is compatible with it. It allows it to do live migration with CVQ, dynamically switching between passthrough and shadow virtqueue.

Same comment.

Quoted 38 lines
---
 drivers/vdpa/mlx5/net/mlx5_vnet.c | 7 +++++++
 1 file changed, 7 insertions(+)
drivers/vdpa/mlx5/net/mlx5_vnet.c +7 -0
index 9138ef2fb2c8..5f309a16b9dc 100644
@@ -7,6 +7,7 @@
 #include <uapi/linux/virtio_net.h>
 #include <uapi/linux/virtio_ids.h>
 #include <uapi/linux/vdpa.h>
+#include <uapi/linux/vhost_types.h>
 #include <linux/virtio_config.h>
 #include <linux/auxiliary_bus.h>
 #include <linux/mlx5/cq.h>
@@ -2499,6 +2500,11 @@ static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
 		flush_workqueue(ndev->mvdev.wq);
 }
 
+static u64 mlx5_vdpa_get_backend_features(const struct vdpa_device *vdpa)
+{
+	return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
+}
+
 static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
 {
 	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -3140,6 +3146,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = {
 	.get_vq_align = mlx5_vdpa_get_vq_align,
 	.get_vq_group = mlx5_vdpa_get_vq_group,
 	.get_device_features = mlx5_vdpa_get_device_features,
+	.get_backend_features = mlx5_vdpa_get_backend_features,
 	.set_driver_features = mlx5_vdpa_set_driver_features,
 	.get_driver_features = mlx5_vdpa_get_driver_features,
 	.set_config_cb = mlx5_vdpa_set_config_cb,
Signature · 1 line
2.39.3

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#2 lore raw
Quoted 7 lines · Michael S. Tsirkin
Quoted 5 lines · Eugenio Pérez

Offer this backend feature as mlx5 is compatible with it. It allows it to do live migration with CVQ, dynamically switching between passthrough and shadow virtqueue.

Same comment.

to which?

-Siwei

Quoted 31 lines
Quoted 31 lines
---
  drivers/vdpa/mlx5/net/mlx5_vnet.c | 7 +++++++
  1 file changed, 7 insertions(+)
drivers/vdpa/mlx5/net/mlx5_vnet.c +6 -0
index 9138ef2fb2c8..5f309a16b9dc 100644
@@ -7,6 +7,7 @@
  #include <uapi/linux/virtio_net.h>
  #include <uapi/linux/virtio_ids.h>
  #include <uapi/linux/vdpa.h>
+#include <uapi/linux/vhost_types.h>
  #include <linux/virtio_config.h>
  #include <linux/auxiliary_bus.h>
  #include <linux/mlx5/cq.h>
@@ -2499,6 +2500,11 @@ static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
  		flush_workqueue(ndev->mvdev.wq);
  }
 
+static u64 mlx5_vdpa_get_backend_features(const struct vdpa_device *vdpa)
+{
+	return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
+}
+
  static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
  {
  	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -3140,6 +3146,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = {
  	.get_vq_align = mlx5_vdpa_get_vq_align,
Quoted 8 lines
Quoted 8 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -0
  	.get_vq_group = mlx5_vdpa_get_vq_group,
  	.get_device_features = mlx5_vdpa_get_device_features,
+	.get_backend_features = mlx5_vdpa_get_backend_features,
  	.set_driver_features = mlx5_vdpa_set_driver_features,
  	.get_driver_features = mlx5_vdpa_get_driver_features,
  	.set_config_cb = mlx5_vdpa_set_config_cb,
Signature · 1 line
2.39.3

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#3 lore raw
Quoted 8 lines · Si-Wei Liu
Quoted 5 lines · Michael S. Tsirkin
Quoted 4 lines · Eugenio Pérez

Offer this backend feature as mlx5 is compatible with it. It allows it to do live migration with CVQ, dynamically switching between passthrough and shadow virtqueue.

Quoted 5 lines
Quoted 2 lines
Quoted 1 line

Same comment.

to which?

-Siwei

VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK is too narrow a use-case to commit to it as a kernel/userspace ABI: what if one wants to start rings in some other specific order?
As was discussed on list, a better promise is not to access ring until the 1st kick. vdpa can then do a kick when it wants
the device to start accessing rings.

Quoted 30 lines
Quoted 30 lines
Quoted 30 lines
---
  drivers/vdpa/mlx5/net/mlx5_vnet.c | 7 +++++++
  1 file changed, 7 insertions(+)
drivers/vdpa/mlx5/net/mlx5_vnet.c +6 -0
index 9138ef2fb2c8..5f309a16b9dc 100644
@@ -7,6 +7,7 @@
  #include <uapi/linux/virtio_net.h>
  #include <uapi/linux/virtio_ids.h>
  #include <uapi/linux/vdpa.h>
+#include <uapi/linux/vhost_types.h>
  #include <linux/virtio_config.h>
  #include <linux/auxiliary_bus.h>
  #include <linux/mlx5/cq.h>
@@ -2499,6 +2500,11 @@ static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
  		flush_workqueue(ndev->mvdev.wq);
  }
+static u64 mlx5_vdpa_get_backend_features(const struct vdpa_device *vdpa)
+{
+	return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
+}
+
  static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
  {
  	struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -3140,6 +3146,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = {
  	.get_vq_align = mlx5_vdpa_get_vq_align,
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -0
  	.get_vq_group = mlx5_vdpa_get_vq_group,
  	.get_device_features = mlx5_vdpa_get_device_features,
+	.get_backend_features = mlx5_vdpa_get_backend_features,
  	.set_driver_features = mlx5_vdpa_set_driver_features,
  	.get_driver_features = mlx5_vdpa_get_driver_features,
  	.set_config_cb = mlx5_vdpa_set_config_cb,
Signature · 1 line
2.39.3

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#4 lore raw
Quoted 11 lines · Michael S. Tsirkin
Quoted 9 lines · Si-Wei Liu
Quoted 6 lines · Michael S. Tsirkin
Quoted 5 lines · Eugenio Pérez

Offer this backend feature as mlx5 is compatible with it. It allows it to do live migration with CVQ, dynamically switching between passthrough and shadow virtqueue.

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

Same comment.

to which?

-Siwei

VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK is too narrow a use-case to commit to it as a kernel/userspace ABI: what if one wants to start rings in some other specific order?

Just enable a queue by writing e.g 1 to queue_enable in a specific order?

Quoted 3 lines

As was discussed on list, a better promise is not to access ring until the 1st kick. vdpa can then do a kick when it wants the device to start accessing rings.

Rethink about the ACCESS_AFTER_KICK, it sounds functional equivalent to allow queue_enable after DRIVER_OK, but it seems to have distanvages:

A busy polling software device may disable notifications and never respond to register to any kick notifiers. ACCESS_AFTER_KICK will introduce overhead to those implementations.

Thanks

Quoted 31 lines
Quoted 31 lines
Quoted 31 lines
Quoted 31 lines

---
  drivers/vdpa/mlx5/net/mlx5_vnet.c | 7 +++++++
  1 file changed, 7 insertions(+)
drivers/vdpa/mlx5/net/mlx5_vnet.c +6 -0
index 9138ef2fb2c8..5f309a16b9dc 100644
@@ -7,6 +7,7 @@
  #include <uapi/linux/virtio_net.h>
  #include <uapi/linux/virtio_ids.h>
  #include <uapi/linux/vdpa.h>
+#include <uapi/linux/vhost_types.h>
  #include <linux/virtio_config.h>
  #include <linux/auxiliary_bus.h>
  #include <linux/mlx5/cq.h>
@@ -2499,6 +2500,11 @@ static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
                  flush_workqueue(ndev->mvdev.wq);
  }
+static u64 mlx5_vdpa_get_backend_features(const struct vdpa_device *vdpa)
+{
+ return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
+}
+
  static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
  {
          struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -3140,6 +3146,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = {
          .get_vq_align = mlx5_vdpa_get_vq_align,
Quoted 9 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -1
          .get_vq_group = mlx5_vdpa_get_vq_group,
          .get_device_features = mlx5_vdpa_get_device_features,
+ .get_backend_features = mlx5_vdpa_get_backend_features,
          .set_driver_features = mlx5_vdpa_set_driver_features,
          .get_driver_features = mlx5_vdpa_get_driver_features,
          .set_config_cb = mlx5_vdpa_set_config_cb,
--

2.39.3

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#5 lore raw
Quoted 15 lines · Jason Wang
Quoted 14 lines · Michael S. Tsirkin
Quoted 12 lines · Si-Wei Liu
Quoted 7 lines · Michael S. Tsirkin
Quoted 5 lines · Eugenio Pérez

Offer this backend feature as mlx5 is compatible with it. It allows it to do live migration with CVQ, dynamically switching between passthrough and shadow virtqueue.

Same comment.

to which?

Quoted 7 lines
Quoted 5 lines
Quoted 1 line

-Siwei

VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK is too narrow a use-case to commit to it as a kernel/userspace ABI: what if one wants to start rings in some other specific order?

Just enable a queue by writing e.g 1 to queue_enable in a specific order?

But then at driver ok time we don't know how many queues are there.

Quoted 8 lines
Quoted 3 lines

As was discussed on list, a better promise is not to access ring until the 1st kick. vdpa can then do a kick when it wants the device to start accessing rings.

Rethink about the ACCESS_AFTER_KICK, it sounds functional equivalent to allow queue_enable after DRIVER_OK, but it seems to have distanvages:

Quoted 5 lines

A busy polling software device may disable notifications and never respond to register to any kick notifiers. ACCESS_AFTER_KICK will introduce overhead to those implementations.

Thanks

It's just the 1st kick, then you can disable. No?

Quoted 32 lines
Quoted 32 lines
Quoted 32 lines
Quoted 32 lines
Quoted 32 lines

---
  drivers/vdpa/mlx5/net/mlx5_vnet.c | 7 +++++++
  1 file changed, 7 insertions(+)
drivers/vdpa/mlx5/net/mlx5_vnet.c +6 -0
index 9138ef2fb2c8..5f309a16b9dc 100644
@@ -7,6 +7,7 @@
  #include <uapi/linux/virtio_net.h>
  #include <uapi/linux/virtio_ids.h>
  #include <uapi/linux/vdpa.h>
+#include <uapi/linux/vhost_types.h>
  #include <linux/virtio_config.h>
  #include <linux/auxiliary_bus.h>
  #include <linux/mlx5/cq.h>
@@ -2499,6 +2500,11 @@ static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
                  flush_workqueue(ndev->mvdev.wq);
  }
+static u64 mlx5_vdpa_get_backend_features(const struct vdpa_device *vdpa)
+{
+ return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
+}
+
  static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
  {
          struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -3140,6 +3146,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = {
          .get_vq_align = mlx5_vdpa_get_vq_align,
          .get_vq_group = mlx5_vdpa_get_vq_group,
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -1
          .get_device_features = mlx5_vdpa_get_device_features,
+ .get_backend_features = mlx5_vdpa_get_backend_features,
          .set_driver_features = mlx5_vdpa_set_driver_features,
          .get_driver_features = mlx5_vdpa_get_driver_features,
          .set_config_cb = mlx5_vdpa_set_config_cb,
--

2.39.3

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#6 lore raw
Quoted 19 lines · Michael S. Tsirkin
Quoted 17 lines · Jason Wang
Quoted 16 lines · Michael S. Tsirkin
Quoted 13 lines · Si-Wei Liu
Quoted 7 lines · Michael S. Tsirkin
Quoted 5 lines · Eugenio Pérez

Offer this backend feature as mlx5 is compatible with it. It allows it to do live migration with CVQ, dynamically switching between passthrough and shadow virtqueue.

Same comment.

to which?

-Siwei

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines

VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK is too narrow a use-case to commit to it as a kernel/userspace ABI: what if one wants to start rings in some other specific order?

Just enable a queue by writing e.g 1 to queue_enable in a specific order?

But then at driver ok time we don't know how many queues are there.

There should be a device specific interface for this, for example, num_queue_paris. So the device knows at most how many queues there are. Or anything I miss?

Quoted 9 lines
Quoted 9 lines
Quoted 4 lines

As was discussed on list, a better promise is not to access ring until the 1st kick. vdpa can then do a kick when it wants the device to start accessing rings.

Rethink about the ACCESS_AFTER_KICK, it sounds functional equivalent to allow queue_enable after DRIVER_OK, but it seems to have distanvages:

Quoted 7 lines
Quoted 5 lines

A busy polling software device may disable notifications and never respond to register to any kick notifiers. ACCESS_AFTER_KICK will introduce overhead to those implementations.

Thanks

It's just the 1st kick, then you can disable. No?

Yes, but:

1) adding hooks for queue_enable
2) adding new codes to register event notifier and toggle the notifier

1) seems much easier? Or for most devices, it already behaves like this.

Thanks

Quoted 33 lines
Quoted 33 lines
Quoted 33 lines
Quoted 33 lines
Quoted 33 lines
Quoted 33 lines


---
  drivers/vdpa/mlx5/net/mlx5_vnet.c | 7 +++++++
  1 file changed, 7 insertions(+)
drivers/vdpa/mlx5/net/mlx5_vnet.c +6 -0
index 9138ef2fb2c8..5f309a16b9dc 100644
@@ -7,6 +7,7 @@
  #include <uapi/linux/virtio_net.h>
  #include <uapi/linux/virtio_ids.h>
  #include <uapi/linux/vdpa.h>
+#include <uapi/linux/vhost_types.h>
  #include <linux/virtio_config.h>
  #include <linux/auxiliary_bus.h>
  #include <linux/mlx5/cq.h>
@@ -2499,6 +2500,11 @@ static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
                  flush_workqueue(ndev->mvdev.wq);
  }
+static u64 mlx5_vdpa_get_backend_features(const struct vdpa_device *vdpa)
+{
+ return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
+}
+
  static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
  {
          struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -3140,6 +3146,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = {
          .get_vq_align = mlx5_vdpa_get_vq_align,
          .get_vq_group = mlx5_vdpa_get_vq_group,
Quoted 9 lines
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -1
          .get_device_features = mlx5_vdpa_get_device_features,
+ .get_backend_features = mlx5_vdpa_get_backend_features,
          .set_driver_features = mlx5_vdpa_set_driver_features,
          .get_driver_features = mlx5_vdpa_get_driver_features,
          .set_config_cb = mlx5_vdpa_set_config_cb,
--

2.39.3

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#7 lore raw
Quoted 24 lines · Jason Wang
Quoted 23 lines · Michael S. Tsirkin
Quoted 21 lines · Jason Wang
Quoted 19 lines · Michael S. Tsirkin
Quoted 8 lines · Si-Wei Liu
Quoted 5 lines · Michael S. Tsirkin
Quoted 4 lines · Eugenio Pérez

Offer this backend feature as mlx5 is compatible with it. It allows it to do live migration with CVQ, dynamically switching between passthrough and shadow virtqueue.

Quoted 5 lines
Quoted 2 lines
Quoted 1 line

Same comment.

to which?

-Siwei

VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK is too narrow a use-case to commit to it as a kernel/userspace ABI: what if one wants to start rings in some other specific order?

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

Just enable a queue by writing e.g 1 to queue_enable in a specific order?

But then at driver ok time we don't know how many queues are there.

There should be a device specific interface for this, for example, num_queue_paris. So the device knows at most how many queues there are. Or anything I miss?

That's a device limitations. Does not tell the device how much is used.

Quoted 17 lines
Quoted 9 lines
Quoted 9 lines
Quoted 4 lines

As was discussed on list, a better promise is not to access ring until the 1st kick. vdpa can then do a kick when it wants the device to start accessing rings.

Rethink about the ACCESS_AFTER_KICK, it sounds functional equivalent to allow queue_enable after DRIVER_OK, but it seems to have distanvages:

Quoted 7 lines
Quoted 5 lines

A busy polling software device may disable notifications and never respond to register to any kick notifiers. ACCESS_AFTER_KICK will introduce overhead to those implementations.

Thanks

It's just the 1st kick, then you can disable. No?

Quoted 8 lines

Yes, but:

1) adding hooks for queue_enable
2) adding new codes to register event notifier and toggle the notifier

1) seems much easier? Or for most devices, it already behaves like this.

Thanks

Well libvhostuser checks enabled queues at DRIVER_OK, does it not?

Quoted 34 lines
Quoted 34 lines
Quoted 34 lines
Quoted 34 lines
Quoted 34 lines
Quoted 34 lines
Quoted 34 lines


---
  drivers/vdpa/mlx5/net/mlx5_vnet.c | 7 +++++++
  1 file changed, 7 insertions(+)
drivers/vdpa/mlx5/net/mlx5_vnet.c +6 -0
index 9138ef2fb2c8..5f309a16b9dc 100644
@@ -7,6 +7,7 @@
  #include <uapi/linux/virtio_net.h>
  #include <uapi/linux/virtio_ids.h>
  #include <uapi/linux/vdpa.h>
+#include <uapi/linux/vhost_types.h>
  #include <linux/virtio_config.h>
  #include <linux/auxiliary_bus.h>
  #include <linux/mlx5/cq.h>
@@ -2499,6 +2500,11 @@ static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
                  flush_workqueue(ndev->mvdev.wq);
  }
+static u64 mlx5_vdpa_get_backend_features(const struct vdpa_device *vdpa)
+{
+ return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
+}
+
  static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
  {
          struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -3140,6 +3146,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = {
          .get_vq_align = mlx5_vdpa_get_vq_align,
          .get_vq_group = mlx5_vdpa_get_vq_group,
          .get_device_features = mlx5_vdpa_get_device_features,
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -1
+ .get_backend_features = mlx5_vdpa_get_backend_features,
          .set_driver_features = mlx5_vdpa_set_driver_features,
          .get_driver_features = mlx5_vdpa_get_driver_features,
          .set_config_cb = mlx5_vdpa_set_config_cb,
--

2.39.3

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#8 lore raw
Quoted 29 lines · Michael S. Tsirkin
Quoted 27 lines · Jason Wang
Quoted 26 lines · Michael S. Tsirkin
Quoted 22 lines · Jason Wang
Quoted 11 lines · Michael S. Tsirkin
Quoted 9 lines · Si-Wei Liu
Quoted 6 lines · Michael S. Tsirkin
Quoted 5 lines · Eugenio Pérez

Offer this backend feature as mlx5 is compatible with it. It allows it to do live migration with CVQ, dynamically switching between passthrough and shadow virtqueue.

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

Same comment.

to which?

-Siwei

VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK is too narrow a use-case to commit to it as a kernel/userspace ABI: what if one wants to start rings in some other specific order?

Just enable a queue by writing e.g 1 to queue_enable in a specific order?

Quoted 7 lines
Quoted 5 lines
Quoted 1 line

But then at driver ok time we don't know how many queues are there.

There should be a device specific interface for this, for example, num_queue_paris. So the device knows at most how many queues there are. Or anything I miss?

That's a device limitations. Does not tell the device how much is used.

I think I miss something, how kick differs from queue_enable in this way?

Quoted 20 lines
Quoted 20 lines
Quoted 10 lines
Quoted 10 lines
Quoted 5 lines

As was discussed on list, a better promise is not to access ring until the 1st kick. vdpa can then do a kick when it wants the device to start accessing rings.

Rethink about the ACCESS_AFTER_KICK, it sounds functional equivalent to allow queue_enable after DRIVER_OK, but it seems to have distanvages:

Quoted 7 lines
Quoted 5 lines

A busy polling software device may disable notifications and never respond to register to any kick notifiers. ACCESS_AFTER_KICK will introduce overhead to those implementations.

Thanks

It's just the 1st kick, then you can disable. No?

Yes, but:

Quoted 8 lines
Quoted 6 lines

1) adding hooks for queue_enable
2) adding new codes to register event notifier and toggle the notifier

1) seems much easier? Or for most devices, it already behaves like this.

Thanks

Well libvhostuser checks enabled queues at DRIVER_OK, does it not?

Probably, but I meant:

1) This behaviour has been supported by some device (e.g MLX)
2) This is the current behaviour of Qemu for vhost-net devices:

static void virtio_net_queue_enable(VirtIODevice *vdev, uint32_t queue_index)

{
    VirtIONet *n = VIRTIO_NET(vdev);
    NetClientState *nc;
    int r;

    ....

    if (get_vhost_net(nc->peer) &&
        nc->peer->info->type == NET_CLIENT_DRIVER_TAP) {
        r = vhost_net_virtqueue_restart(vdev, nc, queue_index);
        if (r < 0) {
            error_report("unable to restart vhost net virtqueue: %d, "
                            "when resetting the queue", queue_index);
        }
    }
}

Thanks

Quoted 35 lines
Quoted 35 lines
Quoted 35 lines
Quoted 35 lines
Quoted 35 lines
Quoted 35 lines
Quoted 35 lines
Quoted 35 lines



---
  drivers/vdpa/mlx5/net/mlx5_vnet.c | 7 +++++++
  1 file changed, 7 insertions(+)
drivers/vdpa/mlx5/net/mlx5_vnet.c +6 -0
index 9138ef2fb2c8..5f309a16b9dc 100644
@@ -7,6 +7,7 @@
  #include <uapi/linux/virtio_net.h>
  #include <uapi/linux/virtio_ids.h>
  #include <uapi/linux/vdpa.h>
+#include <uapi/linux/vhost_types.h>
  #include <linux/virtio_config.h>
  #include <linux/auxiliary_bus.h>
  #include <linux/mlx5/cq.h>
@@ -2499,6 +2500,11 @@ static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
                  flush_workqueue(ndev->mvdev.wq);
  }
+static u64 mlx5_vdpa_get_backend_features(const struct vdpa_device *vdpa)
+{
+ return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
+}
+
  static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
  {
          struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -3140,6 +3146,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = {
          .get_vq_align = mlx5_vdpa_get_vq_align,
          .get_vq_group = mlx5_vdpa_get_vq_group,
          .get_device_features = mlx5_vdpa_get_device_features,
Quoted 9 lines
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -1
+ .get_backend_features = mlx5_vdpa_get_backend_features,
          .set_driver_features = mlx5_vdpa_set_driver_features,
          .get_driver_features = mlx5_vdpa_get_driver_features,
          .set_config_cb = mlx5_vdpa_set_config_cb,
--

2.39.3

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

author maintainer
#9 lore raw
In reply to Michael S. Tsirkin
Quoted 14 lines · Michael S. Tsirkin
Quoted 12 lines · Si-Wei Liu
Quoted 7 lines · Michael S. Tsirkin
Quoted 5 lines · Eugenio Pérez

Offer this backend feature as mlx5 is compatible with it. It allows it to do live migration with CVQ, dynamically switching between passthrough and shadow virtqueue.

Same comment.

to which?

Quoted 9 lines
Quoted 1 line

-Siwei

VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK is too narrow a use-case to commit to it as a kernel/userspace ABI: what if one wants to start rings in some other specific order?
As was discussed on list, a better promise is not to access ring until the 1st kick. vdpa can then do a kick when it wants
the device to start accessing rings.

Friendly ping about this series,

Now that VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK has been merged for vdpa_sim, does it make sense for mlx too?

Thanks!

Quoted 39 lines
Quoted 38 lines
Quoted 38 lines
Quoted 38 lines
---
  drivers/vdpa/mlx5/net/mlx5_vnet.c | 7 +++++++
  1 file changed, 7 insertions(+)
drivers/vdpa/mlx5/net/mlx5_vnet.c +7 -1
index 9138ef2fb2c8..5f309a16b9dc 100644
@@ -7,6 +7,7 @@
  #include <uapi/linux/virtio_net.h>
  #include <uapi/linux/virtio_ids.h>
  #include <uapi/linux/vdpa.h>
+#include <uapi/linux/vhost_types.h>
  #include <linux/virtio_config.h>
  #include <linux/auxiliary_bus.h>
  #include <linux/mlx5/cq.h>
@@ -2499,6 +2500,11 @@ static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
                  flush_workqueue(ndev->mvdev.wq);
  }
+static u64 mlx5_vdpa_get_backend_features(const struct vdpa_device *vdpa)
+{
+ return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
+}
+
  static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
  {
          struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -3140,6 +3146,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = {
          .get_vq_align = mlx5_vdpa_get_vq_align,
          .get_vq_group = mlx5_vdpa_get_vq_group,
          .get_device_features = mlx5_vdpa_get_device_features,
+ .get_backend_features = mlx5_vdpa_get_backend_features,
          .set_driver_features = mlx5_vdpa_set_driver_features,
          .get_driver_features = mlx5_vdpa_get_driver_features,
          .set_config_cb = mlx5_vdpa_set_config_cb,
--

2.39.3

#10 lore raw
Quoted 25 lines · Eugenio Perez Martin
Quoted 14 lines · Michael S. Tsirkin
Quoted 12 lines · Si-Wei Liu
Quoted 7 lines · Michael S. Tsirkin
Quoted 5 lines · Eugenio Pérez

Offer this backend feature as mlx5 is compatible with it. It allows it to do live migration with CVQ, dynamically switching between passthrough and shadow virtqueue.

Same comment.

to which?

Quoted 9 lines
Quoted 1 line

-Siwei

VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK is too narrow a use-case to commit to it as a kernel/userspace ABI: what if one wants to start rings in some other specific order?
As was discussed on list, a better promise is not to access ring until the 1st kick. vdpa can then do a kick when it wants
the device to start accessing rings.

Quoted 6 lines

Friendly ping about this series,

Now that VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK has been merged for vdpa_sim, does it make sense for mlx too?

Thanks!

For sure. I was just busy with a qemu pull, will handle this next.

Quoted 31 lines
Quoted 31 lines
Quoted 31 lines
Quoted 31 lines
Quoted 31 lines
---
  drivers/vdpa/mlx5/net/mlx5_vnet.c | 7 +++++++
  1 file changed, 7 insertions(+)
drivers/vdpa/mlx5/net/mlx5_vnet.c +6 -0
index 9138ef2fb2c8..5f309a16b9dc 100644
@@ -7,6 +7,7 @@
  #include <uapi/linux/virtio_net.h>
  #include <uapi/linux/virtio_ids.h>
  #include <uapi/linux/vdpa.h>
+#include <uapi/linux/vhost_types.h>
  #include <linux/virtio_config.h>
  #include <linux/auxiliary_bus.h>
  #include <linux/mlx5/cq.h>
@@ -2499,6 +2500,11 @@ static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
                  flush_workqueue(ndev->mvdev.wq);
  }
+static u64 mlx5_vdpa_get_backend_features(const struct vdpa_device *vdpa)
+{
+ return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
+}
+
  static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
  {
          struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -3140,6 +3146,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = {
          .get_vq_align = mlx5_vdpa_get_vq_align,
          .get_vq_group = mlx5_vdpa_get_vq_group,
Quoted 8 lines
Quoted 8 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -1
          .get_device_features = mlx5_vdpa_get_device_features,
+ .get_backend_features = mlx5_vdpa_get_backend_features,
          .set_driver_features = mlx5_vdpa_set_driver_features,
          .get_driver_features = mlx5_vdpa_get_driver_features,
          .set_config_cb = mlx5_vdpa_set_config_cb,
--

2.39.3

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

author maintainer
#11 lore raw
Quoted 27 lines · Michael S. Tsirkin
Quoted 25 lines · Eugenio Perez Martin
Quoted 23 lines · Michael S. Tsirkin
Quoted 8 lines · Si-Wei Liu
Quoted 5 lines · Michael S. Tsirkin
Quoted 4 lines · Eugenio Pérez

Offer this backend feature as mlx5 is compatible with it. It allows it to do live migration with CVQ, dynamically switching between passthrough and shadow virtqueue.

Quoted 5 lines
Quoted 2 lines
Quoted 1 line

Same comment.

to which?

-Siwei

VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK is too narrow a use-case to commit to it as a kernel/userspace ABI: what if one wants to start rings in some other specific order?
As was discussed on list, a better promise is not to access ring until the 1st kick. vdpa can then do a kick when it wants
the device to start accessing rings.

Quoted 9 lines
Quoted 6 lines

Friendly ping about this series,

Now that VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK has been merged for vdpa_sim, does it make sense for mlx too?

Thanks!

For sure. I was just busy with a qemu pull, will handle this next.

Sounds great, thanks!

Quoted 40 lines
Quoted 39 lines
Quoted 39 lines
Quoted 38 lines
Quoted 38 lines
Quoted 38 lines
---
  drivers/vdpa/mlx5/net/mlx5_vnet.c | 7 +++++++
  1 file changed, 7 insertions(+)
drivers/vdpa/mlx5/net/mlx5_vnet.c +7 -1
index 9138ef2fb2c8..5f309a16b9dc 100644
@@ -7,6 +7,7 @@
  #include <uapi/linux/virtio_net.h>
  #include <uapi/linux/virtio_ids.h>
  #include <uapi/linux/vdpa.h>
+#include <uapi/linux/vhost_types.h>
  #include <linux/virtio_config.h>
  #include <linux/auxiliary_bus.h>
  #include <linux/mlx5/cq.h>
@@ -2499,6 +2500,11 @@ static void unregister_link_notifier(struct mlx5_vdpa_net *ndev)
                  flush_workqueue(ndev->mvdev.wq);
  }
+static u64 mlx5_vdpa_get_backend_features(const struct vdpa_device *vdpa)
+{
+ return BIT_ULL(VHOST_BACKEND_F_ENABLE_AFTER_DRIVER_OK);
+}
+
  static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
  {
          struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev);
@@ -3140,6 +3146,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = {
          .get_vq_align = mlx5_vdpa_get_vq_align,
          .get_vq_group = mlx5_vdpa_get_vq_group,
          .get_device_features = mlx5_vdpa_get_device_features,
+ .get_backend_features = mlx5_vdpa_get_backend_features,
          .set_driver_features = mlx5_vdpa_set_driver_features,
          .get_driver_features = mlx5_vdpa_get_driver_features,
          .set_config_cb = mlx5_vdpa_set_config_cb,
--

2.39.3