Skip to contentSkip to conversation

vdpa: Avoid taking cf_mutex lock on get status

PATCH Si-Wei Liu 3 participants 13 messages Jan 11, 2022 – Jan 14, 2022 6 messages not in the archive virtio core

lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

A message that is not in the archive (sent off-list or not yet received): [email protected]

Si-Wei Liu

author Reviewed-by
#2 lore raw
Quoted 5 lines · Eli Cohen

Avoid the wrapper holding cf_mutex since it is not protecting anything. To avoid confusion and unnecessary overhead incurred by it, remove.

Fixes f489f27bc0ab ("vdpa: Sync calls set/get config/status with cf_mutex")
Quoted 58 lines
---
  drivers/vdpa/vdpa.c  | 11 -----------
  drivers/vhost/vdpa.c |  5 +++--
  include/linux/vdpa.h |  1 -
  3 files changed, 3 insertions(+), 14 deletions(-)
drivers/vdpa/vdpa.c +0 -11
index 4380367d00b5..9846c9de4bfa 100644
@@ -21,17 +21,6 @@ static LIST_HEAD(mdev_head);
  static DEFINE_MUTEX(vdpa_dev_mutex);
  static DEFINE_IDA(vdpa_index_ida);
 
-u8 vdpa_get_status(struct vdpa_device *vdev)
-{
-	u8 status;
-
-	mutex_lock(&vdev->cf_mutex);
-	status = vdev->config->get_status(vdev);
-	mutex_unlock(&vdev->cf_mutex);
-	return status;
-}
-EXPORT_SYMBOL(vdpa_get_status);
-
  void vdpa_set_status(struct vdpa_device *vdev, u8 status)
  {
  	mutex_lock(&vdev->cf_mutex);
drivers/vhost/vdpa.c +3 -2
index 6e7edaf2472b..0ed6cbadb52d 100644
@@ -142,9 +142,10 @@ static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
  static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)
  {
  	struct vdpa_device *vdpa = v->vdpa;
+	const struct vdpa_config_ops *ops = vdpa->config;
  	u8 status;
 
-	status = vdpa_get_status(vdpa);
+	status = ops->get_status(vdpa);
 
  	if (copy_to_user(statusp, &status, sizeof(status)))
  		return -EFAULT;
@@ -163,7 +164,7 @@ static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)
  	if (copy_from_user(&status, statusp, sizeof(status)))
  		return -EFAULT;
 
-	status_old = vdpa_get_status(vdpa);
+	status_old = ops->get_status(vdpa);
 
  	/*
  	 * Userspace shouldn't remove status bits unless reset the
include/linux/vdpa.h +0 -0
index a6047fd6cf12..2de442ececae 100644

include/linux/vdpa.h void vdpa_get_config(struct vdpa_device *vdev, unsigned int offset, · line 426

Quoted 7 lines
include/linux/vdpa.h +0 -1
@@ -421,7 +421,6 @@ void vdpa_get_config(struct vdpa_device *vdev, unsigned int offset,
  		     void *buf, unsigned int len);
  void vdpa_set_config(struct vdpa_device *dev, unsigned int offset,
  		     const void *buf, unsigned int length);
-u8 vdpa_get_status(struct vdpa_device *vdev);
  void vdpa_set_status(struct vdpa_device *vdev, u8 status);
 
  /**

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Acked-by
#3 lore raw
In reply to a message not in the archive
Quoted 6 lines · Eli Cohen

Avoid the wrapper holding cf_mutex since it is not protecting anything. To avoid confusion and unnecessary overhead incurred by it, remove.

Fixes f489f27bc0ab ("vdpa: Sync calls set/get config/status with cf_mutex")
Quoted 66 lines
---
 drivers/vdpa/vdpa.c  | 11 -----------
 drivers/vhost/vdpa.c |  5 +++--
 include/linux/vdpa.h |  1 -
 3 files changed, 3 insertions(+), 14 deletions(-)
drivers/vdpa/vdpa.c +0 -11
index 4380367d00b5..9846c9de4bfa 100644
@@ -21,17 +21,6 @@ static LIST_HEAD(mdev_head);
 static DEFINE_MUTEX(vdpa_dev_mutex);
 static DEFINE_IDA(vdpa_index_ida);
 
-u8 vdpa_get_status(struct vdpa_device *vdev)
-{
-       u8 status;
-
-       mutex_lock(&vdev->cf_mutex);
-       status = vdev->config->get_status(vdev);
-       mutex_unlock(&vdev->cf_mutex);
-       return status;
-}
-EXPORT_SYMBOL(vdpa_get_status);
-
 void vdpa_set_status(struct vdpa_device *vdev, u8 status)
 {
        mutex_lock(&vdev->cf_mutex);
drivers/vhost/vdpa.c +3 -2
index 6e7edaf2472b..0ed6cbadb52d 100644
@@ -142,9 +142,10 @@ static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)
 static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)
 {
        struct vdpa_device *vdpa = v->vdpa;
+       const struct vdpa_config_ops *ops = vdpa->config;
        u8 status;
 
-       status = vdpa_get_status(vdpa);
+       status = ops->get_status(vdpa);
 
        if (copy_to_user(statusp, &status, sizeof(status)))
                return -EFAULT;
@@ -163,7 +164,7 @@ static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)
        if (copy_from_user(&status, statusp, sizeof(status)))
                return -EFAULT;
 
-       status_old = vdpa_get_status(vdpa);
+       status_old = ops->get_status(vdpa);
 
        /*
         * Userspace shouldn't remove status bits unless reset the
include/linux/vdpa.h +0 -2
index a6047fd6cf12..2de442ececae 100644
@@ -421,7 +421,6 @@ void vdpa_get_config(struct vdpa_device *vdev, unsigned int offset,
                     void *buf, unsigned int len);
 void vdpa_set_config(struct vdpa_device *dev, unsigned int offset,
                     const void *buf, unsigned int length);
-u8 vdpa_get_status(struct vdpa_device *vdev);
 void vdpa_set_status(struct vdpa_device *vdev, u8 status);
 
 /**
--
Quoted 2 lines

2.34.1

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

A message that is not in the archive (sent off-list or not yet received): [email protected]

author Reviewed-by
#5 lore raw

Re: [PATCH 2/4] vdpa: Protect vdpa reset with cf_mutex

Quoted 5 lines · Eli Cohen

Call reset using the wrapper function vdpa_reset() to make sure the operation is serialized with cf_mutex.

This comes to protect from the following possible scenario:

Quoted 7 lines

vhost_vdpa_set_status() could call the reset op. Since the call is not protected by cf_mutex, a netlink thread calling vdpa_dev_config_fill could get passed the VIRTIO_CONFIG_S_FEATURES_OK check in vdpa_dev_config_fill() and end up reporting wrong features.

Fixes 5f6e85953d8f ("vdpa: Read device configuration only if FEATURES_OK")
Quoted 9 lines
---
  drivers/vhost/vdpa.c | 2 +-
  1 file changed, 1 insertion(+), 1 deletion(-)
drivers/vhost/vdpa.c +0 -0
index 0ed6cbadb52d..851539807bc9 100644

drivers/vhost/vdpa.c static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp) · line 184

Quoted 8 lines
drivers/vhost/vdpa.c +1 -1
@@ -178,7 +178,7 @@ static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)
  			vhost_vdpa_unsetup_vq_irq(v, i);
 
  	if (status == 0) {
-		ret = ops->reset(vdpa);
+		ret = vdpa_reset(vdpa);
  		if (ret)
  			return ret;
  	} else

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Acked-by
#6 lore raw

Re: [PATCH 2/4] vdpa: Protect vdpa reset with cf_mutex

In reply to a message not in the archive
Quoted 6 lines · Eli Cohen

Call reset using the wrapper function vdpa_reset() to make sure the operation is serialized with cf_mutex.

This comes to protect from the following possible scenario:

Quoted 7 lines

vhost_vdpa_set_status() could call the reset op. Since the call is not protected by cf_mutex, a netlink thread calling vdpa_dev_config_fill could get passed the VIRTIO_CONFIG_S_FEATURES_OK check in vdpa_dev_config_fill() and end up reporting wrong features.

Fixes 5f6e85953d8f ("vdpa: Read device configuration only if FEATURES_OK")
Quoted 18 lines
---
 drivers/vhost/vdpa.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/vhost/vdpa.c +1 -2
index 0ed6cbadb52d..851539807bc9 100644
@@ -178,7 +178,7 @@ static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)
                        vhost_vdpa_unsetup_vq_irq(v, i);
 
        if (status == 0) {
-               ret = ops->reset(vdpa);
+               ret = vdpa_reset(vdpa);
                if (ret)
                        return ret;
        } else
--
Quoted 2 lines

2.34.1

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

A message that is not in the archive (sent off-list or not yet received): [email protected]

author Reviewed-by
#8 lore raw

Re: [PATCH 3/4] vdpa/mlx5: Fix is_index_valid() to refer to features

Quoted 8 lines · Eli Cohen

Make sure the decision whether an index received through a callback is valid or not consults the negotiated features.

The motivation for this was due to a case encountered where I shut down the VM. After the reset operation was called features were already clear, I got get_vq_state() call which caused out array bounds access since is_index_valid() reported the index value.

Quoted 4 lines

So this is more of not hit a bug since the call shouldn't have been made first place.

Quoted 16 lines
---
  drivers/vdpa/mlx5/net/mlx5_vnet.c | 10 +++++++---
  1 file changed, 7 insertions(+), 3 deletions(-)
drivers/vdpa/mlx5/net/mlx5_vnet.c +2 -2
index d1ff65065fb1..9eacfdb48434 100644
@@ -133,10 +133,14 @@ struct mlx5_vdpa_virtqueue {
 
  static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx)
  {
-	if (unlikely(idx > mvdev->max_idx))
-		return false;
+	if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) {
+		if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)))

drivers/vdpa/mlx5/net/mlx5_vnet.c struct mlx5_vdpa_virtqueue { · line 146

Quoted 10 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +5 -1
+			return idx < 2;
+		else
+			return idx < 3;
+	}
 
-	return true;
+	return idx <= mvdev->max_idx;
  }
 
  struct mlx5_vdpa_net {

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Acked-by
#9 lore raw

Re: [PATCH 3/4] vdpa/mlx5: Fix is_index_valid() to refer to features

In reply to a message not in the archive
Quoted 9 lines · Eli Cohen

Make sure the decision whether an index received through a callback is valid or not consults the negotiated features.

The motivation for this was due to a case encountered where I shut down the VM. After the reset operation was called features were already clear, I got get_vq_state() call which caused out array bounds access since is_index_valid() reported the index value.

Quoted 4 lines

So this is more of not hit a bug since the call shouldn't have been made first place.

Quoted 27 lines
---
 drivers/vdpa/mlx5/net/mlx5_vnet.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)
drivers/vdpa/mlx5/net/mlx5_vnet.c +7 -4
index d1ff65065fb1..9eacfdb48434 100644
@@ -133,10 +133,14 @@ struct mlx5_vdpa_virtqueue {
 
 static bool is_index_valid(struct mlx5_vdpa_dev *mvdev, u16 idx)
 {
-       if (unlikely(idx > mvdev->max_idx))
-               return false;
+       if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_MQ))) {
+               if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)))
+                       return idx < 2;
+               else
+                       return idx < 3;
+       }
 
-       return true;
+       return idx <= mvdev->max_idx;
 }
 
 struct mlx5_vdpa_net {
--
Quoted 2 lines

2.34.1

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

A message that is not in the archive (sent off-list or not yet received): [email protected]

author Reviewed-by
#11 lore raw

Re: [PATCH 4/4] vdpa/mlx5: Fix tracking of current number of VQs

Quoted 21 lines · Eli Cohen

Modify the code such that ndev->cur_num_vqs better reflects the actual number of data virtqueues. The value can be accurately realized after features have been negotiated.

This is to prevent possible failures when modifying the RQT object if the cur_num_vqs bears invalid value.

No issue was actually encountered but this also makes the code more readable.

Fixes c5a5cd3d3217 ("vdpa/mlx5: Support configuring max data virtqueue")
---
  drivers/vdpa/mlx5/net/mlx5_vnet.c | 12 ++++++++----
  1 file changed, 8 insertions(+), 4 deletions(-)
drivers/vdpa/mlx5/net/mlx5_vnet.c +0 -0
index 9eacfdb48434..b53603d94082 100644

drivers/vdpa/mlx5/net/mlx5_vnet.c static int create_rqt(struct mlx5_vdpa_net *ndev) · line 1249

Quoted 6 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -2
@@ -1246,8 +1246,7 @@ static int create_rqt(struct mlx5_vdpa_net *ndev)
  	if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ)))
  		num = 1;
  	else
-		num = mlx5vdpa16_to_cpu(&ndev->mvdev,
-					ndev->config.max_virtqueue_pairs);
+		num = ndev->cur_num_vqs / 2;

Nit: the if branch can be consolidated

drivers/vdpa/mlx5/net/mlx5_vnet.c static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) · line 1986

Quoted 9 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +2 -0
  	max_rqt = min_t(int, roundup_pow_of_two(num),
  			1 << MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size));
@@ -1983,6 +1982,11 @@ static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
  		return err;
 
  	ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features;
+	if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ))
+		ndev->cur_num_vqs = 2 * mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs);

Hmmm, not this patch, but there should've been validation done in the upper layer to guarantee set_featuers() for VIRTIO_NET_F_MQ always comes with VIRTIO_NET_F_CTRL_VQ. Maybe checking both: BIT_ULL(VIRTIO_NET_F_MQ) |  BIT_ULL(VIRTIO_NET_F_CTRL_VQ)?

otherwise it looks good to me.

Quoted 15 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +4 -0
+	else
+		ndev->cur_num_vqs = 2;
+
  	update_cvq_info(mvdev);
  	return err;
  }
@@ -2233,6 +2237,7 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev)
  	clear_vqs_ready(ndev);
  	mlx5_vdpa_destroy_mr(&ndev->mvdev);
  	ndev->mvdev.status = 0;
+	ndev->cur_num_vqs = 0;
  	memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1));
  	ndev->mvdev.actual_features = 0;
  	++mvdev->generation;

drivers/vdpa/mlx5/net/mlx5_vnet.c static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, · line 2653

Quoted 10 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -2
@@ -2641,9 +2646,8 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
 
  	ndev->nb.notifier_call = event_handler;
  	mlx5_notifier_register(mdev, &ndev->nb);
-	ndev->cur_num_vqs = 2 * mlx5_vdpa_max_qps(max_vqs);
  	mvdev->vdev.mdev = &mgtdev->mgtdev;
-	err = _vdpa_register_device(&mvdev->vdev, ndev->cur_num_vqs + 1);
+	err = _vdpa_register_device(&mvdev->vdev, 2 * mlx5_vdpa_max_qps(max_vqs) + 1);
  	if (err)
  		goto err_reg;
 

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#12 lore raw

Re: [PATCH 4/4] vdpa/mlx5: Fix tracking of current number of VQs

Quoted 39 lines · Si-Wei Liu
Quoted 21 lines · Eli Cohen

Modify the code such that ndev->cur_num_vqs better reflects the actual number of data virtqueues. The value can be accurately realized after features have been negotiated.

This is to prevent possible failures when modifying the RQT object if the cur_num_vqs bears invalid value.

No issue was actually encountered but this also makes the code more readable.

Fixes c5a5cd3d3217 ("vdpa/mlx5: Support configuring max data virtqueue")
---
  drivers/vdpa/mlx5/net/mlx5_vnet.c | 12 ++++++++----
  1 file changed, 8 insertions(+), 4 deletions(-)
drivers/vdpa/mlx5/net/mlx5_vnet.c +0 -0
index 9eacfdb48434..b53603d94082 100644

drivers/vdpa/mlx5/net/mlx5_vnet.c static int create_rqt(struct mlx5_vdpa_net *ndev) · line 1249

Quoted 6 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -2
@@ -1246,8 +1246,7 @@ static int create_rqt(struct mlx5_vdpa_net *ndev)
      if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ)))
              num = 1;
      else
-             num = mlx5vdpa16_to_cpu(&ndev->mvdev,
-                                     ndev->config.max_virtqueue_pairs);
+             num = ndev->cur_num_vqs / 2;

Nit: the if branch can be consolidated

Quoted 6 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +0 -0
      max_rqt = min_t(int, roundup_pow_of_two(num),
                      1 << MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size));
@@ -1983,6 +1982,11 @@ static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
              return err;
 
Quoted 7 lines

drivers/vdpa/mlx5/net/mlx5_vnet.c static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) · line 1986

Quoted 3 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +2 -0
      ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features;
+     if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ))
+             ndev->cur_num_vqs = 2 * mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs);

Hmmm, not this patch, but there should've been validation done in the upper layer to guarantee set_featuers() for VIRTIO_NET_F_MQ always comes with VIRTIO_NET_F_CTRL_VQ. Maybe checking both: BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ)?

So the upper layer is unaware of the device type. It's better to do that in mlx5's set_features() according to the spec:

The device MUST NOT offer a feature which requires another feature which was not offered.

Thanks

Quoted 20 lines

otherwise it looks good to me.

Quoted 16 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +4 -0
+     else
+             ndev->cur_num_vqs = 2;
+
      update_cvq_info(mvdev);
      return err;
  }
@@ -2233,6 +2237,7 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev)
      clear_vqs_ready(ndev);
      mlx5_vdpa_destroy_mr(&ndev->mvdev);
      ndev->mvdev.status = 0;
+     ndev->cur_num_vqs = 0;
      memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1));
      ndev->mvdev.actual_features = 0;
      ++mvdev->generation;
@@ -2641,9 +2646,8 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
 
Quoted 10 lines

drivers/vdpa/mlx5/net/mlx5_vnet.c static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, · line 2653

Quoted 9 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -2
      ndev->nb.notifier_call = event_handler;
      mlx5_notifier_register(mdev, &ndev->nb);
-     ndev->cur_num_vqs = 2 * mlx5_vdpa_max_qps(max_vqs);
      mvdev->vdev.mdev = &mgtdev->mgtdev;
-     err = _vdpa_register_device(&mvdev->vdev, ndev->cur_num_vqs + 1);
+     err = _vdpa_register_device(&mvdev->vdev, 2 * mlx5_vdpa_max_qps(max_vqs) + 1);
      if (err)
              goto err_reg;
 

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

author
#13 lore raw

Re: [PATCH 4/4] vdpa/mlx5: Fix tracking of current number of VQs

Quoted 45 lines · Jason Wang
Quoted 37 lines · Si-Wei Liu
Quoted 21 lines · Eli Cohen

Modify the code such that ndev->cur_num_vqs better reflects the actual number of data virtqueues. The value can be accurately realized after features have been negotiated.

This is to prevent possible failures when modifying the RQT object if the cur_num_vqs bears invalid value.

No issue was actually encountered but this also makes the code more readable.

Fixes c5a5cd3d3217 ("vdpa/mlx5: Support configuring max data virtqueue")
---
   drivers/vdpa/mlx5/net/mlx5_vnet.c | 12 ++++++++----
   1 file changed, 8 insertions(+), 4 deletions(-)
drivers/vdpa/mlx5/net/mlx5_vnet.c +0 -0
index 9eacfdb48434..b53603d94082 100644

drivers/vdpa/mlx5/net/mlx5_vnet.c static int create_rqt(struct mlx5_vdpa_net *ndev) · line 1249

Quoted 6 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -2
@@ -1246,8 +1246,7 @@ static int create_rqt(struct mlx5_vdpa_net *ndev)
       if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ)))
               num = 1;
       else
-             num = mlx5vdpa16_to_cpu(&ndev->mvdev,
-                                     ndev->config.max_virtqueue_pairs);
+             num = ndev->cur_num_vqs / 2;

Nit: the if branch can be consolidated

Quoted 5 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +0 -0
       max_rqt = min_t(int, roundup_pow_of_two(num),
                       1 << MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size));
@@ -1983,6 +1982,11 @@ static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
               return err;
 
Quoted 7 lines

drivers/vdpa/mlx5/net/mlx5_vnet.c static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) · line 1986

Quoted 3 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +2 -0
       ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features;
+     if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ))
+             ndev->cur_num_vqs = 2 * mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs);

Hmmm, not this patch, but there should've been validation done in the upper layer to guarantee set_featuers() for VIRTIO_NET_F_MQ always comes with VIRTIO_NET_F_CTRL_VQ. Maybe checking both: BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ)?

Quoted 2 lines

So the upper layer is unaware of the device type. It's better to do that in mlx5's set_features()

That'll be fine. I thought the upper layer can be made device type aware and consolidate it to common library routines avoiding duplicated code in every individual driver of the same type. If this is against the goal of making vdpa core device type agnostic, then it's perhaps not needed.

-Siwei

Quoted 26 lines

according to the spec:

The device MUST NOT offer a feature which requires another feature which was not offered.

Thanks

Quoted 19 lines

otherwise it looks good to me.

Quoted 16 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +4 -0
+     else
+             ndev->cur_num_vqs = 2;
+
       update_cvq_info(mvdev);
       return err;
   }
@@ -2233,6 +2237,7 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev)
       clear_vqs_ready(ndev);
       mlx5_vdpa_destroy_mr(&ndev->mvdev);
       ndev->mvdev.status = 0;
+     ndev->cur_num_vqs = 0;
       memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1));
       ndev->mvdev.actual_features = 0;
       ++mvdev->generation;
@@ -2641,9 +2646,8 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
 
Quoted 9 lines
Quoted 9 lines

drivers/vdpa/mlx5/net/mlx5_vnet.c static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, · line 2653

Quoted 9 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -2
       ndev->nb.notifier_call = event_handler;
       mlx5_notifier_register(mdev, &ndev->nb);
-     ndev->cur_num_vqs = 2 * mlx5_vdpa_max_qps(max_vqs);
       mvdev->vdev.mdev = &mgtdev->mgtdev;
-     err = _vdpa_register_device(&mvdev->vdev, ndev->cur_num_vqs + 1);
+     err = _vdpa_register_device(&mvdev->vdev, 2 * mlx5_vdpa_max_qps(max_vqs) + 1);
       if (err)
               goto err_reg;
 

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#14 lore raw

Re: [PATCH 4/4] vdpa/mlx5: Fix tracking of current number of VQs

Quoted 49 lines · Si-Wei Liu
Quoted 45 lines · Jason Wang
Quoted 44 lines · Si-Wei Liu
Quoted 21 lines · Eli Cohen

Modify the code such that ndev->cur_num_vqs better reflects the actual number of data virtqueues. The value can be accurately realized after features have been negotiated.

This is to prevent possible failures when modifying the RQT object if the cur_num_vqs bears invalid value.

No issue was actually encountered but this also makes the code more readable.

Fixes c5a5cd3d3217 ("vdpa/mlx5: Support configuring max data virtqueue")
---
   drivers/vdpa/mlx5/net/mlx5_vnet.c | 12 ++++++++----
   1 file changed, 8 insertions(+), 4 deletions(-)
drivers/vdpa/mlx5/net/mlx5_vnet.c +0 -0
index 9eacfdb48434..b53603d94082 100644

drivers/vdpa/mlx5/net/mlx5_vnet.c static int create_rqt(struct mlx5_vdpa_net *ndev) · line 1249

Quoted 6 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -2
@@ -1246,8 +1246,7 @@ static int create_rqt(struct mlx5_vdpa_net *ndev)
       if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ)))
               num = 1;
       else
-             num = mlx5vdpa16_to_cpu(&ndev->mvdev,
-                                     ndev->config.max_virtqueue_pairs);
+             num = ndev->cur_num_vqs / 2;

Nit: the if branch can be consolidated

drivers/vdpa/mlx5/net/mlx5_vnet.c static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) · line 1986

Quoted 8 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +2 -0
       max_rqt = min_t(int, roundup_pow_of_two(num),
                       1 << MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size));
@@ -1983,6 +1982,11 @@ static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
               return err;
 
       ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features;
+     if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ))
+             ndev->cur_num_vqs = 2 * mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs);

Hmmm, not this patch, but there should've been validation done in the upper layer to guarantee set_featuers() for VIRTIO_NET_F_MQ always comes with VIRTIO_NET_F_CTRL_VQ. Maybe checking both: BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ)?

Quoted 5 lines
Quoted 2 lines

So the upper layer is unaware of the device type. It's better to do that in mlx5's set_features()

That'll be fine. I thought the upper layer can be made device type aware and consolidate it to common library routines avoiding duplicated code in every individual driver of the same type.

This is possible but not implemented so far. It looks more like an intermediate layer between vdpa and it's parent.

Quoted 2 lines

If this is against the goal
of making vdpa core device type agnostic, then it's perhaps not needed.

Yes, that's the goal when developing vDPA core.

Thanks

Quoted 29 lines

-Siwei

Quoted 26 lines

according to the spec:

The device MUST NOT offer a feature which requires another feature which was not offered.

Thanks

Quoted 19 lines

otherwise it looks good to me.

Quoted 16 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +4 -0
+     else
+             ndev->cur_num_vqs = 2;
+
       update_cvq_info(mvdev);
       return err;
   }
@@ -2233,6 +2237,7 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev)
       clear_vqs_ready(ndev);
       mlx5_vdpa_destroy_mr(&ndev->mvdev);
       ndev->mvdev.status = 0;
+     ndev->cur_num_vqs = 0;
       memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1));
       ndev->mvdev.actual_features = 0;
       ++mvdev->generation;
@@ -2641,9 +2646,8 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
 
Quoted 10 lines
Quoted 9 lines
Quoted 9 lines

drivers/vdpa/mlx5/net/mlx5_vnet.c static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, · line 2653

Quoted 9 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -2
       ndev->nb.notifier_call = event_handler;
       mlx5_notifier_register(mdev, &ndev->nb);
-     ndev->cur_num_vqs = 2 * mlx5_vdpa_max_qps(max_vqs);
       mvdev->vdev.mdev = &mgtdev->mgtdev;
-     err = _vdpa_register_device(&mvdev->vdev, ndev->cur_num_vqs + 1);
+     err = _vdpa_register_device(&mvdev->vdev, 2 * mlx5_vdpa_max_qps(max_vqs) + 1);
       if (err)
               goto err_reg;
 

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

A message that is not in the archive (sent off-list or not yet received): [email protected]

author
#16 lore raw

Re: [PATCH 4/4] vdpa/mlx5: Fix tracking of current number of VQs

Quoted 43 lines · Eli Cohen
Quoted 31 lines · Si-Wei Liu
Quoted 21 lines · Eli Cohen

Modify the code such that ndev->cur_num_vqs better reflects the actual number of data virtqueues. The value can be accurately realized after features have been negotiated.

This is to prevent possible failures when modifying the RQT object if the cur_num_vqs bears invalid value.

No issue was actually encountered but this also makes the code more readable.

Fixes c5a5cd3d3217 ("vdpa/mlx5: Support configuring max data virtqueue")
---
   drivers/vdpa/mlx5/net/mlx5_vnet.c | 12 ++++++++----
   1 file changed, 8 insertions(+), 4 deletions(-)
drivers/vdpa/mlx5/net/mlx5_vnet.c +0 -0
index 9eacfdb48434..b53603d94082 100644

drivers/vdpa/mlx5/net/mlx5_vnet.c static int create_rqt(struct mlx5_vdpa_net *ndev) · line 1249

Quoted 6 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -2
@@ -1246,8 +1246,7 @@ static int create_rqt(struct mlx5_vdpa_net *ndev)
   	if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ)))
   		num = 1;
   	else
-		num = mlx5vdpa16_to_cpu(&ndev->mvdev,
-					ndev->config.max_virtqueue_pairs);
+		num = ndev->cur_num_vqs / 2;

Nit: the if branch can be consolidated

Quoted 11 lines

drivers/vdpa/mlx5/net/mlx5_vnet.c static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features) · line 1985

Quoted 7 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +2 -0
   	max_rqt = min_t(int, roundup_pow_of_two(num),
   			1 << MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size));
@@ -1983,6 +1982,11 @@ static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
   		return err;
   	ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features;
+	if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ))
+		ndev->cur_num_vqs = 2 * mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs);

Hmmm, not this patch, but there should've been validation done in the upper layer to guarantee set_featuers() for VIRTIO_NET_F_MQ always comes with VIRTIO_NET_F_CTRL_VQ. Maybe checking both: BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ)?

Quoted 3 lines

Not sure I follow. It's valid to have only CTRL_VQ feature but not MQ. In that case we should have only two data VQs but then cur_num_vqs would be set to whatever was configured by the user which could more than 2.

In that case (CTRL_VQ && !MQ), it will end up with just two data VQs (plus one control queue which is not accounted in 'ethtool -l') negotiated. There should be some validation in handle_ctrl_mq to disallow user from configuring more than 2 data VQs for that case. You don't need to worry too much for that at the moment, I'll post a patch  following your series to fix it. Let's keep it simple for just checking both BIT_ULL(VIRTIO_NET_F_MQ) | BIT_ULL(VIRTIO_NET_F_CTRL_VQ) together.

Thanks,
-Siwei

Quoted 19 lines
Quoted 19 lines

otherwise it looks good to me.

Quoted 15 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +4 -0
+	else
+		ndev->cur_num_vqs = 2;
+
   	update_cvq_info(mvdev);
   	return err;
   }
@@ -2233,6 +2237,7 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev)
   	clear_vqs_ready(ndev);
   	mlx5_vdpa_destroy_mr(&ndev->mvdev);
   	ndev->mvdev.status = 0;
+	ndev->cur_num_vqs = 0;
   	memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1));
   	ndev->mvdev.actual_features = 0;
   	++mvdev->generation;
@@ -2641,9 +2646,8 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
Quoted 8 lines
Quoted 8 lines

drivers/vdpa/mlx5/net/mlx5_vnet.c static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, · line 2651

Quoted 8 lines
drivers/vdpa/mlx5/net/mlx5_vnet.c +1 -2
   	ndev->nb.notifier_call = event_handler;
   	mlx5_notifier_register(mdev, &ndev->nb);
-	ndev->cur_num_vqs = 2 * mlx5_vdpa_max_qps(max_vqs);
   	mvdev->vdev.mdev = &mgtdev->mgtdev;
-	err = _vdpa_register_device(&mvdev->vdev, ndev->cur_num_vqs + 1);
+	err = _vdpa_register_device(&mvdev->vdev, 2 * mlx5_vdpa_max_qps(max_vqs) + 1);
   	if (err)
   		goto err_reg;

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

maintainer Reviewed-by Si-Wei Liu
#17 lore raw

Re: [PATCH 4/4] vdpa/mlx5: Fix tracking of current number of VQs

In reply to Si-Wei Liu
Quoted 1 line · Si-Wei Liu

Note: the correct format is:

(with a space before <>).

Pls take care in the future.
Thanks!

Signature · 6 lines
MST

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Acked-by
#18 lore raw

Re: [PATCH 4/4] vdpa/mlx5: Fix tracking of current number of VQs

In reply to a message not in the archive
Quoted 5 lines · Eli Cohen

Modify the code such that ndev->cur_num_vqs better reflects the actual number of data virtqueues. The value can be accurately realized after features have been negotiated.

Quoted 8 lines

This is to prevent possible failures when modifying the RQT object if the cur_num_vqs bears invalid value.

No issue was actually encountered but this also makes the code more readable.

Fixes c5a5cd3d3217 ("vdpa/mlx5: Support configuring max data virtqueue")
Quoted 50 lines
---
 drivers/vdpa/mlx5/net/mlx5_vnet.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)
drivers/vdpa/mlx5/net/mlx5_vnet.c +8 -5
index 9eacfdb48434..b53603d94082 100644
@@ -1246,8 +1246,7 @@ static int create_rqt(struct mlx5_vdpa_net *ndev)
        if (!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ)))
                num = 1;
        else
-               num = mlx5vdpa16_to_cpu(&ndev->mvdev,
-                                       ndev->config.max_virtqueue_pairs);
+               num = ndev->cur_num_vqs / 2;
 
        max_rqt = min_t(int, roundup_pow_of_two(num),
                        1 << MLX5_CAP_GEN(ndev->mvdev.mdev, log_max_rqt_size));
@@ -1983,6 +1982,11 @@ static int mlx5_vdpa_set_driver_features(struct vdpa_device *vdev, u64 features)
                return err;
 
        ndev->mvdev.actual_features = features & ndev->mvdev.mlx_features;
+       if (ndev->mvdev.actual_features & BIT_ULL(VIRTIO_NET_F_MQ))
+               ndev->cur_num_vqs = 2 * mlx5vdpa16_to_cpu(mvdev, ndev->config.max_virtqueue_pairs);
+       else
+               ndev->cur_num_vqs = 2;
+
        update_cvq_info(mvdev);
        return err;
 }
@@ -2233,6 +2237,7 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev)
        clear_vqs_ready(ndev);
        mlx5_vdpa_destroy_mr(&ndev->mvdev);
        ndev->mvdev.status = 0;
+       ndev->cur_num_vqs = 0;
        memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1));
        ndev->mvdev.actual_features = 0;
        ++mvdev->generation;
@@ -2641,9 +2646,8 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
 
        ndev->nb.notifier_call = event_handler;
        mlx5_notifier_register(mdev, &ndev->nb);
-       ndev->cur_num_vqs = 2 * mlx5_vdpa_max_qps(max_vqs);
        mvdev->vdev.mdev = &mgtdev->mgtdev;
-       err = _vdpa_register_device(&mvdev->vdev, ndev->cur_num_vqs + 1);
+       err = _vdpa_register_device(&mvdev->vdev, 2 * mlx5_vdpa_max_qps(max_vqs) + 1);
        if (err)
                goto err_reg;
 
--
Quoted 2 lines

2.34.1

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization