VDPA mlx5 fixes

In cases where the vdpa instance uses a SF (sub function), the DMA device is the parent device. Use a function to retrieve the correct DMA device.

Fixes 1958fc2f0712 ("net/mlx5: SF, Add auxiliary device driver")

Signed-off-by Eli Cohen <[email protected]>

Reviewed-by Parav Pandit <[email protected]>

---
  drivers/vdpa/mlx5/core/mr.c | 9 +++++++--
  1 file changed, 7 insertions(+), 2 deletions(-)

drivers/vdpa/mlx5/core/mr.c +7 -2

index d300f799efcd..3908ff28eec0 100644
@@ -219,6 +219,11 @@ static void destroy_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_m
  	mlx5_vdpa_destroy_mkey(mvdev, &mkey->mkey);
  }
 
+static struct device *get_dma_device(struct mlx5_vdpa_dev *mvdev)
+{
+	return &mvdev->mdev->pdev->dev;
+}
+
  static int map_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr,
  			 struct vhost_iotlb *iotlb)
  {

@@ -234,7 +239,7 @@ static int map_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr
  	u64 pa;
  	u64 paend;
  	struct scatterlist *sg;
-	struct device *dma = mvdev->mdev->device;
+	struct device *dma = get_dma_device(mvdev);
 
  	for (map = vhost_iotlb_itree_first(iotlb, mr->start, mr->end - 1);
  	     map; map = vhost_iotlb_itree_next(map, start, mr->end - 1)) {

@@ -291,7 +296,7 @@ static int map_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr
 
  static void unmap_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)
  {
-	struct device *dma = mvdev->mdev->device;
+	struct device *dma = get_dma_device(mvdev);
 
  	destroy_direct_mr(mvdev, mr);
  	dma_unmap_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0);

struct mlx5_core_dev has a bar_addr field that contains the correct bar address for the function regardless of whether it is pci function or sub function. Use it.

Fixes 1958fc2f0712 ("net/mlx5: SF, Add auxiliary device driver")

Signed-off-by Eli Cohen <[email protected]>

Reviewed-by Parav Pandit <[email protected]>

---
  drivers/vdpa/mlx5/core/resources.c | 3 ++-
  1 file changed, 2 insertions(+), 1 deletion(-)

drivers/vdpa/mlx5/core/resources.c +2 -1

index 96e6421c5d1c..6521cbd0f5c2 100644
@@ -246,7 +246,8 @@ int mlx5_vdpa_alloc_resources(struct mlx5_vdpa_dev *mvdev)
  	if (err)
  		goto err_key;
 
-	kick_addr = pci_resource_start(mdev->pdev, 0) + offset;
+	kick_addr = mdev->bar_addr + offset;
+
  	res->kick_addr = ioremap(kick_addr, PAGE_SIZE);
  	if (!res->kick_addr) {
  		err = -ENOMEM;

VIRTIO_F_VERSION_1 is a bit number. Use BIT_ULL() with mask conditionals.

Also, in mlx5_vdpa_is_little_endian() use BIT_ULL for consistency with the rest of the code.

Fixes 1a86b377aa21 ("vdpa/mlx5: Add VDPA driver for supported mlx5 devices")

Signed-off-by Eli Cohen <[email protected]>

---
  drivers/vdpa/mlx5/net/mlx5_vnet.c | 4 ++--
  1 file changed, 2 insertions(+), 2 deletions(-)

drivers/vdpa/mlx5/net/mlx5_vnet.c +2 -2

index a49ebb250253..6fe61fc57790 100644
@@ -820,7 +820,7 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque
  	MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn);
  	MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent);
  	MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0,
-		 !!(ndev->mvdev.actual_features & VIRTIO_F_VERSION_1));
+		 !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1)));
  	MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr);
  	MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr);
  	MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr);

@@ -1554,7 +1554,7 @@ static void clear_virtqueues(struct mlx5_vdpa_net *ndev)
  static inline bool mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev *mvdev)
  {
  	return virtio_legacy_is_little_endian() ||
-		(mvdev->actual_features & (1ULL << VIRTIO_F_VERSION_1));
+		(mvdev->actual_features & BIT_ULL(VIRTIO_F_VERSION_1));
  }
 
  static __virtio16 cpu_to_mlx5vdpa16(struct mlx5_vdpa_dev *mvdev, u16 val)

When we suspend the VM, the VDPA interface will be reset. When the VM is resumed again, clear_virtqueues() will clear the available and used indices resulting in hardware virqtqueue objects becoming out of sync. We can avoid this function alltogether since qemu will clear them if required, e.g. when the VM went through a reboot.

Moreover, since the hw available and used indices should always be identical on query and should be restored to the same value same value for virtqueues that complete in order, we set the single value provided by set_vq_state(). In get_vq_state() we return the value of hardware used index.

Fixes b35ccebe3ef7 ("vdpa/mlx5: Restore the hardware used index after change map")

Fixes 1a86b377aa21 ("vdpa/mlx5: Add VDPA driver for supported mlx5 devices")

Signed-off-by Eli Cohen <[email protected]>

---

  drivers/vdpa/mlx5/net/mlx5_vnet.c | 21 ++++++++-------------
  1 file changed, 8 insertions(+), 13 deletions(-)

drivers/vdpa/mlx5/net/mlx5_vnet.c +8 -13

index 6fe61fc57790..4d2809c7d4e3 100644
@@ -1169,6 +1169,7 @@ static void suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *m
  		return;
  	}
  	mvq->avail_idx = attr.available_index;
+	mvq->used_idx = attr.used_index;
  }
 
  static void suspend_vqs(struct mlx5_vdpa_net *ndev)

@@ -1426,6 +1427,7 @@ static int mlx5_vdpa_set_vq_state(struct vdpa_device *vdev, u16 idx,
  		return -EINVAL;
  	}
 
+	mvq->used_idx = state->avail_index;
  	mvq->avail_idx = state->avail_index;
  	return 0;
  }

@@ -1443,7 +1445,11 @@ static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa
  	 * that cares about emulating the index after vq is stopped.
  	 */
  	if (!mvq->initialized) {
-		state->avail_index = mvq->avail_idx;
+		/* Firmware returns a wrong value for the available index.
+		 * Since both values should be identical, we take the value of
+		 * used_idx which is reported correctly.
+		 */
+		state->avail_index = mvq->used_idx;
  		return 0;
  	}
 

@@ -1452,7 +1458,7 @@ static int mlx5_vdpa_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa
  		mlx5_vdpa_warn(mvdev, "failed to query virtqueue\n");
  		return err;
  	}
-	state->avail_index = attr.available_index;
+	state->avail_index = attr.used_index;
  	return 0;
  }
 

@@ -1540,16 +1546,6 @@ static void teardown_virtqueues(struct mlx5_vdpa_net *ndev)
  	}
  }
 
-static void clear_virtqueues(struct mlx5_vdpa_net *ndev)
-{
-	int i;
-
-	for (i = ndev->mvdev.max_vqs - 1; i >= 0; i--) {
-		ndev->vqs[i].avail_idx = 0;
-		ndev->vqs[i].used_idx = 0;
-	}
-}
-
  /* TODO: cross-endian support */
  static inline bool mlx5_vdpa_is_little_endian(struct mlx5_vdpa_dev *mvdev)
  {

@@ -1785,7 +1781,6 @@ static void mlx5_vdpa_set_status(struct vdpa_device *vdev, u8 status)
  	if (!status) {
  		mlx5_vdpa_info(mvdev, "performing device reset\n");
  		teardown_driver(ndev);
-		clear_virtqueues(ndev);
  		mlx5_vdpa_destroy_mr(&ndev->mvdev);
  		ndev->mvdev.status = 0;
  		ndev->mvdev.mlx_features = 0;

Hi Michael,

The following series contains fixes to mlx5 vdpa driver. Included first is Siwei's fix to queried MTU was already reviewed a while ago and is not in your tree.

Patches 2 and 3 are required to allow mlx5_vdpa run on sub functions.

This series contains patches that were included in Parav's series
http://lists.infradead.org/pipermail/linux-mtd/2016-January/064878.html
but that series will be sent again at a later time.

Eli Cohen (4):
vdpa/mlx5: Use the correct dma device when registering memory vdpa/mlx5: Retrieve BAR address suitable any function vdpa/mlx5: Fix wrong use of bit numbers
vdpa/mlx5: Fix suspend/resume index restoration

Si-Wei Liu (1):
vdpa/mlx5: should exclude header length and fcs from mtu

 drivers/vdpa/mlx5/core/mlx5_vdpa.h |  4 +++
 drivers/vdpa/mlx5/core/mr.c        |  9 +++++--
 drivers/vdpa/mlx5/core/resources.c |  3 ++-
 drivers/vdpa/mlx5/net/mlx5_vnet.c  | 40 ++++++++++++++++++------------
 4 files changed, 37 insertions(+), 19 deletions(-)

Signature · 1 line

2.30.1