Skip to contentSkip to conversation

IB/mlx5: Use ib_dma APIs instead of open access to parent device

PATCH In review Leon Romanovsky 4 participants 8 messages Nov 23, 2020 – Nov 24, 2020

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 IB/mlx5: Use ib_dma APIs instead of open access to parent device

From: Parav Pandit <[email protected]>

DMA operation of the IB device is done using ib_device->dma_device. This is well abstracted using ib_dma APIs.

Hence, instead of doing open access to parent device, use IB core provided dma mapping APIs.

---
 drivers/infiniband/hw/mlx5/mr.c | 40 +++++++++++++--------------------
 1 file changed, 15 insertions(+), 25 deletions(-)
drivers/infiniband/hw/mlx5/mr.c +15 -25
index 090e204ef1e1..d24ac339c053 100644
@@ -42,7 +42,7 @@
 #include "mlx5_ib.h"
 
 /*
- * We can't use an array for xlt_emergency_page because dma_map_single doesn't
+ * We can't use an array for xlt_emergency_page because ib_dma_map_single doesn't
  * work on kernel modules memory
  */
 void *xlt_emergency_page;
@@ -1081,7 +1081,6 @@ static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr,
 				   unsigned int flags)
 {
 	struct mlx5_ib_dev *dev = mr->dev;
-	struct device *ddev = dev->ib_dev.dev.parent;
 	dma_addr_t dma;
 	void *xlt;
 
@@ -1089,8 +1088,8 @@ static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr,
 				flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC :
 								 GFP_KERNEL);
 	sg->length = nents * ent_size;
-	dma = dma_map_single(ddev, xlt, sg->length, DMA_TO_DEVICE);
-	if (dma_mapping_error(ddev, dma)) {
+	dma = ib_dma_map_single(&dev->ib_dev, xlt, sg->length, DMA_TO_DEVICE);
+	if (ib_dma_mapping_error(&dev->ib_dev, dma)) {
 		mlx5_ib_err(dev, "unable to map DMA during XLT update.\n");
 		mlx5_ib_free_xlt(xlt, sg->length);
 		return NULL;
@@ -1118,9 +1117,7 @@ static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr,
 static void mlx5_ib_unmap_free_xlt(struct mlx5_ib_dev *dev, void *xlt,
 				   struct ib_sge *sg)
 {
-	struct device *ddev = dev->ib_dev.dev.parent;
-
-	dma_unmap_single(ddev, sg->addr, sg->length, DMA_TO_DEVICE);
+	ib_dma_unmap_single(&dev->ib_dev, sg->addr, sg->length, DMA_TO_DEVICE);
 	mlx5_ib_free_xlt(xlt, sg->length);
 }
 
@@ -1143,7 +1140,6 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
 		       int page_shift, int flags)
 {
 	struct mlx5_ib_dev *dev = mr->dev;
-	struct device *ddev = dev->ib_dev.dev.parent;
 	void *xlt;
 	struct mlx5_umr_wr wr;
 	struct ib_sge sg;
@@ -1195,11 +1191,9 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
 	     pages_mapped += pages_iter, idx += pages_iter) {
 		npages = min_t(int, pages_iter, pages_to_map - pages_mapped);
 		size_to_map = npages * desc_size;
-		dma_sync_single_for_cpu(ddev, sg.addr, sg.length,
-					DMA_TO_DEVICE);
+		ib_dma_sync_single_for_cpu(&dev->ib_dev, sg.addr, sg.length, DMA_TO_DEVICE);
 		mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags);
-		dma_sync_single_for_device(ddev, sg.addr, sg.length,
-					   DMA_TO_DEVICE);
+		ib_dma_sync_single_for_device(&dev->ib_dev, sg.addr, sg.length, DMA_TO_DEVICE);
 
 		sg.length = ALIGN(size_to_map, MLX5_UMR_MTT_ALIGNMENT);
 
@@ -1222,7 +1216,6 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
 static int mlx5_ib_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags)
 {
 	struct mlx5_ib_dev *dev = mr->dev;
-	struct device *ddev = dev->ib_dev.dev.parent;
 	struct ib_block_iter biter;
 	struct mlx5_mtt *cur_mtt;
 	struct mlx5_umr_wr wr;
@@ -1247,13 +1240,13 @@ static int mlx5_ib_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags)
 	rdma_for_each_block (mr->umem->sg_head.sgl, &biter, mr->umem->nmap,
 			     BIT(mr->page_shift)) {
 		if (cur_mtt == (void *)mtt + sg.length) {
-			dma_sync_single_for_device(ddev, sg.addr, sg.length,
-						   DMA_TO_DEVICE);
+			ib_dma_sync_single_for_device(&dev->ib_dev, sg.addr, sg.length,
+						      DMA_TO_DEVICE);
 			err = mlx5_ib_post_send_wait(dev, &wr);
 			if (err)
 				goto err;
-			dma_sync_single_for_cpu(ddev, sg.addr, sg.length,
-						DMA_TO_DEVICE);
+			ib_dma_sync_single_for_cpu(&dev->ib_dev, sg.addr, sg.length,
+						   DMA_TO_DEVICE);
 			wr.offset += sg.length;
 			cur_mtt = mtt;
 		}
@@ -1270,7 +1263,7 @@ static int mlx5_ib_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags)
 	wr.wr.send_flags |= xlt_wr_final_send_flags(flags);
 	wr.xlt_size = sg.length;
 
-	dma_sync_single_for_device(ddev, sg.addr, sg.length, DMA_TO_DEVICE);
+	ib_dma_sync_single_for_device(&dev->ib_dev, sg.addr, sg.length, DMA_TO_DEVICE);
 	err = mlx5_ib_post_send_wait(dev, &wr);
 
 err:
@@ -1763,12 +1756,10 @@ mlx5_alloc_priv_descs(struct ib_device *device,
 
 	mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN);
 
-	mr->desc_map = dma_map_single(device->dev.parent, mr->descs,
-				      size, DMA_TO_DEVICE);
-	if (dma_mapping_error(device->dev.parent, mr->desc_map)) {
-		ret = -ENOMEM;
+	mr->desc_map = ib_dma_map_single(device, mr->descs, size, DMA_TO_DEVICE);
+	ret = ib_dma_mapping_error(device, mr->desc_map);
+	if (ret)
 		goto err;
-	}
 
 	return 0;
 err:
@@ -1784,8 +1775,7 @@ mlx5_free_priv_descs(struct mlx5_ib_mr *mr)
 		struct ib_device *device = mr->ibmr.device;
 		int size = mr->max_descs * mr->desc_size;
 
-		dma_unmap_single(device->dev.parent, mr->desc_map,
-				 size, DMA_TO_DEVICE);
+		ib_dma_unmap_single(device, mr->desc_map, size, DMA_TO_DEVICE);
 		kfree(mr->descs_alloc);
 		mr->descs = NULL;
 	}
Signature · 1 line
2.28.0
maintainer
#1 lore raw
Quoted 7 lines · Leon Romanovsky

From: Parav Pandit <[email protected]>

DMA operation of the IB device is done using ib_device->dma_device. This is well abstracted using ib_dma APIs.

Hence, instead of doing open access to parent device, use IB core provided dma mapping APIs.

Why?

The ib DMA APIs are for people using verbs, they are only needed to pack things into the ib_sge

If you are inside a driver, not using the verbs API, or not using ib_sge, then you should not be using the ib_dma API

It is an abberation, we should minimize its use.

drivers/infiniband/hw/mlx5/mr.c static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr, · line 1084

Quoted 11 lines

/*
- * We can't use an array for xlt_emergency_page because dma_map_single doesn't
+ * We can't use an array for xlt_emergency_page because ib_dma_map_single doesn't
* work on kernel modules memory
*/
void *xlt_emergency_page;

drivers/infiniband/hw/mlx5/mr.c +0 -1
@@ -1081,7 +1081,6 @@ static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr,
 				   unsigned int flags)
 {
 	struct mlx5_ib_dev *dev = mr->dev;
-	struct device *ddev = dev->ib_dev.dev.parent;

Though this looks wrong, it should be dev->mdev->pdev.dev

ie it is always OK to use a PCI device with the normal DMA API

Jason

#2 lore raw
Quoted 14 lines

From: Jason Gunthorpe <[email protected]>
Sent: Monday, November 23, 2020 7:30 PM

Quoted 7 lines · Leon Romanovsky

From: Parav Pandit <[email protected]>

DMA operation of the IB device is done using ib_device->dma_device. This is well abstracted using ib_dma APIs.

Hence, instead of doing open access to parent device, use IB core provided dma mapping APIs.

Quoted 8 lines

Why?

The ib DMA APIs are for people using verbs, they are only needed to pack things into the ib_sge

If you are inside a driver, not using the verbs API, or not using ib_sge, then you should not be using the ib_dma API

Thanks for clarifying this. Using ib_dma apis make the code clear for dma device access clear and explicit.

Quoted 1 line

It is an abberation, we should minimize its use.

Alright. In that case will use the pci_dev as mlx5 driver internally has the knowledge of it and avoid using ib_dma APIs.

Quoted 11 lines

Review static void *mlx5_ib_create_xlt_wr(struct

Quoted 10 lines

/*
- * We can't use an array for xlt_emergency_page because dma_map_single doesn't
+ * We can't use an array for xlt_emergency_page because
+ ib_dma_map_single doesn't
* work on kernel modules memory
*/
void *xlt_emergency_page;

(quoted diff) +0 -0
@@ -1081,7 +1081,6 @@ static void *mlx5_ib_create_xlt_wr(struct

mlx5_ib_mr *mr,

Quoted 9 lines

drivers/infiniband/hw/mlx5/mr.c static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr, · line 1084

Quoted 4 lines
drivers/infiniband/hw/mlx5/mr.c +0 -1
 				   unsigned int flags)
 {
 	struct mlx5_ib_dev *dev = mr->dev;
-	struct device *ddev = dev->ib_dev.dev.parent;

Though this looks wrong, it should be dev->mdev->pdev.dev

ie it is always OK to use a PCI device with the normal DMA API

Ok. will send v2 that uses pci device with existing DMA APIs.

#3 lore raw
Quoted 19 lines · Parav Pandit
Quoted 19 lines

From: Jason Gunthorpe <[email protected]>
Sent: Monday, November 23, 2020 7:30 PM

Quoted 7 lines · Leon Romanovsky

From: Parav Pandit <[email protected]>

DMA operation of the IB device is done using ib_device->dma_device. This is well abstracted using ib_dma APIs.

Hence, instead of doing open access to parent device, use IB core provided dma mapping APIs.

Why?

The ib DMA APIs are for people using verbs, they are only needed to pack things into the ib_sge

Quoted 7 lines
Quoted 3 lines

If you are inside a driver, not using the verbs API, or not using ib_sge, then you should not be using the ib_dma API

Thanks for clarifying this. Using ib_dma apis make the code clear for dma device access clear and explicit.

Quoted 1 line

It is an abberation, we should minimize its use.

Alright. In that case will use the pci_dev as mlx5 driver internally has the knowledge of it and avoid using ib_dma APIs.

Yeah, let's do v2, although I don't understand the purpose of ib_dma helpers if ib drivers can't use it.

Thanks

#5 lore raw
In reply to Leon Romanovsky

drivers/infiniband/hw/mlx5/mr.c line 45

Quoted 10 lines · Leon Romanovsky
drivers/infiniband/hw/mlx5/mr.c +1 -1
index 090e204ef1e1..d24ac339c053 100644
@@ -42,7 +42,7 @@
 #include "mlx5_ib.h"
 
 /*
- * We can't use an array for xlt_emergency_page because dma_map_single doesn't
+ * We can't use an array for xlt_emergency_page because ib_dma_map_single doesn't

Please avoid the pointlessly overly long line.

drivers/infiniband/hw/mlx5/mr.c int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, · line 1194

Quoted 1 line
drivers/infiniband/hw/mlx5/mr.c +1 -0
+		ib_dma_sync_single_for_cpu(&dev->ib_dev, sg.addr, sg.length, DMA_TO_DEVICE);

And here and much more.

#6 lore raw
Quoted 14 lines · Christoph Hellwig

drivers/infiniband/hw/mlx5/mr.c line 45

Quoted 10 lines · Leon Romanovsky
drivers/infiniband/hw/mlx5/mr.c +1 -1
index 090e204ef1e1..d24ac339c053 100644
@@ -42,7 +42,7 @@
 #include "mlx5_ib.h"
 
 /*
- * We can't use an array for xlt_emergency_page because dma_map_single doesn't
+ * We can't use an array for xlt_emergency_page because ib_dma_map_single doesn't

Please avoid the pointlessly overly long line.

Quoted 3 lines

drivers/infiniband/hw/mlx5/mr.c int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, · line 1194

Quoted 1 line
drivers/infiniband/hw/mlx5/mr.c +1 -0
+		ib_dma_sync_single_for_cpu(&dev->ib_dev, sg.addr, sg.length, DMA_TO_DEVICE);

And here and much more.

No problem, I will reduce checkpatch limit from its default.

Thanks

#7 lore raw
Quoted 1 line · Leon Romanovsky

No problem, I will reduce checkpatch limit from its default.

checkpatch unfortunately does not match what is documented in the codingstyle document and leads to these kinds of problems :(

maintainer
#4 lore raw
In reply to Leon Romanovsky
Quoted 23 lines · Leon Romanovsky
Quoted 22 lines · Parav Pandit
Quoted 22 lines

From: Jason Gunthorpe <[email protected]>
Sent: Monday, November 23, 2020 7:30 PM

Quoted 7 lines · Leon Romanovsky

From: Parav Pandit <[email protected]>

DMA operation of the IB device is done using ib_device->dma_device. This is well abstracted using ib_dma APIs.

Hence, instead of doing open access to parent device, use IB core provided dma mapping APIs.

Why?

The ib DMA APIs are for people using verbs, they are only needed to pack things into the ib_sge

If you are inside a driver, not using the verbs API, or not using ib_sge, then you should not be using the ib_dma API

Quoted 7 lines
Quoted 4 lines

Thanks for clarifying this. Using ib_dma apis make the code clear for dma device access clear and explicit.

Quoted 1 line

It is an abberation, we should minimize its use.

Alright. In that case will use the pci_dev as mlx5 driver internally has the knowledge of it and avoid using ib_dma APIs.

Yeah, let's do v2, although I don't understand the purpose of ib_dma helpers if ib drivers can't use it.

They are for ULPs, not drivers. Drivers are supposed to know how they are doing dma mapping already

Jason