IB/mlx5: Use ib_dma APIs instead of open access to parent device
Subsystems (2)
PATCH 1/1 IB/mlx5: Use ib_dma APIs instead of open access to parent device
From: Parav Pandit <[email protected]>
DMA operation of the IB device is done using ib_device->dma_device. This is well abstracted using ib_dma APIs.
Hence, instead of doing open access to parent device, use IB core provided dma mapping APIs.
--- drivers/infiniband/hw/mlx5/mr.c | 40 +++++++++++++-------------------- 1 file changed, 15 insertions(+), 25 deletions(-)
drivers/infiniband/hw/mlx5/mr.c +15 -25
@@ -42,7 +42,7 @@#include "mlx5_ib.h" /* -* We can't use an array for xlt_emergency_page because dma_map_single doesn't +* We can't use an array for xlt_emergency_page because ib_dma_map_single doesn't * work on kernel modules memory */ void *xlt_emergency_page;
@@ -1081,7 +1081,6 @@ static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr, unsigned int flags) {struct mlx5_ib_dev *dev = mr->dev; -struct device *ddev = dev->ib_dev.dev.parent;dma_addr_t dma;void *xlt;
@@ -1089,8 +1088,8 @@ static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr, flags &MLX5_IB_UPD_XLT_ATOMIC ?GFP_ATOMIC : GFP_KERNEL); sg->length = nents * ent_size; - dma =dma_map_single (ddev, xlt, sg->length,DMA_TO_DEVICE ); -if (dma_mapping_error (ddev, dma)) { + dma =ib_dma_map_single (&dev->ib_dev, xlt, sg->length,DMA_TO_DEVICE ); +if (ib_dma_mapping_error (&dev->ib_dev, dma)) {mlx5_ib_err (dev,"unable to map DMA during XLT update.\n" );mlx5_ib_free_xlt (xlt, sg->length);return NULL ;
@@ -1118,9 +1117,7 @@ static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr,static void mlx5_ib_unmap_free_xlt (struct mlx5_ib_dev *dev,void *xlt,struct ib_sge *sg) { -struct device *ddev = dev->ib_dev.dev.parent; - -dma_unmap_single (ddev, sg->addr, sg->length,DMA_TO_DEVICE ); +ib_dma_unmap_single (&dev->ib_dev, sg->addr, sg->length,DMA_TO_DEVICE );mlx5_ib_free_xlt (xlt, sg->length); }
@@ -1143,7 +1140,6 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, int page_shift, int flags) {struct mlx5_ib_dev *dev = mr->dev; -struct device *ddev = dev->ib_dev.dev.parent;void *xlt;struct mlx5_umr_wr wr;struct ib_sge sg;
@@ -1195,11 +1191,9 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, pages_mapped += pages_iter, idx += pages_iter) { npages =min_t (int, pages_iter, pages_to_map - pages_mapped); size_to_map = npages * desc_size; -dma_sync_single_for_cpu (ddev, sg.addr, sg.length, -DMA_TO_DEVICE ); +ib_dma_sync_single_for_cpu (&dev->ib_dev, sg.addr, sg.length,DMA_TO_DEVICE );mlx5_odp_populate_xlt (xlt, idx, npages, mr, flags); -dma_sync_single_for_device (ddev, sg.addr, sg.length, -DMA_TO_DEVICE ); +ib_dma_sync_single_for_device (&dev->ib_dev, sg.addr, sg.length,DMA_TO_DEVICE ); sg.length =ALIGN (size_to_map,MLX5_UMR_MTT_ALIGNMENT );
@@ -1222,7 +1216,6 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,static int mlx5_ib_update_mr_pas (struct mlx5_ib_mr *mr,unsigned int flags) {struct mlx5_ib_dev *dev = mr->dev; -struct device *ddev = dev->ib_dev.dev.parent;struct ib_block_iter biter;struct mlx5_mtt *cur_mtt;struct mlx5_umr_wr wr;
@@ -1247,13 +1240,13 @@ static int mlx5_ib_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags)rdma_for_each_block (mr->umem->sg_head.sgl, &biter, mr->umem->nmap, BIT(mr->page_shift)) {if (cur_mtt == (void *)mtt + sg.length) { -dma_sync_single_for_device (ddev, sg.addr, sg.length, -DMA_TO_DEVICE ); +ib_dma_sync_single_for_device (&dev->ib_dev, sg.addr, sg.length, +DMA_TO_DEVICE ); err =mlx5_ib_post_send_wait (dev, &wr);if (err) goto err; -dma_sync_single_for_cpu (ddev, sg.addr, sg.length, -DMA_TO_DEVICE ); +ib_dma_sync_single_for_cpu (&dev->ib_dev, sg.addr, sg.length, +DMA_TO_DEVICE ); wr.offset += sg.length; cur_mtt = mtt; }
@@ -1270,7 +1263,7 @@ static int mlx5_ib_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags) wr.wr.send_flags |=xlt_wr_final_send_flags (flags); wr.xlt_size = sg.length; -dma_sync_single_for_device (ddev, sg.addr, sg.length,DMA_TO_DEVICE ); +ib_dma_sync_single_for_device (&dev->ib_dev, sg.addr, sg.length,DMA_TO_DEVICE ); err =mlx5_ib_post_send_wait (dev, &wr); err:
@@ -1763,12 +1756,10 @@ mlx5_alloc_priv_descs(struct ib_device *device, mr->descs =PTR_ALIGN (mr->descs_alloc,MLX5_UMR_ALIGN ); - mr->desc_map =dma_map_single (device->dev.parent, mr->descs, - size,DMA_TO_DEVICE ); -if (dma_mapping_error (device->dev.parent, mr->desc_map)) { - ret = -ENOMEM ; + mr->desc_map =ib_dma_map_single (device, mr->descs, size,DMA_TO_DEVICE ); + ret =ib_dma_mapping_error (device, mr->desc_map); +if (ret) goto err; - }return 0 ; err:
@@ -1784,8 +1775,7 @@ mlx5_free_priv_descs(struct mlx5_ib_mr *mr)struct ib_device *device = mr->ibmr.device;int size = mr->max_descs * mr->desc_size; -dma_unmap_single (device->dev.parent, mr->desc_map, - size,DMA_TO_DEVICE ); +ib_dma_unmap_single (device, mr->desc_map, size,DMA_TO_DEVICE );kfree (mr->descs_alloc); mr->descs =NULL ; }
Signature · 1 line
2.28.0
Jason Gunthorpe
maintainerQuoted 7 lines · Leon Romanovsky
From: Parav Pandit <[email protected]>
DMA operation of the IB device is done using
ib_device->dma_device. This is well abstracted usingib_dmaAPIs.Hence, instead of doing open access to parent device, use IB core provided dma mapping APIs.
Why?
The ib DMA APIs are for people using verbs, they are only needed to pack things into the ib_sge
If you are inside a driver, not using the verbs API, or not using ib_sge, then you should not be using the ib_dma API
It is an abberation, we should minimize its use.
drivers/infiniband/hw/mlx5/mr.c static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr, · line 1084
Quoted 11 lines
/*
- * We can't use an array forxlt_emergency_pagebecausedma_map_singledoesn't
+ * We can't use an array forxlt_emergency_pagebecauseib_dma_map_singledoesn't
* work on kernel modules memory
*/
void*xlt_emergency_page;drivers/infiniband/hw/mlx5/mr.c +0
-1@@ -1081,7 +1081,6 @@ static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr, unsigned int flags) {struct mlx5_ib_dev *dev = mr->dev; -struct device *ddev = dev->ib_dev.dev.parent;
Though this looks wrong, it should be dev->mdev->pdev.dev
ie it is always OK to use a PCI device with the normal DMA API
Jason
Quoted 14 lines
From: Jason Gunthorpe <[email protected]>
Sent: Monday, November 23, 2020 7:30 PMQuoted 7 lines · Leon Romanovsky
From: Parav Pandit <[email protected]>
DMA operation of the IB device is done using
ib_device->dma_device. This is well abstracted usingib_dmaAPIs.Hence, instead of doing open access to parent device, use IB core provided dma mapping APIs.
Quoted 8 lines
Why?
The ib DMA APIs are for people using verbs, they are only needed to pack things into the
ib_sgeIf you are inside a driver, not using the verbs API, or not using
ib_sge, then you should not be using theib_dmaAPI
Thanks for clarifying this. Using ib_dma apis make the code clear for dma device access clear and explicit.
Quoted 1 line
It is an abberation, we should minimize its use.
Alright. In that case will use the pci_dev as mlx5 driver internally has the knowledge of it and avoid using ib_dma APIs.
Quoted 11 lines
Review static void *mlx5_ib_create_xlt_wr(struct
Quoted 10 lines
/*
- * We can't use an array forxlt_emergency_pagebecausedma_map_singledoesn't
+ * We can't use an array forxlt_emergency_pagebecause
+ib_dma_map_singledoesn't
* work on kernel modules memory
*/
void*xlt_emergency_page;(quoted diff) +0
-0@@ -1081,7 +1081,6 @@ static void *mlx5_ib_create_xlt_wr(struct
mlx5_ib_mr*mr,
Quoted 9 lines
drivers/infiniband/hw/mlx5/mr.c static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr, · line 1084
Quoted 4 lines
drivers/infiniband/hw/mlx5/mr.c +0
-1unsigned int flags) {struct mlx5_ib_dev *dev = mr->dev; -struct device *ddev = dev->ib_dev.dev.parent;Though this looks wrong, it should be
dev->mdev->pdev.devie it is always OK to use a PCI device with the normal DMA API
Ok. will send v2 that uses pci device with existing DMA APIs.
Quoted 19 lines · Parav Pandit
Quoted 19 lines
From: Jason Gunthorpe <[email protected]>
Sent: Monday, November 23, 2020 7:30 PMQuoted 7 lines · Leon Romanovsky
From: Parav Pandit <[email protected]>
DMA operation of the IB device is done using
ib_device->dma_device. This is well abstracted usingib_dmaAPIs.Hence, instead of doing open access to parent device, use IB core provided dma mapping APIs.
Why?
The ib DMA APIs are for people using verbs, they are only needed to pack things into the
ib_sge
Quoted 7 lines
Quoted 3 lines
If you are inside a driver, not using the verbs API, or not using
ib_sge, then you should not be using theib_dmaAPIThanks for clarifying this. Using
ib_dmaapis make the code clear for dma device access clear and explicit.Quoted 1 line
It is an abberation, we should minimize its use.
Alright. In that case will use the
pci_devas mlx5 driver internally has the knowledge of it and avoid usingib_dmaAPIs.
Yeah, let's do v2, although I don't understand the purpose of ib_dma helpers if ib drivers can't use it.
Thanks
drivers/infiniband/hw/mlx5/mr.c line 45
Quoted 10 lines · Leon Romanovsky
drivers/infiniband/hw/mlx5/mr.c +1
-1@@ -42,7 +42,7 @@#include "mlx5_ib.h" /* -* We can't use an array for xlt_emergency_page because dma_map_single doesn't +* We can't use an array for xlt_emergency_page because ib_dma_map_single doesn't
Please avoid the pointlessly overly long line.
drivers/infiniband/hw/mlx5/mr.c int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, · line 1194
Quoted 1 line
drivers/infiniband/hw/mlx5/mr.c +1
-0+ib_dma_sync_single_for_cpu (&dev->ib_dev, sg.addr, sg.length,DMA_TO_DEVICE );
And here and much more.
Quoted 14 lines · Christoph Hellwig
drivers/infiniband/hw/mlx5/mr.c line 45
Quoted 10 lines · Leon Romanovsky
drivers/infiniband/hw/mlx5/mr.c +1
-1@@ -42,7 +42,7 @@#include "mlx5_ib.h" /* -* We can't use an array for xlt_emergency_page because dma_map_single doesn't +* We can't use an array for xlt_emergency_page because ib_dma_map_single doesn't Please avoid the pointlessly overly long line.
Quoted 3 lines
drivers/infiniband/hw/mlx5/mr.c int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, · line 1194
Quoted 1 line
drivers/infiniband/hw/mlx5/mr.c +1
-0+ib_dma_sync_single_for_cpu (&dev->ib_dev, sg.addr, sg.length,DMA_TO_DEVICE );And here and much more.
No problem, I will reduce checkpatch limit from its default.
Thanks
Quoted 1 line · Leon Romanovsky
No problem, I will reduce checkpatch limit from its default.
checkpatch unfortunately does not match what is documented in the codingstyle document and leads to these kinds of problems :(
Jason Gunthorpe
maintainerQuoted 23 lines · Leon Romanovsky
Quoted 22 lines · Parav Pandit
Quoted 22 lines
From: Jason Gunthorpe <[email protected]>
Sent: Monday, November 23, 2020 7:30 PMQuoted 7 lines · Leon Romanovsky
From: Parav Pandit <[email protected]>
DMA operation of the IB device is done using
ib_device->dma_device. This is well abstracted usingib_dmaAPIs.Hence, instead of doing open access to parent device, use IB core provided dma mapping APIs.
Why?
The ib DMA APIs are for people using verbs, they are only needed to pack things into the
ib_sgeIf you are inside a driver, not using the verbs API, or not using
ib_sge, then you should not be using theib_dmaAPI
Quoted 7 lines
Quoted 4 lines
Thanks for clarifying this. Using
ib_dmaapis make the code clear for dma device access clear and explicit.Quoted 1 line
It is an abberation, we should minimize its use.
Alright. In that case will use the
pci_devas mlx5 driver internally has the knowledge of it and avoid usingib_dmaAPIs.Yeah, let's do v2, although I don't understand the purpose of
ib_dmahelpers if ib drivers can't use it.
They are for ULPs, not drivers. Drivers are supposed to know how they are doing dma mapping already
Jason