RDMA/erdma: Add DMA-BUF memory registration
Subsystems (2)
v3 cover RDMA/erdma: Add DMA-BUF memory registration
Hi,
This patch series adds DMA-BUF memory registration support to the erdma driver:
- #1 renames get/put_mtt_entries to erdma_mem_init/uninit to better reflect their purpose of initializing the struct erdma_mem.
- #2 introduces struct erdma_mem_init_attr to pass parameters to erdma_mem_init(), improving code maintainability and preparing for DMA-BUF support.
- #3 implements erdma_reg_user_mr_dmabuf() to enable DMA-BUF based user memory registration using the revocable pinned dmabuf interface.
One known limitation: if the DEREG_MR command fails in the revoke callback, the driver can only log an error, since erdma does not support a function-level reset like irdma's request_reset() yet. We plan to introduce such a mechanism in future work.
Thanks,
Boshi Yu
---
v3:
- Patch#3: Switch to the revocable pinned dmabuf interface as suggested by Jason Gunthorpe.
v2:
- Patch#2: Add validation for the return value of ib_umem_find_best_pgsz().
v1:
Boshi Yu (3):
RDMA/erdma: Rename get/put_mtt_entries to erdma_mem_init/uninit RDMA/erdma: Introduce struct erdma_mem_init_attr RDMA/erdma: Implement erdma_reg_user_mr_dmabuf
drivers/infiniband/hw/erdma/erdma_main.c | 1 + drivers/infiniband/hw/erdma/erdma_verbs.c | 262 +++++++++++++++------- drivers/infiniband/hw/erdma/erdma_verbs.h | 17 ++ 3 files changed, 201 insertions(+), 79 deletions(-)
Signature · 1 line
2.46.0
PATCH v3 1/3 RDMA/erdma: Rename get/put_mtt_entries to erdma_mem_init/uninit
The get_mtt_entries() interface actually initializes the struct erdma_mem. Rename the get_mtt_entries/put_mtt_entries to erdma_mem_init/erdma_mem_uninit, respectively.
--- drivers/infiniband/hw/erdma/erdma_verbs.c | 64 +++++++++++------------ 1 file changed, 32 insertions(+), 32 deletions(-)
drivers/infiniband/hw/erdma/erdma_verbs.c +32 -32
@@ -828,9 +828,9 @@ static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt) } } -static int get_mtt_entries (struct erdma_dev *dev,struct erdma_mem *mem, -u64 start,u64 len,int access,u64 virt, -unsigned long req_page_size,bool force_continuous) +static int erdma_mem_init (struct erdma_dev *dev,struct erdma_mem *mem, +u64 start,u64 len,int access,u64 virt, +unsigned long req_page_size,bool force_continuous) {int ret =0 ;
@@ -867,7 +867,7 @@ static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,return ret; } -static void put_mtt_entries (struct erdma_dev *dev,struct erdma_mem *mem) +static void erdma_mem_uninit (struct erdma_dev *dev,struct erdma_mem *mem) {if (mem->mtt)erdma_destroy_mtt (dev, mem->mtt);
@@ -951,45 +951,45 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, qp->attrs.rq_size * RQE_SIZE))return -EINVAL ; - ret =get_mtt_entries (qp->dev, &qp->user_qp.sq_mem, va, - qp->attrs.sq_size <<SQEBB_SHIFT ,0 , va, - (SZ_1M -SZ_4K ), true); + ret =erdma_mem_init (qp->dev, &qp->user_qp.sq_mem, va, + qp->attrs.sq_size <<SQEBB_SHIFT ,0 , va, + (SZ_1M -SZ_4K ), true);if (ret)return ret; rq_offset =ALIGN (qp->attrs.sq_size <<SQEBB_SHIFT ,ERDMA_HW_PAGE_SIZE ); qp->user_qp.rq_offset = rq_offset; - ret =get_mtt_entries (qp->dev, &qp->user_qp.rq_mem, va + rq_offset, - qp->attrs.rq_size <<RQE_SHIFT ,0 , va + rq_offset, - (SZ_1M -SZ_4K ), true); + ret =erdma_mem_init (qp->dev, &qp->user_qp.rq_mem, va + rq_offset, + qp->attrs.rq_size <<RQE_SHIFT ,0 , va + rq_offset, + (SZ_1M -SZ_4K ), true);if (ret) - goto put_sq_mtt; + goto uninit_sq_mem; ret =erdma_map_user_dbrecords (uctx, dbrec_va, &qp->user_qp.user_dbr_page, &dbrec_dma);if (ret) - goto put_rq_mtt; + goto uninit_rq_mem; qp->user_qp.sq_dbrec_dma = dbrec_dma; qp->user_qp.rq_dbrec_dma = dbrec_dma +ERDMA_DB_SIZE ;return 0 ; -put_rq_mtt: -put_mtt_entries (qp->dev, &qp->user_qp.rq_mem); +uninit_rq_mem: +erdma_mem_uninit (qp->dev, &qp->user_qp.rq_mem); -put_sq_mtt: -put_mtt_entries (qp->dev, &qp->user_qp.sq_mem); +uninit_sq_mem: +erdma_mem_uninit (qp->dev, &qp->user_qp.sq_mem);return ret; }static void free_user_qp (struct erdma_qp *qp,struct erdma_ucontext *uctx) { -put_mtt_entries (qp->dev, &qp->user_qp.sq_mem); -put_mtt_entries (qp->dev, &qp->user_qp.rq_mem); +erdma_mem_uninit (qp->dev, &qp->user_qp.sq_mem); +erdma_mem_uninit (qp->dev, &qp->user_qp.rq_mem);erdma_unmap_user_dbrecords (uctx, &qp->user_qp.user_dbr_page); }
@@ -1251,14 +1251,14 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,if (!mr)return ERR_PTR (-ENOMEM ); - ret =get_mtt_entries (dev, &mr->mem, start, len, access, virt, -SZ_2G -SZ_4K , false); + ret =erdma_mem_init (dev, &mr->mem, start, len, access, virt, +SZ_2G -SZ_4K , false);if (ret) goto err_out_free; ret =erdma_create_stag (dev, &stag);if (ret) - goto err_out_put_mtt; + goto err_uninit_mem; mr->ibmr.lkey = mr->ibmr.rkey = stag; mr->ibmr.pd = ibpd;
@@ -1278,8 +1278,8 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,erdma_free_idx (&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX ], mr->ibmr.lkey >>8 ); -err_out_put_mtt: -put_mtt_entries (dev, &mr->mem); +err_uninit_mem: +erdma_mem_uninit (dev, &mr->mem); err_out_free:kfree (mr);
@@ -1309,7 +1309,7 @@ int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)erdma_free_idx (&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX ], ibmr->lkey >>8 ); -put_mtt_entries (dev, &mr->mem); +erdma_mem_uninit (dev, &mr->mem);kfree (mr);return 0 ;
@@ -1340,7 +1340,7 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) cq->kern_cq.dbrec_dma); }else {erdma_unmap_user_dbrecords (ctx, &cq->user_cq.user_dbr_page); -put_mtt_entries (dev, &cq->user_cq.qbuf_mem); +erdma_mem_uninit (dev, &cq->user_cq.qbuf_mem); }xa_erase (&dev->cq_xa, cq->cqn);
@@ -1387,8 +1387,8 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)if (rdma_is_kernel_res (&qp->ibqp.res)) {free_kernel_qp (qp); }else { -put_mtt_entries (dev, &qp->user_qp.sq_mem); -put_mtt_entries (dev, &qp->user_qp.rq_mem); +erdma_mem_uninit (dev, &qp->user_qp.sq_mem); +erdma_mem_uninit (dev, &qp->user_qp.rq_mem);erdma_unmap_user_dbrecords (ctx, &qp->user_qp.user_dbr_page); }
@@ -1910,9 +1910,9 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,int ret;struct erdma_dev *dev =to_edev (cq->ibcq.device); - ret =get_mtt_entries (dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va, - ureq->qbuf_len,0 , ureq->qbuf_va,SZ_64M -SZ_4K , - true); + ret =erdma_mem_init (dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va, + ureq->qbuf_len,0 , ureq->qbuf_va,SZ_64M -SZ_4K , + true);if (ret)return ret;
@@ -1920,7 +1920,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, &cq->user_cq.user_dbr_page, &cq->user_cq.dbrec_dma);if (ret) -put_mtt_entries (dev, &cq->user_cq.qbuf_mem); +erdma_mem_uninit (dev, &cq->user_cq.qbuf_mem);return ret; }
@@ -2011,7 +2011,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, err_free_res:if (!rdma_is_kernel_res (&ibcq->res)) {erdma_unmap_user_dbrecords (ctx, &cq->user_cq.user_dbr_page); -put_mtt_entries (dev, &cq->user_cq.qbuf_mem); +erdma_mem_uninit (dev, &cq->user_cq.qbuf_mem); }else {dma_free_coherent (&dev->pdev->dev, depth <<CQE_SHIFT , cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
Signature · 1 line
2.46.0
PATCH v3 2/3 RDMA/erdma: Introduce struct erdma_mem_init_attr
In reply to Boshi YuPass struct erdma_mem_init_attr to the erdma_mem_init() interface. Additionally, validate the return value of ib_umem_find_best_pgsz().
--- drivers/infiniband/hw/erdma/erdma_verbs.c | 65 +++++++++++++++-------- drivers/infiniband/hw/erdma/erdma_verbs.h | 13 +++++ 2 files changed, 55 insertions(+), 23 deletions(-)
drivers/infiniband/hw/erdma/erdma_verbs.c +42 -23
@@ -829,26 +829,33 @@ static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt) }static int erdma_mem_init (struct erdma_dev *dev,struct erdma_mem *mem, -u64 start,u64 len,int access,u64 virt, -unsigned long req_page_size,bool force_continuous) +struct erdma_mem_init_attr *attr) {int ret =0 ; - mem->umem =ib_umem_get_va (&dev->ibdev, start, len, access); + mem->umem = +ib_umem_get_va (&dev->ibdev, attr->start, attr->len, attr->access);if (IS_ERR (mem->umem)) { ret =PTR_ERR (mem->umem); mem->umem =NULL ;return ret; } - mem->va = virt; - mem->len = len; - mem->page_size =ib_umem_find_best_pgsz (mem->umem, req_page_size, virt); - mem->page_offset = start & (mem->page_size -1 ); + mem->va = attr->virt; + mem->len = attr->len; + + mem->page_size =ib_umem_find_best_pgsz (mem->umem, attr->req_page_size, + attr->virt); +if (!mem->page_size) { + ret = -EINVAL ; + goto error_ret; + } + + mem->page_offset = attr->start & (mem->page_size -1 ); mem->mtt_nents =ib_umem_num_dma_blocks (mem->umem, mem->page_size); mem->page_cnt = mem->mtt_nents; mem->mtt =erdma_create_mtt (dev,MTT_SIZE (mem->page_cnt), - force_continuous); + attr->flags &ERDMA_MEM_FLAG_MTT_PHYS_CONT );if (IS_ERR (mem->mtt)) { ret =PTR_ERR (mem->mtt); goto error_ret;
@@ -943,6 +950,7 @@ erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,static int init_user_qp (struct erdma_qp *qp,struct erdma_ucontext *uctx,u64 va,u32 len,u64 dbrec_va) { +struct erdma_mem_init_attr attr = {};dma_addr_t dbrec_dma;u32 rq_offset;int ret;
@@ -951,18 +959,22 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, qp->attrs.rq_size * RQE_SIZE))return -EINVAL ; - ret =erdma_mem_init (qp->dev, &qp->user_qp.sq_mem, va, - qp->attrs.sq_size <<SQEBB_SHIFT ,0 , va, - (SZ_1M -SZ_4K ), true); + attr.virt = va; + attr.start = va; + attr.req_page_size =SZ_1M -SZ_4K ; + attr.len = qp->attrs.sq_size <<SQEBB_SHIFT ; + attr.flags =ERDMA_MEM_FLAG_MTT_PHYS_CONT ; + ret =erdma_mem_init (qp->dev, &qp->user_qp.sq_mem, &attr);if (ret)return ret; rq_offset =ALIGN (qp->attrs.sq_size <<SQEBB_SHIFT ,ERDMA_HW_PAGE_SIZE ); qp->user_qp.rq_offset = rq_offset; - ret =erdma_mem_init (qp->dev, &qp->user_qp.rq_mem, va + rq_offset, - qp->attrs.rq_size <<RQE_SHIFT ,0 , va + rq_offset, - (SZ_1M -SZ_4K ), true); + attr.virt = va + rq_offset; + attr.start = va + rq_offset; + attr.len = qp->attrs.rq_size <<RQE_SHIFT ; + ret =erdma_mem_init (qp->dev, &qp->user_qp.rq_mem, &attr);if (ret) goto uninit_sq_mem;
@@ -1236,8 +1248,9 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, u64 virt, int access, struct ib_dmah *dmah, struct ib_udata *udata) { - struct erdma_mr *mr = NULL; struct erdma_dev *dev = to_edev(ibpd->device); +struct erdma_mem_init_attr attr = {}; +struct erdma_mr *mr =NULL ;u32 stag;int ret;
@@ -1251,8 +1264,12 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,if (!mr)return ERR_PTR (-ENOMEM ); - ret =erdma_mem_init (dev, &mr->mem, start, len, access, virt, -SZ_2G -SZ_4K , false); + attr.len = len; + attr.virt = virt; + attr.start = start; + attr.access = access; + attr.req_page_size =SZ_2G -SZ_4K ; + ret =erdma_mem_init (dev, &mr->mem, &attr);if (ret) goto err_out_free;
@@ -1262,8 +1279,6 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, mr->ibmr.lkey = mr->ibmr.rkey = stag; mr->ibmr.pd = ibpd; - mr->mem.va = virt; - mr->mem.len = len; mr->access =ERDMA_MR_ACC_LR |to_erdma_access_flags (access); mr->valid =1 ; mr->type =ERDMA_MR_TYPE_NORMAL ;
@@ -1907,12 +1922,16 @@ int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,static int erdma_init_user_cq (struct erdma_ucontext *ctx,struct erdma_cq *cq,struct erdma_ureq_create_cq *ureq) { -int ret;struct erdma_dev *dev =to_edev (cq->ibcq.device); +struct erdma_mem_init_attr attr = {}; +int ret; - ret =erdma_mem_init (dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va, - ureq->qbuf_len,0 , ureq->qbuf_va,SZ_64M -SZ_4K , - true); + attr.len = ureq->qbuf_len; + attr.virt = ureq->qbuf_va; + attr.start = ureq->qbuf_va; + attr.req_page_size =SZ_64M -SZ_4K ; + attr.flags =ERDMA_MEM_FLAG_MTT_PHYS_CONT ; + ret =erdma_mem_init (dev, &cq->user_cq.qbuf_mem, &attr);if (ret)return ret;
drivers/infiniband/hw/erdma/erdma_verbs.h +13 -0
@@ -108,6 +108,19 @@ struct erdma_mtt {struct erdma_mtt *low_level; }; +enum erdma_mem_flags { +ERDMA_MEM_FLAG_MTT_PHYS_CONT = (1 <<0 ), +}; + +struct erdma_mem_init_attr { +u64 start; +u64 virt; +u64 len; +unsigned long req_page_size; +int access; +u32 flags; +}; +struct erdma_mem {struct ib_umem *umem;struct erdma_mtt *mtt;
Signature · 1 line
2.46.0
PATCH v3 3/3 RDMA/erdma: Implement erdma_reg_user_mr_dmabuf
In reply to Boshi YuSupport DMA-BUF based user memory registration using the pinned revocable dmabuf interface, so that erdma can import dmabufs from exporters that may require revocation, such as VFIO.
On revocation, the revoke callback destroys the MPT entry in hardware to stop the device from accessing the revoked memory, while the stag index is reserved until the MR is fully deregistered, ensuring the key cannot be obtained by a new region in the meantime.
Refactor the existing erdma_reg_user_mr() into a common _erdma_reg_user_mr() function that handles both regular and DMA-BUF memory registration.
--- drivers/infiniband/hw/erdma/erdma_main.c | 1 + drivers/infiniband/hw/erdma/erdma_verbs.c | 189 ++++++++++++++++------ drivers/infiniband/hw/erdma/erdma_verbs.h | 4 + 3 files changed, 142 insertions(+), 52 deletions(-)
drivers/infiniband/hw/erdma/erdma_main.c +1 -0
@@ -526,6 +526,7 @@ static const struct ib_device_ops erdma_device_ops = { .query_qp = erdma_query_qp, .req_notify_cq = erdma_req_notify_cq, .reg_user_mr = erdma_reg_user_mr, + .reg_user_mr_dmabuf = erdma_reg_user_mr_dmabuf, .modify_qp = erdma_modify_qp, INIT_RDMA_OBJ_SIZE(ib_cq, erdma_cq, ibcq),
drivers/infiniband/hw/erdma/erdma_verbs.c +137 -52
268 lines not on this page · raw message
drivers/infiniband/hw/erdma/erdma_verbs.h +4 -0
@@ -467,6 +467,10 @@ int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);struct ib_mr *erdma_reg_user_mr (struct ib_pd *ibpd,u64 start,u64 len,u64 virt,int access,struct ib_dmah *dmah,struct ib_udata *udata); +struct ib_mr *erdma_reg_user_mr_dmabuf (struct ib_pd *ibpd,u64 start,u64 len, +u64 virt,int fd,int access, +struct ib_dmah *dmah, +struct uverbs_attr_bundle *attrs);struct ib_mr *erdma_get_dma_mr (struct ib_pd *ibpd,int rights);int erdma_dereg_mr (struct ib_mr *ibmr,struct ib_udata *data);int erdma_mmap (struct ib_ucontext *ctx,struct vm_area_struct *vma);
Signature · 1 line
2.46.0
Re: [PATCH for-next v3 3/3] RDMA/erdma: Implement erdma_reg_user_mr_dmabuf
Quoted 13 lines · Boshi Yu
Hi,
This patch series adds DMA-BUF memory registration support to the erdma driver:
- #1 renames get/put_mtt_entries to erdma_mem_init/uninit to better reflect their purpose of initializing the struct
erdma_mem.
- #2 introduces structerdma_mem_init_attrto pass parameters toerdma_mem_init(), improving code maintainability and preparing for DMA-BUF support.
- #3 implementserdma_reg_user_mr_dmabuf()to enable DMA-BUF based user memory registration using the revocable pinned dmabuf interface.
Quoted 3 lines
One known limitation: if the
DEREG_MRcommand fails in the revoke callback, the driver can only log an error, since erdma does not support a function-level reset like irdma'srequest_reset()yet.
I hoped someone would comment on this. If revocation fails, the hardware may continue accessing memory that the system considers released.
From comment:
* When a revocation occurs, the revoke callback will be called. The driver must
* ensure that the region is no longer accessed when the callback returns. Any
* subsequent access attempts should also probably cause an AE for MRs.
Quoted 1 line
We plan to introduce such a mechanism in future work.
That mechanism needs to be included in this series.
Thanks
Quoted 31 lines
Thanks,
Boshi Yu---
v3:
- Patch#3: Switch to the revocable pinned dmabuf interface as suggested by Jason Gunthorpe.v2:
- Patch#2: Add validation for the return value ofib_umem_find_best_pgsz().v1:
Boshi Yu (3):
RDMA/erdma: Rename get/put_mtt_entries to erdma_mem_init/uninit RDMA/erdma: Introduce structerdma_mem_init_attrRDMA/erdma: Implementerdma_reg_user_mr_dmabufdrivers/infiniband/hw/erdma/erdma_main.c | 1 + drivers/infiniband/hw/erdma/erdma_verbs.c | 262 +++++++++++++++------- drivers/infiniband/hw/erdma/erdma_verbs.h | 17 ++ 3 files changed, 201 insertions(+), 79 deletions(-)Signature · 1 line
2.46.0
Quoted 27 lines · Leon Romanovsky
Quoted 13 lines · Boshi Yu
Hi,
This patch series adds DMA-BUF memory registration support to the erdma driver:
- #1 renames get/put_mtt_entries to erdma_mem_init/uninit to better reflect their purpose of initializing the struct
erdma_mem.
- #2 introduces structerdma_mem_init_attrto pass parameters toerdma_mem_init(), improving code maintainability and preparing for DMA-BUF support.
- #3 implementserdma_reg_user_mr_dmabuf()to enable DMA-BUF based user memory registration using the revocable pinned dmabuf interface.Quoted 3 lines
One known limitation: if the
DEREG_MRcommand fails in the revoke callback, the driver can only log an error, since erdma does not support a function-level reset like irdma'srequest_reset()yet.I hoped someone would comment on this. If revocation fails, the hardware may continue accessing memory that the system considers released.
From comment:
* When a revocation occurs, the revoke callback will be called. The driver must
* ensure that the region is no longer accessed when the callback returns. Any
* subsequent access attempts should also probably cause an AE for MRs.
Quoted 6 lines
Quoted 1 line
We plan to introduce such a mechanism in future work.
That mechanism needs to be included in this series.
Thanks
Hi, Leon
Thanks for your suggestion. I will implement the reset mechanism in the next version of this series.
Thanks,
Boshi Yu
Quoted 31 lines
Quoted 31 lines
Thanks,
Boshi Yu---
v3:
- Patch#3: Switch to the revocable pinned dmabuf interface as suggested by Jason Gunthorpe.v2:
- Patch#2: Add validation for the return value ofib_umem_find_best_pgsz().v1:
link: https://lore.kernel.org/all/[email protected]/Boshi Yu (3):
RDMA/erdma: Rename get/put_mtt_entries to erdma_mem_init/uninit RDMA/erdma: Introduce structerdma_mem_init_attrRDMA/erdma: Implementerdma_reg_user_mr_dmabufdrivers/infiniband/hw/erdma/erdma_main.c | 1 + drivers/infiniband/hw/erdma/erdma_verbs.c | 262 +++++++++++++++------- drivers/infiniband/hw/erdma/erdma_verbs.h | 17 ++ 3 files changed, 201 insertions(+), 79 deletions(-)Signature · 1 line
2.46.0