Skip to contentSkip to conversation

RDMA/irdma: Add support for dmabuf pin memory regions

PATCH Superseded Zhu Yanjun 3 participants 6 messages Jan 2, 2023 – Jan 4, 2023

Subsystems (2)

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

Zhu Yanjun

author
#0 lore raw

PATCH v2 1/1 RDMA/irdma: Add support for dmabuf pin memory regions

From: Zhu Yanjun <[email protected]>

This is a followup to the EFA dmabuf[1]. Irdma driver currently does not support on-demand-paging(ODP). So it uses habanalabs as the dmabuf exporter, and irdma as the importer to allow for peer2peer access through libibverbs.

In this commit, the function ib_umem_dmabuf_get_pinned() is used. This function is introduced in EFA dmabuf[1] which allows the driver to get a dmabuf umem which is pinned and does not require move_notify callback implementation. The returned umem is pinned and DMA mapped like standard cpu umems, and is released through ib_umem_release().

[1]https://lore.kernel.org/lkml/[email protected]/t/

---
V1->V2: Fix the build warning by adding a static

---
 drivers/infiniband/hw/irdma/verbs.c | 158 ++++++++++++++++++++++++++++
 1 file changed, 158 insertions(+)
drivers/infiniband/hw/irdma/verbs.c +158 -0
index f6973ea55eda..1572baa93856 100644
@@ -2912,6 +2912,163 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
 	return ERR_PTR(err);
 }
 
+static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start,
+					      u64 len, u64 virt,
+					      int fd, int access,
+					      struct ib_udata *udata)
+{
+	struct irdma_device *iwdev = to_iwdev(pd->device);
+	struct irdma_ucontext *ucontext;
+	struct irdma_pble_alloc *palloc;
+	struct irdma_pbl *iwpbl;
+	struct irdma_mr *iwmr;
+	struct irdma_mem_reg_req req;
+	u32 total, stag = 0;
+	u8 shadow_pgcnt = 1;
+	bool use_pbles = false;
+	unsigned long flags;
+	int err = -EINVAL;
+	struct ib_umem_dmabuf *umem_dmabuf;
+
+	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
+		return ERR_PTR(-EINVAL);
+
+	if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN)
+		return ERR_PTR(-EINVAL);
+
+	umem_dmabuf = ib_umem_dmabuf_get_pinned(pd->device, start, len, fd,
+						access);
+	if (IS_ERR(umem_dmabuf)) {
+		err = PTR_ERR(umem_dmabuf);
+		ibdev_dbg(&iwdev->ibdev, "Failed to get dmabuf umem[%d]\n", err);
+		return ERR_PTR(err);
+	}
+
+	if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) {
+		ib_umem_release(&umem_dmabuf->umem);
+		return ERR_PTR(-EFAULT);
+	}
+
+	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
+	if (!iwmr) {
+		ib_umem_release(&umem_dmabuf->umem);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	iwpbl = &iwmr->iwpbl;
+	iwpbl->iwmr = iwmr;
+	iwmr->region = &umem_dmabuf->umem;
+	iwmr->ibmr.pd = pd;
+	iwmr->ibmr.device = pd->device;
+	iwmr->ibmr.iova = virt;
+	iwmr->page_size = PAGE_SIZE;
+
+	if (req.reg_type == IRDMA_MEMREG_TYPE_MEM) {
+		iwmr->page_size = ib_umem_find_best_pgsz(iwmr->region,
+							 iwdev->rf->sc_dev.hw_attrs.page_size_cap,
+							 virt);
+		if (unlikely(!iwmr->page_size)) {
+			kfree(iwmr);
+			ib_umem_release(iwmr->region);
+			return ERR_PTR(-EOPNOTSUPP);
+		}
+	}
+	iwmr->len = iwmr->region->length;
+	iwpbl->user_base = virt;
+	palloc = &iwpbl->pble_alloc;
+	iwmr->type = req.reg_type;
+	iwmr->page_cnt = ib_umem_num_dma_blocks(iwmr->region, iwmr->page_size);
+
+	switch (req.reg_type) {
+	case IRDMA_MEMREG_TYPE_QP:
+		total = req.sq_pages + req.rq_pages + shadow_pgcnt;
+		if (total > iwmr->page_cnt) {
+			err = -EINVAL;
+			goto error;
+		}
+		total = req.sq_pages + req.rq_pages;
+		use_pbles = (total > 2);
+		err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+		if (err)
+			goto error;
+
+		ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
+						     ibucontext);
+		spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
+		list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
+		iwpbl->on_list = true;
+		spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+		break;
+	case IRDMA_MEMREG_TYPE_CQ:
+		if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE)
+			shadow_pgcnt = 0;
+		total = req.cq_pages + shadow_pgcnt;
+		if (total > iwmr->page_cnt) {
+			err = -EINVAL;
+			goto error;
+		}
+
+		use_pbles = (req.cq_pages > 1);
+		err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+		if (err)
+			goto error;
+
+		ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
+						     ibucontext);
+		spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
+		list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
+		iwpbl->on_list = true;
+		spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
+		break;
+	case IRDMA_MEMREG_TYPE_MEM:
+		use_pbles = (iwmr->page_cnt != 1);
+
+		err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false);
+		if (err)
+			goto error;
+
+		if (use_pbles) {
+			err = irdma_check_mr_contiguous(palloc,
+							iwmr->page_size);
+			if (err) {
+				irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+				iwpbl->pbl_allocated = false;
+			}
+		}
+
+		stag = irdma_create_stag(iwdev);
+		if (!stag) {
+			err = -ENOMEM;
+			goto error;
+		}
+
+		iwmr->stag = stag;
+		iwmr->ibmr.rkey = stag;
+		iwmr->ibmr.lkey = stag;
+		err = irdma_hwreg_mr(iwdev, iwmr, access);
+		if (err) {
+			irdma_free_stag(iwdev, stag);
+			goto error;
+		}
+
+		break;
+	default:
+		goto error;
+	}
+
+	iwmr->type = req.reg_type;
+
+	return &iwmr->ibmr;
+
+error:
+	if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
+		irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+	ib_umem_release(iwmr->region);
+	kfree(iwmr);
+
+	return ERR_PTR(err);
+}
+
 /**
  * irdma_reg_phys_mr - register kernel physical memory
  * @pd: ibpd pointer
@@ -4418,6 +4575,7 @@ static const struct ib_device_ops irdma_dev_ops = {
 	.query_port = irdma_query_port,
 	.query_qp = irdma_query_qp,
 	.reg_user_mr = irdma_reg_user_mr,
+	.reg_user_mr_dmabuf = irdma_reg_user_mr_dmabuf,
 	.req_notify_cq = irdma_req_notify_cq,
 	.resize_cq = irdma_resize_cq,
 	INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd),
Signature · 1 line
2.27.0
#1 lore raw
Quoted 15 lines

Subject: [PATCHv2 1/1] RDMA/irdma: Add support for dmabuf pin memory regions

From: Zhu Yanjun <[email protected]>

This is a followup to the EFA dmabuf[1]. Irdma driver currently does not support on-demand-paging(ODP). So it uses habanalabs as the dmabuf exporter, and irdma as the importer to allow for peer2peer access through libibverbs.

In this commit, the function ib_umem_dmabuf_get_pinned() is used. This function is introduced in EFA dmabuf[1] which allows the driver to get a dmabuf umem which is pinned and does not require move_notify callback implementation. The returned umem is pinned and DMA mapped like standard cpu umems, and is released through ib_umem_release().

Quoted 3 lines

[1]https://lore.kernel.org/lkml/[email protected]/t/

Is there a corresponding user-space patch?

Quoted 123 lines

---
V1->V2: Fix the build warning by adding a static

---
 drivers/infiniband/hw/irdma/verbs.c | 158 ++++++++++++++++++++++++++++
 1 file changed, 158 insertions(+)
drivers/infiniband/hw/irdma/verbs.c +82 -0
index f6973ea55eda..1572baa93856 100644
@@ -2912,6 +2912,163 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd
 *pd, u64 start, u64 len,
 	return ERR_PTR(err);
 }
 
+static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start,
+					      u64 len, u64 virt,
+					      int fd, int access,
+					      struct ib_udata *udata)
+{
+	struct irdma_device *iwdev = to_iwdev(pd->device);
+	struct irdma_ucontext *ucontext;
+	struct irdma_pble_alloc *palloc;
+	struct irdma_pbl *iwpbl;
+	struct irdma_mr *iwmr;
+	struct irdma_mem_reg_req req;
+	u32 total, stag = 0;
+	u8 shadow_pgcnt = 1;
+	bool use_pbles = false;
+	unsigned long flags;
+	int err = -EINVAL;
+	struct ib_umem_dmabuf *umem_dmabuf;
+
+	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
+		return ERR_PTR(-EINVAL);
+
+	if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN)
+		return ERR_PTR(-EINVAL);
+
+	umem_dmabuf = ib_umem_dmabuf_get_pinned(pd->device, start, len, fd,
+						access);
+	if (IS_ERR(umem_dmabuf)) {
+		err = PTR_ERR(umem_dmabuf);
+		ibdev_dbg(&iwdev->ibdev, "Failed to get dmabuf umem[%d]\n",
 err);
+		return ERR_PTR(err);
+	}
+
+	if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) {
+		ib_umem_release(&umem_dmabuf->umem);
+		return ERR_PTR(-EFAULT);
+	}
+
+	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
+	if (!iwmr) {
+		ib_umem_release(&umem_dmabuf->umem);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	iwpbl = &iwmr->iwpbl;
+	iwpbl->iwmr = iwmr;
+	iwmr->region = &umem_dmabuf->umem;
+	iwmr->ibmr.pd = pd;
+	iwmr->ibmr.device = pd->device;
+	iwmr->ibmr.iova = virt;
+	iwmr->page_size = PAGE_SIZE;
+
+	if (req.reg_type == IRDMA_MEMREG_TYPE_MEM) {
+		iwmr->page_size = ib_umem_find_best_pgsz(iwmr->region,
+							 iwdev->rf-
 >sc_dev.hw_attrs.page_size_cap,
+							 virt);
+		if (unlikely(!iwmr->page_size)) {
+			kfree(iwmr);
+			ib_umem_release(iwmr->region);
+			return ERR_PTR(-EOPNOTSUPP);
+		}
+	}
+	iwmr->len = iwmr->region->length;
+	iwpbl->user_base = virt;
+	palloc = &iwpbl->pble_alloc;
+	iwmr->type = req.reg_type;
+	iwmr->page_cnt = ib_umem_num_dma_blocks(iwmr->region,
+iwmr->page_size);
+
+	switch (req.reg_type) {
+	case IRDMA_MEMREG_TYPE_QP:
+		total = req.sq_pages + req.rq_pages + shadow_pgcnt;
+		if (total > iwmr->page_cnt) {
+			err = -EINVAL;
+			goto error;
+		}
+		total = req.sq_pages + req.rq_pages;
+		use_pbles = (total > 2);
+		err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+		if (err)
+			goto error;
+
+		ucontext = rdma_udata_to_drv_context(udata, struct

irdma_ucontext,

drivers/infiniband/hw/irdma/verbs.c +8 -0
+						     ibucontext);
+		spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
+		list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
+		iwpbl->on_list = true;
+		spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+		break;
+	case IRDMA_MEMREG_TYPE_CQ:
+		if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags &

IRDMA_FEATURE_CQ_RESIZE)

drivers/infiniband/hw/irdma/verbs.c +13 -0
+			shadow_pgcnt = 0;
+		total = req.cq_pages + shadow_pgcnt;
+		if (total > iwmr->page_cnt) {
+			err = -EINVAL;
+			goto error;
+		}
+
+		use_pbles = (req.cq_pages > 1);
+		err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+		if (err)
+			goto error;
+
+		ucontext = rdma_udata_to_drv_context(udata, struct

irdma_ucontext,

drivers/infiniband/hw/irdma/verbs.c static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, · line 3022

Quoted 6 lines
drivers/infiniband/hw/irdma/verbs.c +6 -0
+						     ibucontext);
+		spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
+		list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
+		iwpbl->on_list = true;
+		spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
+		break;

I don't think we want to do this for user QP, CQ pinned memory. In fact, it will just be dead-code.

The irdma provider implementation of the ibv_reg_dmabuf_mr will just default to IRDMA_MEMREG_TYPE_MEM type similar to how irdma_ureg_mr is implemented.

https://github.com/linux-rdma/rdma-core/blob/master/providers/irdma/uverbs.c#L128

It should simplify this function a lot.

Quoted 35 lines
drivers/infiniband/hw/irdma/verbs.c +35 -0
+	case IRDMA_MEMREG_TYPE_MEM:
+		use_pbles = (iwmr->page_cnt != 1);
+
+		err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false);
+		if (err)
+			goto error;
+
+		if (use_pbles) {
+			err = irdma_check_mr_contiguous(palloc,
+							iwmr->page_size);
+			if (err) {
+				irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+				iwpbl->pbl_allocated = false;
+			}
+		}
+
+		stag = irdma_create_stag(iwdev);
+		if (!stag) {
+			err = -ENOMEM;
+			goto error;
+		}
+
+		iwmr->stag = stag;
+		iwmr->ibmr.rkey = stag;
+		iwmr->ibmr.lkey = stag;
+		err = irdma_hwreg_mr(iwdev, iwmr, access);
+		if (err) {
+			irdma_free_stag(iwdev, stag);
+			goto error;
+		}
+
+		break;
+	default:
+		goto error;
+	}

drivers/infiniband/hw/irdma/verbs.c static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, · line 3067

Quoted 10 lines
drivers/infiniband/hw/irdma/verbs.c +10 -0
+
+	iwmr->type = req.reg_type;
+
+	return &iwmr->ibmr;
+
+error:
+	if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
+		irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+	ib_umem_release(iwmr->region);
+	kfree(iwmr);

Ideally we want unwind in the reverse order of allocation.

Quoted 17 lines
drivers/infiniband/hw/irdma/verbs.c +5 -1
+
+	return ERR_PTR(err);
+}
+
 /**
  * irdma_reg_phys_mr - register kernel physical memory
  * @pd: ibpd pointer
@@ -4418,6 +4575,7 @@ static const struct ib_device_ops irdma_dev_ops = {
 	.query_port = irdma_query_port,
 	.query_qp = irdma_query_qp,
 	.reg_user_mr = irdma_reg_user_mr,
+	.reg_user_mr_dmabuf = irdma_reg_user_mr_dmabuf,
 	.req_notify_cq = irdma_req_notify_cq,
 	.resize_cq = irdma_resize_cq,
 	INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd),
--

2.27.0

#2 lore raw
Quoted 4 lines

Subject: RE: [PATCHv2 1/1] RDMA/irdma: Add support for dmabuf pin memory regions

[......]

Quoted 49 lines
Quoted 39 lines
drivers/infiniband/hw/irdma/verbs.c +15 -0
+
+	switch (req.reg_type) {
+	case IRDMA_MEMREG_TYPE_QP:
+		total = req.sq_pages + req.rq_pages + shadow_pgcnt;
+		if (total > iwmr->page_cnt) {
+			err = -EINVAL;
+			goto error;
+		}
+		total = req.sq_pages + req.rq_pages;
+		use_pbles = (total > 2);
+		err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+		if (err)
+			goto error;
+
+		ucontext = rdma_udata_to_drv_context(udata, struct

irdma_ucontext,

drivers/infiniband/hw/irdma/verbs.c +8 -0
+						     ibucontext);
+		spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
+		list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
+		iwpbl->on_list = true;
+		spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+		break;
+	case IRDMA_MEMREG_TYPE_CQ:
+		if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags &

IRDMA_FEATURE_CQ_RESIZE)

drivers/infiniband/hw/irdma/verbs.c +13 -0
+			shadow_pgcnt = 0;
+		total = req.cq_pages + shadow_pgcnt;
+		if (total > iwmr->page_cnt) {
+			err = -EINVAL;
+			goto error;
+		}
+
+		use_pbles = (req.cq_pages > 1);
+		err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+		if (err)
+			goto error;
+
+		ucontext = rdma_udata_to_drv_context(udata, struct

irdma_ucontext,

drivers/infiniband/hw/irdma/verbs.c static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, · line 3022

Quoted 6 lines
drivers/infiniband/hw/irdma/verbs.c +6 -0
+						     ibucontext);
+		spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
+		list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
+		iwpbl->on_list = true;
+		spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
+		break;

I don't think we want to do this for user QP, CQ pinned memory. In fact, it will just be dead-code.

Quoted 9 lines

The irdma provider implementation of the ibv_reg_dmabuf_mr will just default to IRDMA_MEMREG_TYPE_MEM type similar to how irdma_ureg_mr is implemented.

https://github.com/linux-rdma/rdma-
core/blob/master/providers/irdma/uverbs.c#L128

It should simplify this function a lot.

Actually I don't see a need even to use the irdma_mem_reg_req ABI struct to pass any info from user-space like reg_type.

Shiraz

#3 lore raw
Quoted 56 lines · Shiraz
Quoted 3 lines

Subject: RE: [PATCHv2 1/1] RDMA/irdma: Add support for dmabuf pin memory regions

[......]

Quoted 51 lines
Quoted 39 lines
drivers/infiniband/hw/irdma/verbs.c +15 -0
+
+	switch (req.reg_type) {
+	case IRDMA_MEMREG_TYPE_QP:
+		total = req.sq_pages + req.rq_pages + shadow_pgcnt;
+		if (total > iwmr->page_cnt) {
+			err = -EINVAL;
+			goto error;
+		}
+		total = req.sq_pages + req.rq_pages;
+		use_pbles = (total > 2);
+		err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+		if (err)
+			goto error;
+
+		ucontext = rdma_udata_to_drv_context(udata, struct

irdma_ucontext,

drivers/infiniband/hw/irdma/verbs.c +8 -0
+						     ibucontext);
+		spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
+		list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
+		iwpbl->on_list = true;
+		spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+		break;
+	case IRDMA_MEMREG_TYPE_CQ:
+		if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags &

IRDMA_FEATURE_CQ_RESIZE)

drivers/infiniband/hw/irdma/verbs.c +13 -0
+			shadow_pgcnt = 0;
+		total = req.cq_pages + shadow_pgcnt;
+		if (total > iwmr->page_cnt) {
+			err = -EINVAL;
+			goto error;
+		}
+
+		use_pbles = (req.cq_pages > 1);
+		err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+		if (err)
+			goto error;
+
+		ucontext = rdma_udata_to_drv_context(udata, struct

irdma_ucontext,

drivers/infiniband/hw/irdma/verbs.c static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, · line 3022

Quoted 6 lines
drivers/infiniband/hw/irdma/verbs.c +6 -0
+						     ibucontext);
+		spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
+		list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
+		iwpbl->on_list = true;
+		spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
+		break;

I don't think we want to do this for user QP, CQ pinned memory. In fact, it will just be dead-code.

The irdma provider implementation of the ibv_reg_dmabuf_mr will just default to IRDMA_MEMREG_TYPE_MEM type similar to how irdma_ureg_mr is implemented.

Quoted 7 lines
Quoted 6 lines

https://github.com/linux-rdma/rdma-
core/blob/master/providers/irdma/uverbs.c#L128

It should simplify this function a lot.

Actually I don't see a need even to use the irdma_mem_reg_req ABI struct to pass any info from user-space like reg_type.

In the latest commit, the irdma_mem_reg_req ABI struct is removed.

I will send out it very soon.

Zhu Yanjun

Quoted 3 lines

Shiraz

#4 lore raw
In reply to Saleem, Shiraz
Quoted 15 lines · Shiraz
Quoted 15 lines

Subject: [PATCHv2 1/1] RDMA/irdma: Add support for dmabuf pin memory regions

From: Zhu Yanjun <[email protected]>

This is a followup to the EFA dmabuf[1]. Irdma driver currently does not support on-demand-paging(ODP). So it uses habanalabs as the dmabuf exporter, and irdma as the importer to allow for peer2peer access through libibverbs.

In this commit, the function ib_umem_dmabuf_get_pinned() is used. This function is introduced in EFA dmabuf[1] which allows the driver to get a dmabuf umem which is pinned and does not require move_notify callback implementation. The returned umem is pinned and DMA mapped like standard cpu umems, and is released through ib_umem_release().

Quoted 5 lines
Quoted 3 lines

[1]https://lore.kernel.org/lkml/[email protected]/t/

Is there a corresponding user-space patch?

Yes. I will send it out very soon.

Zhu Yanjun

Quoted 204 lines
Quoted 125 lines

---
V1->V2: Fix the build warning by adding a static

---
  drivers/infiniband/hw/irdma/verbs.c | 158 ++++++++++++++++++++++++++++
  1 file changed, 158 insertions(+)
drivers/infiniband/hw/irdma/verbs.c +82 -0
index f6973ea55eda..1572baa93856 100644
@@ -2912,6 +2912,163 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd
 *pd, u64 start, u64 len,
  	return ERR_PTR(err);
  }
 
+static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start,
+					      u64 len, u64 virt,
+					      int fd, int access,
+					      struct ib_udata *udata)
+{
+	struct irdma_device *iwdev = to_iwdev(pd->device);
+	struct irdma_ucontext *ucontext;
+	struct irdma_pble_alloc *palloc;
+	struct irdma_pbl *iwpbl;
+	struct irdma_mr *iwmr;
+	struct irdma_mem_reg_req req;
+	u32 total, stag = 0;
+	u8 shadow_pgcnt = 1;
+	bool use_pbles = false;
+	unsigned long flags;
+	int err = -EINVAL;
+	struct ib_umem_dmabuf *umem_dmabuf;
+
+	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
+		return ERR_PTR(-EINVAL);
+
+	if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN)
+		return ERR_PTR(-EINVAL);
+
+	umem_dmabuf = ib_umem_dmabuf_get_pinned(pd->device, start, len, fd,
+						access);
+	if (IS_ERR(umem_dmabuf)) {
+		err = PTR_ERR(umem_dmabuf);
+		ibdev_dbg(&iwdev->ibdev, "Failed to get dmabuf umem[%d]\n",
 err);
+		return ERR_PTR(err);
+	}
+
+	if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) {
+		ib_umem_release(&umem_dmabuf->umem);
+		return ERR_PTR(-EFAULT);
+	}
+
+	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
+	if (!iwmr) {
+		ib_umem_release(&umem_dmabuf->umem);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	iwpbl = &iwmr->iwpbl;
+	iwpbl->iwmr = iwmr;
+	iwmr->region = &umem_dmabuf->umem;
+	iwmr->ibmr.pd = pd;
+	iwmr->ibmr.device = pd->device;
+	iwmr->ibmr.iova = virt;
+	iwmr->page_size = PAGE_SIZE;
+
+	if (req.reg_type == IRDMA_MEMREG_TYPE_MEM) {
+		iwmr->page_size = ib_umem_find_best_pgsz(iwmr->region,
+							 iwdev->rf-
 > sc_dev.hw_attrs.page_size_cap,
+							 virt);
+		if (unlikely(!iwmr->page_size)) {
+			kfree(iwmr);
+			ib_umem_release(iwmr->region);
+			return ERR_PTR(-EOPNOTSUPP);
+		}
+	}
+	iwmr->len = iwmr->region->length;
+	iwpbl->user_base = virt;
+	palloc = &iwpbl->pble_alloc;
+	iwmr->type = req.reg_type;
+	iwmr->page_cnt = ib_umem_num_dma_blocks(iwmr->region,
+iwmr->page_size);
+
+	switch (req.reg_type) {
+	case IRDMA_MEMREG_TYPE_QP:
+		total = req.sq_pages + req.rq_pages + shadow_pgcnt;
+		if (total > iwmr->page_cnt) {
+			err = -EINVAL;
+			goto error;
+		}
+		total = req.sq_pages + req.rq_pages;
+		use_pbles = (total > 2);
+		err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+		if (err)
+			goto error;
+
+		ucontext = rdma_udata_to_drv_context(udata, struct

irdma_ucontext,

drivers/infiniband/hw/irdma/verbs.c +8 -0
+						     ibucontext);
+		spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
+		list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
+		iwpbl->on_list = true;
+		spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+		break;
+	case IRDMA_MEMREG_TYPE_CQ:
+		if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags &

IRDMA_FEATURE_CQ_RESIZE)

drivers/infiniband/hw/irdma/verbs.c +13 -0
+			shadow_pgcnt = 0;
+		total = req.cq_pages + shadow_pgcnt;
+		if (total > iwmr->page_cnt) {
+			err = -EINVAL;
+			goto error;
+		}
+
+		use_pbles = (req.cq_pages > 1);
+		err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+		if (err)
+			goto error;
+
+		ucontext = rdma_udata_to_drv_context(udata, struct

irdma_ucontext,

drivers/infiniband/hw/irdma/verbs.c static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, · line 3022

Quoted 6 lines
drivers/infiniband/hw/irdma/verbs.c +6 -0
+						     ibucontext);
+		spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
+		list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
+		iwpbl->on_list = true;
+		spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
+		break;

I don't think we want to do this for user QP, CQ pinned memory. In fact, it will just be dead-code.

The irdma provider implementation of the ibv_reg_dmabuf_mr will just default to IRDMA_MEMREG_TYPE_MEM type similar to how irdma_ureg_mr is implemented.

https://github.com/linux-rdma/rdma-core/blob/master/providers/irdma/uverbs.c#L128

It should simplify this function a lot.

Quoted 35 lines
drivers/infiniband/hw/irdma/verbs.c +35 -0
+	case IRDMA_MEMREG_TYPE_MEM:
+		use_pbles = (iwmr->page_cnt != 1);
+
+		err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false);
+		if (err)
+			goto error;
+
+		if (use_pbles) {
+			err = irdma_check_mr_contiguous(palloc,
+							iwmr->page_size);
+			if (err) {
+				irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+				iwpbl->pbl_allocated = false;
+			}
+		}
+
+		stag = irdma_create_stag(iwdev);
+		if (!stag) {
+			err = -ENOMEM;
+			goto error;
+		}
+
+		iwmr->stag = stag;
+		iwmr->ibmr.rkey = stag;
+		iwmr->ibmr.lkey = stag;
+		err = irdma_hwreg_mr(iwdev, iwmr, access);
+		if (err) {
+			irdma_free_stag(iwdev, stag);
+			goto error;
+		}
+
+		break;
+	default:
+		goto error;
+	}

drivers/infiniband/hw/irdma/verbs.c static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, · line 3067

Quoted 10 lines
drivers/infiniband/hw/irdma/verbs.c +10 -0
+
+	iwmr->type = req.reg_type;
+
+	return &iwmr->ibmr;
+
+error:
+	if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
+		irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+	ib_umem_release(iwmr->region);
+	kfree(iwmr);

Ideally we want unwind in the reverse order of allocation.

Quoted 17 lines
drivers/infiniband/hw/irdma/verbs.c +5 -1
+
+	return ERR_PTR(err);
+}
+
  /**
   * irdma_reg_phys_mr - register kernel physical memory
   * @pd: ibpd pointer
@@ -4418,6 +4575,7 @@ static const struct ib_device_ops irdma_dev_ops = {
  	.query_port = irdma_query_port,
  	.query_qp = irdma_query_qp,
  	.reg_user_mr = irdma_reg_user_mr,
+	.reg_user_mr_dmabuf = irdma_reg_user_mr_dmabuf,
  	.req_notify_cq = irdma_req_notify_cq,
  	.resize_cq = irdma_resize_cq,
  	INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd),
--

2.27.0

#5 lore raw
In reply to Saleem, Shiraz
Quoted 153 lines · Shiraz
Quoted 15 lines

Subject: [PATCHv2 1/1] RDMA/irdma: Add support for dmabuf pin memory regions

From: Zhu Yanjun <[email protected]>

This is a followup to the EFA dmabuf[1]. Irdma driver currently does not support on-demand-paging(ODP). So it uses habanalabs as the dmabuf exporter, and irdma as the importer to allow for peer2peer access through libibverbs.

In this commit, the function ib_umem_dmabuf_get_pinned() is used. This function is introduced in EFA dmabuf[1] which allows the driver to get a dmabuf umem which is pinned and does not require move_notify callback implementation. The returned umem is pinned and DMA mapped like standard cpu umems, and is released through ib_umem_release().

Quoted 3 lines

[1]https://lore.kernel.org/lkml/[email protected]/t/

Is there a corresponding user-space patch?

Quoted 123 lines

---
V1->V2: Fix the build warning by adding a static

---
  drivers/infiniband/hw/irdma/verbs.c | 158 ++++++++++++++++++++++++++++
  1 file changed, 158 insertions(+)
drivers/infiniband/hw/irdma/verbs.c +82 -0
index f6973ea55eda..1572baa93856 100644
@@ -2912,6 +2912,163 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd
 *pd, u64 start, u64 len,
  	return ERR_PTR(err);
  }
 
+static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start,
+					      u64 len, u64 virt,
+					      int fd, int access,
+					      struct ib_udata *udata)
+{
+	struct irdma_device *iwdev = to_iwdev(pd->device);
+	struct irdma_ucontext *ucontext;
+	struct irdma_pble_alloc *palloc;
+	struct irdma_pbl *iwpbl;
+	struct irdma_mr *iwmr;
+	struct irdma_mem_reg_req req;
+	u32 total, stag = 0;
+	u8 shadow_pgcnt = 1;
+	bool use_pbles = false;
+	unsigned long flags;
+	int err = -EINVAL;
+	struct ib_umem_dmabuf *umem_dmabuf;
+
+	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
+		return ERR_PTR(-EINVAL);
+
+	if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN)
+		return ERR_PTR(-EINVAL);
+
+	umem_dmabuf = ib_umem_dmabuf_get_pinned(pd->device, start, len, fd,
+						access);
+	if (IS_ERR(umem_dmabuf)) {
+		err = PTR_ERR(umem_dmabuf);
+		ibdev_dbg(&iwdev->ibdev, "Failed to get dmabuf umem[%d]\n",
 err);
+		return ERR_PTR(err);
+	}
+
+	if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) {
+		ib_umem_release(&umem_dmabuf->umem);
+		return ERR_PTR(-EFAULT);
+	}
+
+	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
+	if (!iwmr) {
+		ib_umem_release(&umem_dmabuf->umem);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	iwpbl = &iwmr->iwpbl;
+	iwpbl->iwmr = iwmr;
+	iwmr->region = &umem_dmabuf->umem;
+	iwmr->ibmr.pd = pd;
+	iwmr->ibmr.device = pd->device;
+	iwmr->ibmr.iova = virt;
+	iwmr->page_size = PAGE_SIZE;
+
+	if (req.reg_type == IRDMA_MEMREG_TYPE_MEM) {
+		iwmr->page_size = ib_umem_find_best_pgsz(iwmr->region,
+							 iwdev->rf-
 > sc_dev.hw_attrs.page_size_cap,
+							 virt);
+		if (unlikely(!iwmr->page_size)) {
+			kfree(iwmr);
+			ib_umem_release(iwmr->region);
+			return ERR_PTR(-EOPNOTSUPP);
+		}
+	}
+	iwmr->len = iwmr->region->length;
+	iwpbl->user_base = virt;
+	palloc = &iwpbl->pble_alloc;
+	iwmr->type = req.reg_type;
+	iwmr->page_cnt = ib_umem_num_dma_blocks(iwmr->region,
+iwmr->page_size);
+
+	switch (req.reg_type) {
+	case IRDMA_MEMREG_TYPE_QP:
+		total = req.sq_pages + req.rq_pages + shadow_pgcnt;
+		if (total > iwmr->page_cnt) {
+			err = -EINVAL;
+			goto error;
+		}
+		total = req.sq_pages + req.rq_pages;
+		use_pbles = (total > 2);
+		err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+		if (err)
+			goto error;
+
+		ucontext = rdma_udata_to_drv_context(udata, struct

irdma_ucontext,

drivers/infiniband/hw/irdma/verbs.c +8 -0
+						     ibucontext);
+		spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
+		list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
+		iwpbl->on_list = true;
+		spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+		break;
+	case IRDMA_MEMREG_TYPE_CQ:
+		if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags &

IRDMA_FEATURE_CQ_RESIZE)

drivers/infiniband/hw/irdma/verbs.c +13 -0
+			shadow_pgcnt = 0;
+		total = req.cq_pages + shadow_pgcnt;
+		if (total > iwmr->page_cnt) {
+			err = -EINVAL;
+			goto error;
+		}
+
+		use_pbles = (req.cq_pages > 1);
+		err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles);
+		if (err)
+			goto error;
+
+		ucontext = rdma_udata_to_drv_context(udata, struct

irdma_ucontext,

drivers/infiniband/hw/irdma/verbs.c static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, · line 3022

Quoted 6 lines
drivers/infiniband/hw/irdma/verbs.c +6 -0
+						     ibucontext);
+		spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
+		list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
+		iwpbl->on_list = true;
+		spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
+		break;

I don't think we want to do this for user QP, CQ pinned memory. In fact, it will just be dead-code.

Quoted 5 lines

The irdma provider implementation of the ibv_reg_dmabuf_mr will just default to IRDMA_MEMREG_TYPE_MEM type similar to how irdma_ureg_mr is implemented.

https://github.com/linux-rdma/rdma-core/blob/master/providers/irdma/uverbs.c#L128

It should simplify this function a lot.

Got it. Thanks a lot for your advice.

I will send out the latest commit very soon.

Quoted 40 lines
Quoted 40 lines
drivers/infiniband/hw/irdma/verbs.c +38 -0
+	case IRDMA_MEMREG_TYPE_MEM:
+		use_pbles = (iwmr->page_cnt != 1);
+
+		err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false);
+		if (err)
+			goto error;
+
+		if (use_pbles) {
+			err = irdma_check_mr_contiguous(palloc,
+							iwmr->page_size);
+			if (err) {
+				irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+				iwpbl->pbl_allocated = false;
+			}
+		}
+
+		stag = irdma_create_stag(iwdev);
+		if (!stag) {
+			err = -ENOMEM;
+			goto error;
+		}
+
+		iwmr->stag = stag;
+		iwmr->ibmr.rkey = stag;
+		iwmr->ibmr.lkey = stag;
+		err = irdma_hwreg_mr(iwdev, iwmr, access);
+		if (err) {
+			irdma_free_stag(iwdev, stag);
+			goto error;
+		}
+
+		break;
+	default:
+		goto error;
+	}
+
+	iwmr->type = req.reg_type;
+
Quoted 8 lines

drivers/infiniband/hw/irdma/verbs.c static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, · line 3067

Quoted 7 lines
drivers/infiniband/hw/irdma/verbs.c +7 -0
+	return &iwmr->ibmr;
+
+error:
+	if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
+		irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
+	ib_umem_release(iwmr->region);
+	kfree(iwmr);

Ideally we want unwind in the reverse order of allocation.

Good catch.

Thanks

Zhu Yanjun

Quoted 18 lines
Quoted 18 lines
drivers/infiniband/hw/irdma/verbs.c +5 -1
+
+	return ERR_PTR(err);
+}
+
  /**
   * irdma_reg_phys_mr - register kernel physical memory
   * @pd: ibpd pointer
@@ -4418,6 +4575,7 @@ static const struct ib_device_ops irdma_dev_ops = {
  	.query_port = irdma_query_port,
  	.query_qp = irdma_query_qp,
  	.reg_user_mr = irdma_reg_user_mr,
+	.reg_user_mr_dmabuf = irdma_reg_user_mr_dmabuf,
  	.req_notify_cq = irdma_req_notify_cq,
  	.resize_cq = irdma_resize_cq,
  	INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd),
--

2.27.0