Skip to contentSkip to conversation

RDMA/rxe: Add dma-buf support for Soft-RoCE

PATCH In review Zhu Yanjun 2 participants 4 messages Mar 26 – Mar 30

Subsystems (2)
lore
Share & tools
lore

Zhu Yanjun

author
#0 lore raw

cover RDMA/rxe: Add dma-buf support for Soft-RoCE

This patchset introduces dma-buf support for the Soft-RoCE (RXE) driver. By enabling dma-buf, RXE can now participate in zero-copy data transfers with other providers (such as GPUs) that export memory via dma-buf fds.

Traditionally, RXE only supported user-space memory regions (UMEM) based on system RAM. This change extends RXE’s capability to handle peer-to-peer (P2P) like workflows in a software-defined RDMA environment.

This patchset pass the rdma-core tests with the following link for the rdma-core:

https://github.com/linux-rdma/rdma-core/pull/1055

Zhu Yanjun (2):
RDMA/umem: Change for rdma devices has not dma device RDMA/rxe: Add dma-buf support

 drivers/infiniband/core/umem_dmabuf.c | 35 ++++++++++-
 drivers/infiniband/sw/rxe/rxe.c       |  2 +
 drivers/infiniband/sw/rxe/rxe_loc.h   |  2 +
 drivers/infiniband/sw/rxe/rxe_mr.c    | 89 ++++++++++++++++++++++++---
 drivers/infiniband/sw/rxe/rxe_odp.c   |  2 +-
 drivers/infiniband/sw/rxe/rxe_verbs.c | 40 ++++++++++++
 drivers/infiniband/sw/rxe/rxe_verbs.h |  2 +-
 include/rdma/ib_umem.h                |  1 +
 8 files changed, 161 insertions(+), 12 deletions(-)
Signature · 1 line
2.53.0
author
#1 lore raw

PATCH 1/2 RDMA/umem: Change for rdma devices has not dma device

Current implementation requires a dma device for RDMA driver to use dma-buf memory space as RDMA buffer.

---
 drivers/infiniband/core/umem_dmabuf.c | 35 ++++++++++++++++++++++++++-
 include/rdma/ib_umem.h                |  1 +
 2 files changed, 35 insertions(+), 1 deletion(-)
drivers/infiniband/core/umem_dmabuf.c +34 -1
index d30f24b90bca..65c5f09f380f 100644
@@ -142,6 +142,8 @@ ib_umem_dmabuf_get_with_dma_device(struct ib_device *device,
 		goto out_release_dmabuf;
 	}
 
+	umem_dmabuf->dmabuf = dmabuf;
+
 	umem = &umem_dmabuf->umem;
 	umem->ibdev = device;
 	umem->length = size;
@@ -152,6 +154,24 @@ ib_umem_dmabuf_get_with_dma_device(struct ib_device *device,
 	if (!ib_umem_num_pages(umem))
 		goto out_free_umem;
 
+	/* Software RDMA drivers has not dma device. Just get dmabuf from fd */
+	if (!device->dma_device) {
+		struct sg_table *sgt;
+
+		dma_resv_lock(dmabuf->resv, NULL);
+		sgt = dmabuf->ops->map_dma_buf(NULL, DMA_BIDIRECTIONAL);
+		dma_resv_unlock(dmabuf->resv);
+		if (IS_ERR(sgt)) {
+			ret = ERR_CAST(sgt);
+			goto out_free_umem;
+		}
+		umem_dmabuf->sgt = sgt;
+		goto done;
+	}
+
+	if (unlikely(!ops || !ops->move_notify))
+		goto out_free_umem;
+
 	umem_dmabuf->attach = dma_buf_dynamic_attach(
 					dmabuf,
 					dma_device,
@@ -161,6 +181,7 @@ ib_umem_dmabuf_get_with_dma_device(struct ib_device *device,
 		ret = ERR_CAST(umem_dmabuf->attach);
 		goto out_free_umem;
 	}
+done:
 	return umem_dmabuf;
 
 out_free_umem:
@@ -260,11 +281,23 @@ EXPORT_SYMBOL(ib_umem_dmabuf_revoke);
 
 void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf)
 {
-	struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf;
+	struct dma_buf *dmabuf = umem_dmabuf->dmabuf;
+
+	if (!umem_dmabuf->attach) {
+		if (umem_dmabuf->sgt) {
+			dma_resv_lock(dmabuf->resv, NULL);
+			dmabuf->ops->unmap_dma_buf(NULL, umem_dmabuf->sgt,
+							DMA_BIDIRECTIONAL);
+			dma_resv_unlock(dmabuf->resv);
+		}
+		goto free_dmabuf;
+	}
 
 	ib_umem_dmabuf_revoke(umem_dmabuf);
 
 	dma_buf_detach(dmabuf, umem_dmabuf->attach);
+
+free_dmabuf:
 	dma_buf_put(dmabuf);
 	kfree(umem_dmabuf);
 }
include/rdma/ib_umem.h +1 -0
index 0a8e092c0ea8..6eb5760d5ca3 100644
@@ -31,6 +31,7 @@ struct ib_umem {
 struct ib_umem_dmabuf {
 	struct ib_umem umem;
 	struct dma_buf_attachment *attach;
+	struct dma_buf *dmabuf;
 	struct sg_table *sgt;
 	struct scatterlist *first_sg;
 	struct scatterlist *last_sg;
Signature · 1 line
2.53.0
#2 lore raw

Re: [PATCH 1/2] RDMA/umem: Change for rdma devices has not dma device

Quoted 23 lines · Zhu Yanjun

Current implementation requires a dma device for RDMA driver to use dma-buf memory space as RDMA buffer.

---
 drivers/infiniband/core/umem_dmabuf.c | 35 ++++++++++++++++++++++++++-
 include/rdma/ib_umem.h                |  1 +
 2 files changed, 35 insertions(+), 1 deletion(-)
drivers/infiniband/core/umem_dmabuf.c +2 -0
index d30f24b90bca..65c5f09f380f 100644
@@ -142,6 +142,8 @@ ib_umem_dmabuf_get_with_dma_device(struct ib_device *device,
 		goto out_release_dmabuf;
 	}
 
+	umem_dmabuf->dmabuf = dmabuf;
+
 	umem = &umem_dmabuf->umem;
 	umem->ibdev = device;
 	umem->length = size;

drivers/infiniband/core/umem_dmabuf.c ib_umem_dmabuf_get_with_dma_device(struct ib_device *device, · line 158

Quoted 5 lines
drivers/infiniband/core/umem_dmabuf.c +2 -0
@@ -152,6 +154,24 @@ ib_umem_dmabuf_get_with_dma_device(struct ib_device *device,
 	if (!ib_umem_num_pages(umem))
 		goto out_free_umem;
 
+	/* Software RDMA drivers has not dma device. Just get dmabuf from fd */
+	if (!device->dma_device) {

I understand your intent, but RXE cannot support dmabuf, as dmabuf requires a real DMA‑capable device to be attached. In addition, it is unclear how you expect PCI peer‑to‑peer operations to work when one side (RXE) is not a PCI device at all.

Thanks

author
#3 lore raw

PATCH 2/2 RDMA/rxe: Add dma-buf support

In reply to Zhu Yanjun

Implement a ib device operation ‘reg_user_mr_dmabuf’. Generate a rxe_map from the memory space linked the passed dma-buf.

---
 drivers/infiniband/sw/rxe/rxe.c       |  2 +
 drivers/infiniband/sw/rxe/rxe_loc.h   |  2 +
 drivers/infiniband/sw/rxe/rxe_mr.c    | 89 ++++++++++++++++++++++++---
 drivers/infiniband/sw/rxe/rxe_odp.c   |  2 +-
 drivers/infiniband/sw/rxe/rxe_verbs.c | 40 ++++++++++++
 drivers/infiniband/sw/rxe/rxe_verbs.h |  2 +-
 6 files changed, 126 insertions(+), 11 deletions(-)
drivers/infiniband/sw/rxe/rxe.c +2 -0
index e891199cbdef..9920ea3104be 100644
@@ -278,3 +278,5 @@ late_initcall(rxe_module_init);
 module_exit(rxe_module_exit);
 
 MODULE_ALIAS_RDMA_LINK("rxe");
+
+MODULE_IMPORT_NS("DMA_BUF");
drivers/infiniband/sw/rxe/rxe_loc.h +2 -0
index 7992290886e1..dc9a56450c82 100644
@@ -66,6 +66,8 @@ int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr);
 int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length);
 int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
 		unsigned int length, enum rxe_mr_copy_dir dir);
+int rxe_mr_dmabuf_init_user(struct rxe_pd *pd, int fd, u64 start, u64 length,
+			    u64 iova, int access, struct rxe_mr *mr);
 int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma,
 	      void *addr, int length, enum rxe_mr_copy_dir dir);
 int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
drivers/infiniband/sw/rxe/rxe_mr.c +80 -9

148 lines not on this page · raw message

drivers/infiniband/sw/rxe/rxe_odp.c +1 -1
index bc11b1ec59ac..12c48f0cae47 100644
@@ -351,7 +351,7 @@ int rxe_odp_flush_pmem_iova(struct rxe_mr *mr, u64 iova,
 		page = hmm_pfn_to_page(umem_odp->map.pfn_list[index]);
 
 		bytes = min_t(unsigned int, length,
-			      mr_page_size(mr) - page_offset);
+			      rxe_mr_page_size(mr) - page_offset);
 
 		va = kmap_local_page(page);
 		arch_wb_cache_pmem(va + page_offset, bytes);
drivers/infiniband/sw/rxe/rxe_verbs.c +40 -0
index fe41362c5144..1b5381b14d4b 100644
@@ -1358,6 +1358,45 @@ static struct ib_mr *rxe_rereg_user_mr(struct ib_mr *ibmr, int flags,
 	return NULL;
 }
 
+static struct ib_mr *rxe_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start,
+					    u64 length, u64 iova, int fd,
+					    int access, struct ib_dmah *dmah,
+					    struct uverbs_attr_bundle *udata)
+{
+	int err;
+	struct rxe_dev *rxe = to_rdev(ibpd->device);
+	struct rxe_pd *pd = to_rpd(ibpd);
+	struct rxe_mr *mr;
+
+	mr = kzalloc_obj(*mr);
+	if (!mr) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	err = rxe_add_to_pool(&rxe->mr_pool, mr);
+	if (err)
+		goto err_free;
+
+	rxe_get(pd);
+
+	err = rxe_mr_dmabuf_init_user(pd, fd, start, length, iova, access, mr);
+	if (err)
+		goto err3;
+
+	return &mr->ibmr;
+
+err3:
+	rxe_put(pd);
+	rxe_put(mr);
+
+err_free:
+	kfree(mr);
+
+err_out:
+	return ERR_PTR(err);
+}
+
 static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
 				  u32 max_num_sg)
 {
@@ -1517,6 +1556,7 @@ static const struct ib_device_ops rxe_dev_ops = {
 	.query_qp = rxe_query_qp,
 	.query_srq = rxe_query_srq,
 	.reg_user_mr = rxe_reg_user_mr,
+	.reg_user_mr_dmabuf = rxe_reg_user_mr_dmabuf,
 	.req_notify_cq = rxe_req_notify_cq,
 	.rereg_user_mr = rxe_rereg_user_mr,
 	.resize_cq = rxe_resize_cq,
drivers/infiniband/sw/rxe/rxe_verbs.h +1 -1
index fb149f37e91d..9d77bec0bf3c 100644
@@ -364,7 +364,7 @@ struct rxe_mr {
 	struct rxe_mr_page	*page_info;
 };
 
-static inline unsigned int mr_page_size(struct rxe_mr *mr)
+static inline unsigned int rxe_mr_page_size(struct rxe_mr *mr)
 {
 	return mr ? mr->ibmr.page_size : PAGE_SIZE;
 }
Signature · 1 line
2.53.0