Skip to contentSkip to conversation

Providers/rxe: Add dma-buf support

RFC PATCH Superseded Shunsuke Mie 2 participants 8 messages Sep 30, 2021

Subsystems (2)

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

v2 cover Providers/rxe: Add dma-buf support

This is an user space counter-part of the kernel patch set to add dma-buf support to the RXE driver.

Pull request at GitHub: https://github.com/linux-rdma/rdma-core/pull/1055

This is the secound version of the patch. Change log:
v2:
* Fix code formats that were indicated by Azure CI
v1: https://www.spinics.net/lists/linux-rdma/msg105380.html
* Initial patch set
* Implement a callback function for reg_dmabuf_mr

Shunsuke Mie (1):
Providers/rxe: Add dma-buf support

 providers/rxe/rxe.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
Signature · 1 line
2.17.1
#1 lore raw

RFC PATCH v2 1/1 Providers/rxe: Add dma-buf support

Implement a new provider method for dma-buf base memory registration.

---
 providers/rxe/rxe.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
providers/rxe/rxe.c +21 -0
index 3c3ea8bb..84e00e60 100644
@@ -239,6 +239,26 @@ static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
 	return &vmr->ibv_mr;
 }
 
+static struct ibv_mr *rxe_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,
+					size_t length, uint64_t iova, int fd,
+					int access)
+{
+	struct verbs_mr *vmr;
+	int ret;
+
+	vmr = malloc(sizeof(*vmr));
+	if (!vmr)
+		return NULL;
+
+	ret = ibv_cmd_reg_dmabuf_mr(pd, offset, length, iova, fd, access, vmr);
+	if (ret) {
+		free(vmr);
+		return NULL;
+	}
+
+	return &vmr->ibv_mr;
+}
+
 static int rxe_dereg_mr(struct verbs_mr *vmr)
 {
 	int ret;
@@ -1706,6 +1726,7 @@ static const struct verbs_context_ops rxe_ctx_ops = {
 	.alloc_pd = rxe_alloc_pd,
 	.dealloc_pd = rxe_dealloc_pd,
 	.reg_mr = rxe_reg_mr,
+	.reg_dmabuf_mr = rxe_reg_dmabuf_mr,
 	.dereg_mr = rxe_dereg_mr,
 	.alloc_mw = rxe_alloc_mw,
 	.dealloc_mw = rxe_dealloc_mw,
Signature · 1 line
2.17.1
#2 lore raw
Quoted 17 lines · Shunsuke Mie

Implement a new provider method for dma-buf base memory registration.

---
 providers/rxe/rxe.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
providers/rxe/rxe.c +1 -0
index 3c3ea8bb..84e00e60 100644
@@ -239,6 +239,26 @@ static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
        return &vmr->ibv_mr;
 }
 
+static struct ibv_mr *rxe_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,

providers/rxe/rxe.c static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length, · line 252

Quoted 10 lines
providers/rxe/rxe.c +10 -0
+                                       size_t length, uint64_t iova, int fd,
+                                       int access)
+{
+       struct verbs_mr *vmr;
+       int ret;
+
+       vmr = malloc(sizeof(*vmr));
+       if (!vmr)
+               return NULL;
+

Do we need to set vmr to zero like the following?

memset(vmr, 0, sizeof(*vmr));

Zhu Yanjun

Quoted 23 lines
providers/rxe/rxe.c +10 -1
+       ret = ibv_cmd_reg_dmabuf_mr(pd, offset, length, iova, fd, access, vmr);
+       if (ret) {
+               free(vmr);
+               return NULL;
+       }
+
+       return &vmr->ibv_mr;
+}
+
 static int rxe_dereg_mr(struct verbs_mr *vmr)
 {
        int ret;
@@ -1706,6 +1726,7 @@ static const struct verbs_context_ops rxe_ctx_ops = {
        .alloc_pd = rxe_alloc_pd,
        .dealloc_pd = rxe_dealloc_pd,
        .reg_mr = rxe_reg_mr,
+       .reg_dmabuf_mr = rxe_reg_dmabuf_mr,
        .dereg_mr = rxe_dereg_mr,
        .alloc_mw = rxe_alloc_mw,
        .dealloc_mw = rxe_dealloc_mw,
--

2.17.1

#3 lore raw

2021年9月30日(木) 15:37 Zhu Yanjun <[email protected]>:

Quoted 32 lines · Shunsuke Mie
Quoted 17 lines

Implement a new provider method for dma-buf base memory registration.

---
 providers/rxe/rxe.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
providers/rxe/rxe.c +1 -0
index 3c3ea8bb..84e00e60 100644
@@ -239,6 +239,26 @@ static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
        return &vmr->ibv_mr;
 }
 
+static struct ibv_mr *rxe_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,

providers/rxe/rxe.c static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length, · line 252

Quoted 10 lines
providers/rxe/rxe.c +10 -0
+                                       size_t length, uint64_t iova, int fd,
+                                       int access)
+{
+       struct verbs_mr *vmr;
+       int ret;
+
+       vmr = malloc(sizeof(*vmr));
+       if (!vmr)
+               return NULL;
+

Do we need to set vmr to zero like the following?

Quoted 3 lines

memset(vmr, 0, sizeof(*vmr));

Zhu Yanjun

Thank you for your quick response.

I think it is better to clear the vmr. Actually the mlx5 driver allocates the vmr using calloc().

In addition, rxe_reg_mr() (not rxe_reg_dmabuf_mr()) is used the malloc and not clear the vmr. I think It has to be fixed too. Should I make another patch to fix this problem?

Thanks a lot.
Shunsuke

~

#4 lore raw
Quoted 37 lines · Shunsuke Mie

2021年9月30日(木) 15:37 Zhu Yanjun <[email protected]>:

Quoted 32 lines · Shunsuke Mie
Quoted 17 lines

Implement a new provider method for dma-buf base memory registration.

---
 providers/rxe/rxe.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
providers/rxe/rxe.c +1 -0
index 3c3ea8bb..84e00e60 100644
@@ -239,6 +239,26 @@ static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
        return &vmr->ibv_mr;
 }
 
+static struct ibv_mr *rxe_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,

providers/rxe/rxe.c static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length, · line 252

Quoted 10 lines
providers/rxe/rxe.c +10 -0
+                                       size_t length, uint64_t iova, int fd,
+                                       int access)
+{
+       struct verbs_mr *vmr;
+       int ret;
+
+       vmr = malloc(sizeof(*vmr));
+       if (!vmr)
+               return NULL;
+

Do we need to set vmr to zero like the following?

Quoted 3 lines

memset(vmr, 0, sizeof(*vmr));

Zhu Yanjun

Quoted 8 lines

Thank you for your quick response.

I think it is better to clear the vmr. Actually the mlx5 driver allocates the vmr using calloc().

In addition, rxe_reg_mr() (not rxe_reg_dmabuf_mr()) is used the malloc and not clear the vmr. I think It has to be fixed too. Should I make another patch to fix this problem?

Yes. Please.

Zhu Yanjun

Quoted 5 lines

Thanks a lot.
Shunsuke

~

#5 lore raw

2021年9月30日(木) 16:23 Zhu Yanjun <[email protected]>:

Quoted 44 lines · Shunsuke Mie
Quoted 42 lines

2021年9月30日(木) 15:37 Zhu Yanjun <[email protected]>:

Quoted 32 lines · Shunsuke Mie
Quoted 17 lines

Implement a new provider method for dma-buf base memory registration.

---
 providers/rxe/rxe.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
providers/rxe/rxe.c +1 -0
index 3c3ea8bb..84e00e60 100644
@@ -239,6 +239,26 @@ static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
        return &vmr->ibv_mr;
 }
 
+static struct ibv_mr *rxe_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,

providers/rxe/rxe.c static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length, · line 252

Quoted 10 lines
providers/rxe/rxe.c +10 -0
+                                       size_t length, uint64_t iova, int fd,
+                                       int access)
+{
+       struct verbs_mr *vmr;
+       int ret;
+
+       vmr = malloc(sizeof(*vmr));
+       if (!vmr)
+               return NULL;
+

Do we need to set vmr to zero like the following?

Quoted 3 lines

memset(vmr, 0, sizeof(*vmr));

Zhu Yanjun

Thank you for your quick response.

I think it is better to clear the vmr. Actually the mlx5 driver allocates the vmr using calloc().

Quoted 13 lines
Quoted 3 lines

In addition, rxe_reg_mr() (not rxe_reg_dmabuf_mr()) is used the malloc and not clear the vmr. I think It has to be fixed too. Should I make another patch to fix this problem?

Yes. Please.

Zhu Yanjun

Quoted 5 lines

Thanks a lot.
Shunsuke

~

I looked into the vmr more, but there was no need to clear it. Moreover, some implementations also use malloc without memory clear.

Thanks,
Shunsuke

#6 lore raw
Quoted 55 lines · Shunsuke Mie

2021年9月30日(木) 16:23 Zhu Yanjun <[email protected]>:

Quoted 53 lines · Shunsuke Mie
Quoted 37 lines

2021年9月30日(木) 15:37 Zhu Yanjun <[email protected]>:

Quoted 32 lines · Shunsuke Mie
Quoted 17 lines

Implement a new provider method for dma-buf base memory registration.

---
 providers/rxe/rxe.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
providers/rxe/rxe.c +1 -0
index 3c3ea8bb..84e00e60 100644
@@ -239,6 +239,26 @@ static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
        return &vmr->ibv_mr;
 }
 
+static struct ibv_mr *rxe_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,

providers/rxe/rxe.c static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length, · line 252

Quoted 10 lines
providers/rxe/rxe.c +10 -0
+                                       size_t length, uint64_t iova, int fd,
+                                       int access)
+{
+       struct verbs_mr *vmr;
+       int ret;
+
+       vmr = malloc(sizeof(*vmr));
+       if (!vmr)
+               return NULL;
+

Do we need to set vmr to zero like the following?

Quoted 3 lines

memset(vmr, 0, sizeof(*vmr));

Zhu Yanjun

Quoted 8 lines

Thank you for your quick response.

I think it is better to clear the vmr. Actually the mlx5 driver allocates the vmr using calloc().

In addition, rxe_reg_mr() (not rxe_reg_dmabuf_mr()) is used the malloc and not clear the vmr. I think It has to be fixed too. Should I make another patch to fix this problem?

Yes. Please.

Zhu Yanjun

Quoted 1 line
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines

Thanks a lot.
Shunsuke

~

I looked into the vmr more, but there was no need to clear it. Moreover, some implementations also use malloc without memory clear.

I confronted a lot of problems with memory not initialization. And a latest bug is https://www.spinics.net/lists/linux-rdma/msg105001.html

So it is a good habit to clear a newly allocated memory.

Zhu Yanjun

Quoted 2 lines

Thanks,
Shunsuke

#7 lore raw

2021年9月30日(木) 20:18 Zhu Yanjun <[email protected]>:

Quoted 66 lines · Shunsuke Mie
Quoted 55 lines

2021年9月30日(木) 16:23 Zhu Yanjun <[email protected]>:

Quoted 53 lines · Shunsuke Mie
Quoted 37 lines

2021年9月30日(木) 15:37 Zhu Yanjun <[email protected]>:

Quoted 32 lines · Shunsuke Mie
Quoted 17 lines

Implement a new provider method for dma-buf base memory registration.

---
 providers/rxe/rxe.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
providers/rxe/rxe.c +1 -0
index 3c3ea8bb..84e00e60 100644
@@ -239,6 +239,26 @@ static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
        return &vmr->ibv_mr;
 }
 
+static struct ibv_mr *rxe_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,

providers/rxe/rxe.c static struct ibv_mr *rxe_reg_mr(struct ibv_pd *pd, void *addr, size_t length, · line 252

Quoted 10 lines
providers/rxe/rxe.c +10 -0
+                                       size_t length, uint64_t iova, int fd,
+                                       int access)
+{
+       struct verbs_mr *vmr;
+       int ret;
+
+       vmr = malloc(sizeof(*vmr));
+       if (!vmr)
+               return NULL;
+

Do we need to set vmr to zero like the following?

Quoted 3 lines

memset(vmr, 0, sizeof(*vmr));

Zhu Yanjun

Quoted 8 lines

Thank you for your quick response.

I think it is better to clear the vmr. Actually the mlx5 driver allocates the vmr using calloc().

In addition, rxe_reg_mr() (not rxe_reg_dmabuf_mr()) is used the malloc and not clear the vmr. I think It has to be fixed too. Should I make another patch to fix this problem?

Yes. Please.

Zhu Yanjun

Quoted 1 line
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines

Thanks a lot.
Shunsuke

~

I looked into the vmr more, but there was no need to clear it. Moreover, some implementations also use malloc without memory clear.

Quoted 9 lines

I confronted a lot of problems with memory not initialization. And a latest bug is https://www.spinics.net/lists/linux-rdma/msg105001.html

So it is a good habit to clear a newly allocated memory.

Zhu Yanjun

Quoted 2 lines

Thanks,
Shunsuke

Ok, I understand that thought. I'll send a patch.

Shunsuke.