Skip to contentSkip to conversation

vmw_pvrdma: Use physical resource ids from device

PATCH Superseded Adit Ranadive 1 participant 4 messages Oct 22, 2019 InfiniBand

lore
Share & tools
lore

Adit Ranadive

author
#0 lore raw

v3 cover vmw_pvrdma: Use physical resource ids from device

Changelog:
v3:
- Remove create qp flag
- Kernel abi update after cxgb3 removal
v2:
- Used kernel-headers/update script to generate headers commit
v1:
- Added a separate patch for the kernel header
- Dropped the ABI version check
- Added a create qp flag and an in-band check to indicate support
v0:
- https://patchwork.kernel.org/patch/10946987/

Hi,

Here is a patchset for enabling exposing physical resource ids for vmw_pvrdma userspace provider.

rdma-core PR:
- https://github.com/linux-rdma/rdma-core/pull/581

Thanks,
Adit

Adit Ranadive (2):
Update kernel headers
kernel-headers: Remove cxgb3-abi.h

Bryan Tan (1):
vmw_pvrdma: Use resource ids from physical device if available

 CMakeLists.txt                             |  1 -
 kernel-headers/CMakeLists.txt              |  3 +-
 kernel-headers/rdma/cxgb3-abi.h            | 82 ----------------------
 kernel-headers/rdma/ib_user_ioctl_verbs.h  | 22 ++++++
 kernel-headers/rdma/rdma_user_ioctl_cmds.h | 22 ------
 kernel-headers/rdma/rvt-abi.h              | 66 +++++++++++++++++
 kernel-headers/rdma/vmw_pvrdma-abi.h       |  5 ++
 providers/vmw_pvrdma/pvrdma-abi.h          |  2 +-
 providers/vmw_pvrdma/pvrdma.h              |  1 +
 providers/vmw_pvrdma/qp.c                  | 23 +++---
 10 files changed, 108 insertions(+), 119 deletions(-)
 delete mode 100644 kernel-headers/rdma/cxgb3-abi.h
 create mode 100644 kernel-headers/rdma/rvt-abi.h
Signature · 1 line
2.18.1
author
#1 lore raw

PATCH v3 1/3 Update kernel headers

To commit ?? ("RDMA/vmw_pvrdma: Use resource ids from physical device if available")

Signed-off-by Adit Ranadive <[email protected]>
---
 kernel-headers/CMakeLists.txt              |  2 +-
 kernel-headers/rdma/cxgb3-abi.h            | 82 ----------------------
 kernel-headers/rdma/ib_user_ioctl_verbs.h  | 22 ++++++
 kernel-headers/rdma/rdma_user_ioctl_cmds.h | 22 ------
 kernel-headers/rdma/rvt-abi.h              | 66 +++++++++++++++++
 kernel-headers/rdma/vmw_pvrdma-abi.h       |  5 ++
 6 files changed, 94 insertions(+), 105 deletions(-)
 delete mode 100644 kernel-headers/rdma/cxgb3-abi.h
 create mode 100644 kernel-headers/rdma/rvt-abi.h
kernel-headers/CMakeLists.txt +1 -1
index 50bc77e6ab6e..543e0ea1408a 100644
@@ -1,6 +1,5 @@
 publish_internal_headers(rdma
   rdma/bnxt_re-abi.h
-  rdma/cxgb3-abi.h
   rdma/cxgb4-abi.h
   rdma/efa-abi.h
   rdma/hns-abi.h

@@ -23,6 +22,7 @@ publish_internal_headers(rdma
   rdma/rdma_user_ioctl.h
   rdma/rdma_user_ioctl_cmds.h
   rdma/rdma_user_rxe.h
+  rdma/rvt-abi.h
   rdma/siw-abi.h
   rdma/vmw_pvrdma-abi.h
   )
kernel-headers/rdma/cxgb3-abi.h deleted +0 -82

82 lines not on this page · raw message

kernel-headers/rdma/ib_user_ioctl_verbs.h +22 -0
index 72c7fc75f960..9019b2d906ea 100644
@@ -173,4 +173,26 @@ struct ib_uverbs_query_port_resp_ex {
 	__u8  reserved[6];
 };
 
+enum rdma_driver_id {
+	RDMA_DRIVER_UNKNOWN,
+	RDMA_DRIVER_MLX5,
+	RDMA_DRIVER_MLX4,
+	RDMA_DRIVER_CXGB3,
+	RDMA_DRIVER_CXGB4,
+	RDMA_DRIVER_MTHCA,
+	RDMA_DRIVER_BNXT_RE,
+	RDMA_DRIVER_OCRDMA,
+	RDMA_DRIVER_NES,
+	RDMA_DRIVER_I40IW,
+	RDMA_DRIVER_VMW_PVRDMA,
+	RDMA_DRIVER_QEDR,
+	RDMA_DRIVER_HNS,
+	RDMA_DRIVER_USNIC,
+	RDMA_DRIVER_RXE,
+	RDMA_DRIVER_HFI1,
+	RDMA_DRIVER_QIB,
+	RDMA_DRIVER_EFA,
+	RDMA_DRIVER_SIW,
+};
+
 #endif
kernel-headers/rdma/rdma_user_ioctl_cmds.h +0 -22
index b8bb285f6b2a..7b1ec806f8f9 100644
@@ -84,26 +84,4 @@ struct ib_uverbs_ioctl_hdr {
 	struct ib_uverbs_attr  attrs[0];
 };
 
-enum rdma_driver_id {
-	RDMA_DRIVER_UNKNOWN,
-	RDMA_DRIVER_MLX5,
-	RDMA_DRIVER_MLX4,
-	RDMA_DRIVER_CXGB3,
-	RDMA_DRIVER_CXGB4,
-	RDMA_DRIVER_MTHCA,
-	RDMA_DRIVER_BNXT_RE,
-	RDMA_DRIVER_OCRDMA,
-	RDMA_DRIVER_NES,
-	RDMA_DRIVER_I40IW,
-	RDMA_DRIVER_VMW_PVRDMA,
-	RDMA_DRIVER_QEDR,
-	RDMA_DRIVER_HNS,
-	RDMA_DRIVER_USNIC,
-	RDMA_DRIVER_RXE,
-	RDMA_DRIVER_HFI1,
-	RDMA_DRIVER_QIB,
-	RDMA_DRIVER_EFA,
-	RDMA_DRIVER_SIW,
-};
-
 #endif
kernel-headers/rdma/rvt-abi.h new file +66 -0

66 lines not on this page · raw message

kernel-headers/rdma/vmw_pvrdma-abi.h +5 -0
index 6e73f0274e41..f8b638c73371 100644
@@ -179,6 +179,11 @@ struct pvrdma_create_qp {
 	__aligned_u64 qp_addr;
 };
 
+struct pvrdma_create_qp_resp {
+	__u32 qpn;
+	__u32 qp_handle;
+};
+
 /* PVRDMA masked atomic compare and swap */
 struct pvrdma_ex_cmp_swap {
 	__aligned_u64 swap_val;
Signature · 1 line
2.18.1
author
#2 lore raw

PATCH v3 2/3 kernel-headers: Remove cxgb3-abi.h

In reply to Adit Ranadive

Driver is removed from kernel. Also, update cmake to not build library.

Signed-off-by Adit Ranadive <[email protected]>
---
 CMakeLists.txt                | 1 -
 kernel-headers/CMakeLists.txt | 1 -
 2 files changed, 2 deletions(-)
CMakeLists.txt +0 -1
index fc17ef36cf24..cdc8d5444e70 100644
@@ -616,7 +616,6 @@ add_subdirectory(librdmacm/man)
 # Providers
 if (HAVE_COHERENT_DMA)
 add_subdirectory(providers/bnxt_re)
-add_subdirectory(providers/cxgb3) # NO SPARSE
 add_subdirectory(providers/cxgb4) # NO SPARSE
 add_subdirectory(providers/efa)
 add_subdirectory(providers/efa/man)
kernel-headers/CMakeLists.txt +0 -1
index 543e0ea1408a..c4464c579320 100644
@@ -59,7 +59,6 @@ endfunction()
 # Transform the kernel ABIs used by the providers
 rdma_kernel_provider_abi(
   rdma/bnxt_re-abi.h
-  rdma/cxgb3-abi.h
   rdma/cxgb4-abi.h
   rdma/efa-abi.h
   rdma/hns-abi.h
Signature · 1 line
2.18.1
author
#3 lore raw

PATCH v3 3/3 vmw_pvrdma: Use resource ids from physical device if available

In reply to Adit Ranadive

From: Bryan Tan <[email protected]>

This is the accompanying userspace change to allow applications use physical resource ids if provided by the driver/device.

Reviewed-by Jorgen Hansen <[email protected]>
Signed-off-by Adit Ranadive <[email protected]>
Signed-off-by Bryan Tan <[email protected]>
---
 providers/vmw_pvrdma/pvrdma-abi.h |  2 +-
 providers/vmw_pvrdma/pvrdma.h     |  1 +
 providers/vmw_pvrdma/qp.c         | 23 ++++++++++++-----------
 3 files changed, 14 insertions(+), 12 deletions(-)
providers/vmw_pvrdma/pvrdma-abi.h +1 -1
index 77db9ddd1bb7..1a4c3c8a98f2 100644
@@ -55,7 +55,7 @@ DECLARE_DRV_CMD(user_pvrdma_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
 DECLARE_DRV_CMD(user_pvrdma_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
 		pvrdma_create_cq, pvrdma_create_cq_resp);
 DECLARE_DRV_CMD(user_pvrdma_create_qp, IB_USER_VERBS_CMD_CREATE_QP,
-		pvrdma_create_qp, empty);
+		pvrdma_create_qp, pvrdma_create_qp_resp);
 DECLARE_DRV_CMD(user_pvrdma_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ,
 		pvrdma_create_srq, pvrdma_create_srq_resp);
 DECLARE_DRV_CMD(user_pvrdma_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT,
providers/vmw_pvrdma/pvrdma.h +1 -0
index d90bd8096664..bb4a2db08d14 100644
@@ -170,6 +170,7 @@ struct pvrdma_qp {
 	struct pvrdma_wq		sq;
 	struct pvrdma_wq		rq;
 	int				is_srq;
+	uint32_t			qp_handle;
 };
 
 struct pvrdma_ah {
providers/vmw_pvrdma/qp.c +12 -11
index ef429db93a43..10784314f5b5 100644
@@ -211,9 +211,8 @@ struct ibv_qp *pvrdma_create_qp(struct ibv_pd *pd,
 {
 	struct pvrdma_device *dev = to_vdev(pd->context->device);
 	struct user_pvrdma_create_qp cmd;
-	struct ib_uverbs_create_qp_resp resp;
+	struct user_pvrdma_create_qp_resp resp = {};
 	struct pvrdma_qp *qp;
-	int ret;
 	int is_srq = !!(attr->srq);
 
 	attr->cap.max_send_sge = max_t(uint32_t, 1U, attr->cap.max_send_sge);

@@ -282,14 +281,16 @@ struct ibv_qp *pvrdma_create_qp(struct ibv_pd *pd,
 	cmd.rbuf_size = qp->rbuf.length;
 	cmd.qp_addr = (uintptr_t) qp;
 
-	ret = ibv_cmd_create_qp(pd, &qp->ibv_qp, attr,
-				&cmd.ibv_cmd, sizeof(cmd),
-				&resp, sizeof(resp));
-
-	if (ret)
+	if (ibv_cmd_create_qp(pd, &qp->ibv_qp, attr, &cmd.ibv_cmd, sizeof(cmd),
+			      &resp.ibv_resp, sizeof(resp)))
 		goto err_free;
 
-	to_vctx(pd->context)->qp_tbl[qp->ibv_qp.qp_num & 0xFFFF] = qp;
+	if (resp.drv_payload.qp_handle != 0)
+		qp->qp_handle = resp.drv_payload.qp_handle;
+	else
+		qp->qp_handle = qp->ibv_qp.qp_num;
+
+	to_vctx(pd->context)->qp_tbl[qp->qp_handle & 0xFFFF] = qp;
 
 	/* If set, each WR submitted to the SQ generate a completion entry */
 	if (attr->sq_sig_all)

@@ -414,7 +415,7 @@ int pvrdma_destroy_qp(struct ibv_qp *ibqp)
 	free(qp->rq.wrid);
 	pvrdma_free_buf(&qp->rbuf);
 	pvrdma_free_buf(&qp->sbuf);
-	ctx->qp_tbl[ibqp->qp_num & 0xFFFF] = NULL;
+	ctx->qp_tbl[qp->qp_handle & 0xFFFF] = NULL;
 	free(qp);
 
 	return 0;

@@ -547,7 +548,7 @@ out:
 	if (nreq) {
 		udma_to_device_barrier();
 		pvrdma_write_uar_qp(ctx->uar,
-				    PVRDMA_UAR_QP_SEND | ibqp->qp_num);
+				    PVRDMA_UAR_QP_SEND | qp->qp_handle);
 	}
 
 	pthread_spin_unlock(&qp->sq.lock);

@@ -630,7 +631,7 @@ int pvrdma_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
 out:
 	if (nreq)
 		pvrdma_write_uar_qp(ctx->uar,
-				    PVRDMA_UAR_QP_RECV | ibqp->qp_num);
+				    PVRDMA_UAR_QP_RECV | qp->qp_handle);
 
 	pthread_spin_unlock(&qp->rq.lock);
 	return ret;
Signature · 1 line
2.18.1