Skip to contentSkip to conversation

ib_uverbs: XRC RCV qp implementation

PATCH New Jack Morgenstein 1 participant 1 message May 19, 2010 InfiniBand

lore
Share & tools
lore

Jack Morgenstein

author
#0 lore raw

PATCH v3 2/4 ib_uverbs: XRC RCV qp implementation

Implement XRC target QPs (xrc rcv qps) for userspace. The basic verbs are: create/modify/query/destroy.

In addition, added two additional verbs -- register and unregister.

The motivation for register/unregister comes from MPI.
MPI requires XRC receive QPs which are not destroyed when the creating process terminates (but persist so that other processes may still use them as XRC targets).

Solution: Userspace requests that a QP be created in kernel space. Each userspace process using that QP (i.e. receiving packets on an XRC SRQ via the qp), registers with that QP. When the last userspace process unregisters with the QP, it is destroyed. Unregistration is also part of userspace process cleanup, so there is no leakage.

This patch implements the kernel procedures to implement the following (new) libibverbs API:
ibv_create_xrc_rcv_qp
ibv_modify_xrc_rcv_qp
ibv_query_xrc_rcv_qp
ibv_destroy_xrc_rcv_qp

ibv_reg_xrc_rcv_qp
ibv_unreg_xrc_rcv_qp

Note that users who wish to make use of the reg/unreg capability should never call the destroy verb -- the XRC RCV qp is automatically destroyed when all registered processes have unregistered (or terminated). In this case, the process which called "create" may also unregister (or do nothing, and when it terminates, its reference to that QP is removed).

Thus, usage is:
Either: create/modify/query/destroy_xrc_rcv_qp
Or: create/modify/query/reg/unreg_xrc_rcv_qp

V3: renamed ib_xrc_rcv_table_cleanup to ib_xrc_rcv_qp_table_cleanup for consistency.

Quoted 1 line

From 6571bf63dbbbca0e95faabd81a9e57f908d7df17 Mon Sep 17 00:00:00 2001

From: Jack Morgenstein <[email protected]>
Date: Mon, 10 May 2010 20:23:24 +0300
Subject: [PATCH 4/4] ib_uverbs: XRC RCV qp implementation.

Signed-off-by Jack Morgenstein <[email protected]>
---
 drivers/infiniband/core/uverbs.h      |    8 +
 drivers/infiniband/core/uverbs_cmd.c  |  304 +++++++++++++++++++++++++++++++++
 drivers/infiniband/core/uverbs_main.c |   15 ++
 drivers/infiniband/hw/mlx4/main.c     |    4 +-
 include/rdma/ib_user_verbs.h          |   87 +++++++++-
 5 files changed, 416 insertions(+), 2 deletions(-)
drivers/infiniband/core/uverbs.h +8 -0
Index: infiniband/drivers/infiniband/core/uverbs.h
===================================================================
@@ -175,6 +175,8 @@ void ib_uverbs_qp_event_handler(struct i
 void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr);
 void ib_uverbs_event_handler(struct ib_event_handler *handler,
 			     struct ib_event *event);
+void ib_uverbs_xrc_rcv_qp_event_handler(struct ib_event *event,
+					void *context_ptr);
 void ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev,
 			    struct ib_xrcd *xrcd);
 

@@ -214,5 +216,11 @@ IB_UVERBS_DECLARE_CMD(destroy_srq);
 IB_UVERBS_DECLARE_CMD(create_xrc_srq);
 IB_UVERBS_DECLARE_CMD(open_xrcd);
 IB_UVERBS_DECLARE_CMD(close_xrcd);
+IB_UVERBS_DECLARE_CMD(create_xrc_rcv_qp);
+IB_UVERBS_DECLARE_CMD(modify_xrc_rcv_qp);
+IB_UVERBS_DECLARE_CMD(query_xrc_rcv_qp);
+IB_UVERBS_DECLARE_CMD(destroy_xrc_rcv_qp);
+IB_UVERBS_DECLARE_CMD(reg_xrc_rcv_qp);
+IB_UVERBS_DECLARE_CMD(unreg_xrc_rcv_qp);
 
 #endif /* UVERBS_H */
drivers/infiniband/core/uverbs_cmd.c +308 -0

311 lines not on this page · raw message

drivers/infiniband/core/uverbs_main.c +15 -0
Index: infiniband/drivers/infiniband/core/uverbs_main.c
===================================================================
@@ -111,6 +111,12 @@ static ssize_t (*uverbs_cmd_table[])(str
 	[IB_USER_VERBS_CMD_CREATE_XRC_SRQ]	= ib_uverbs_create_xrc_srq,
 	[IB_USER_VERBS_CMD_OPEN_XRCD]		= ib_uverbs_open_xrcd,
 	[IB_USER_VERBS_CMD_CLOSE_XRCD]		= ib_uverbs_close_xrcd,
+	[IB_USER_VERBS_CMD_CREATE_XRC_RCV_QP]	= ib_uverbs_create_xrc_rcv_qp,
+	[IB_USER_VERBS_CMD_MODIFY_XRC_RCV_QP]	= ib_uverbs_modify_xrc_rcv_qp,
+	[IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP]	= ib_uverbs_query_xrc_rcv_qp,
+	[IB_USER_VERBS_CMD_DESTROY_XRC_RCV_QP]	= ib_uverbs_destroy_xrc_rcv_qp,
+	[IB_USER_VERBS_CMD_REG_XRC_RCV_QP]	= ib_uverbs_reg_xrc_rcv_qp,
+	[IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP]	= ib_uverbs_unreg_xrc_rcv_qp,
 };
 
 static void ib_uverbs_add_one(struct ib_device *device);

@@ -246,6 +252,8 @@ static int ib_uverbs_cleanup_ucontext(st
 		kfree(uobj);
 	}
 
+	ib_xrc_rcv_qp_table_cleanup(context->device, file);
+
 	mutex_lock(&file->device->xrcd_tree_mutex);
 	list_for_each_entry_safe(uobj, tmp, &context->xrcd_list, list) {
 		struct ib_xrcd *xrcd = uobj->object;

@@ -479,6 +487,13 @@ void ib_uverbs_qp_event_handler(struct i
 				&uobj->events_reported);
 }
 
+void ib_uverbs_xrc_rcv_qp_event_handler(struct ib_event *event,
+					void *context_ptr)
+{
+	ib_uverbs_async_handler(context_ptr, event->element.xrc_qp_num,
+				event->event, NULL, NULL);
+}
+
 void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr)
 {
 	struct ib_uevent_object *uobj;
include/rdma/ib_user_verbs.h +79 -1

90 lines not on this page · raw message

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html