cm_id resource tracking

-----Original Message-----
From: [email protected] [mailto:linux-rdma- [email protected]] On Behalf Of Steve Wise
Sent: Tuesday, January 30, 2018 10:59 AM
To: [email protected]; Jason Gunthorpe <jgg-VPRAkNaXOzVWk0Htik3J/[email protected]>
Cc: [email protected]; [email protected]
Subject: [PATCH RFC 1/2] RDMA/CM: move rdma_id_private into
include/rdma/rdma_cm.h

So the resource tracking services in core/nldev.c can see useful information about cm_ids.

There are other approaches. I just moved rdma_id_private to develop this prototype quickly, and it was simple. Other approaches include:

1) move the nldev cm_id dumpit functions into cma.c, and have nldev.c call it. This, however puts a ib_core->rdma_cm module dependency which makes the two modules interdependent in both directions. Thus, rdma_cm would have to be merged into ib_core. This might not be a bad idea with all the kernel rdma ULPs now using the rdma_cm.

2) move the specific attributes that are being dumped from the rdma_id_private struct to the rdma_cm_id struct, so nldev.c has access to them.

Signed-off-by Steve Wise <[email protected]>

---
 drivers/infiniband/core/cma.c | 40 ----------------------------------------
 include/rdma/rdma_cm.h        | 41
+++++++++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 40 deletions(-)

drivers/infiniband/core/cma.c index renamed from drivers/infiniband/core/cma.c +0 -0

e66963c..72ad52b 100644

drivers/infiniband/core/cma.c +0 -40

@@ -327,46 +327,6 @@ struct ib_device *cma_get_ib_dev(struct cma_device
 *cma_dev)
  * We do this by disabling removal notification while a callback is in process,
  * and reporting it after the callback completes.
  */
-struct rdma_id_private {
-	struct rdma_cm_id	id;
-
-	struct rdma_bind_list	*bind_list;
-	struct hlist_node	node;
-	struct list_head	list; /* listen_any_list or cma_device.list */
-	struct list_head	listen_list; /* per device listens */
-	struct cma_device	*cma_dev;
-	struct list_head	mc_list;
-
-	int			internal_id;
-	enum rdma_cm_state	state;
-	spinlock_t		lock;
-	struct mutex		qp_mutex;
-
-	struct completion	comp;
-	atomic_t		refcount;
-	struct mutex		handler_mutex;
-
-	int			backlog;
-	int			timeout_ms;
-	struct ib_sa_query	*query;
-	int			query_id;
-	union {
-		struct ib_cm_id	*ib;
-		struct iw_cm_id	*iw;
-	} cm_id;
-
-	u32			seq_num;
-	u32			qkey;
-	u32			qp_num;
-	pid_t			owner;
-	u32			options;
-	u8			srq;
-	u8			tos;
-	bool			tos_set;
-	u8			reuseaddr;
-	u8			afonly;
-	enum ib_gid_type	gid_type;
-};
 
 struct cma_multicast {

drivers/infiniband/core/cma.c +0 -0

 	struct rdma_id_private *id_priv;

include/rdma/rdma_cm.h index renamed from include/rdma/rdma_cm.h +0 -0

6538a5c..5984225 100644

include/rdma/rdma_cm.h +41 -1

@@ -157,6 +157,47 @@ struct rdma_cm_id {
 	u8			 port_num;
 };
 
+struct rdma_id_private {
+	struct rdma_cm_id	id;
+
+	struct rdma_bind_list	*bind_list;
+	struct hlist_node	node;
+	struct list_head	list; /* listen_any_list or cma_device.list */
+	struct list_head	listen_list; /* per device listens */
+	struct cma_device	*cma_dev;
+	struct list_head	mc_list;
+
+	int			internal_id;
+	enum rdma_cm_state	state;
+	spinlock_t		lock;
+	struct mutex		qp_mutex;
+
+	struct completion	comp;
+	atomic_t		refcount;
+	struct mutex		handler_mutex;
+
+	int			backlog;
+	int			timeout_ms;
+	struct ib_sa_query	*query;
+	int			query_id;
+	union {
+		struct ib_cm_id	*ib;
+		struct iw_cm_id	*iw;
+	} cm_id;
+
+	u32			seq_num;
+	u32			qkey;
+	u32			qp_num;
+	pid_t			owner;
+	u32			options;
+	u8			srq;
+	u8			tos;
+	bool			tos_set;
+	u8			reuseaddr;
+	u8			afonly;
+	enum ib_gid_type	gid_type;
+};
+
 /**
  * rdma_create_id - Create an RDMA identifier.
  *
--

1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at
http://vger.kernel.org/majordomo-info.html

-----Original Message-----
From: [email protected] [mailto:linux-rdma- [email protected]] On Behalf Of Steve Wise
Sent: Tuesday, January 30, 2018 10:59 AM
To: [email protected]; Jason Gunthorpe <jgg-VPRAkNaXOzVWk0Htik3J/[email protected]>
Cc: [email protected]; [email protected]
Subject: [PATCH RFC 2/2] RDMA/nldev: provide detailed CM_ID information

Implement RDMA nldev netlink interface to get detailed CM_ID information.

Because cm_id's are attached to rdma devices in various work queue contexts, the pid and task information at device-attach time is sometimes not useful. For example, an nvme/f host connection ends up being bound to a device in a work queue context and the resulting pid at attach time no longer exists after connection setup. So instead we mark all cm_id's created via the rdma_ucm as "user", and all others as "kernel". This required tweaking the restrack code a little. It also required wrapping some rdma_cm functions to allow passing the module name string.

Signed-off-by Steve Wise <[email protected]>

---
 drivers/infiniband/core/cma.c      |  55 ++++++---
 drivers/infiniband/core/nldev.c    | 245
+++++++++++++++++++++++++++++++++++++
 drivers/infiniband/core/restrack.c |  29 ++++-
 drivers/infiniband/core/ucma.c     |   8 +-
 include/rdma/rdma_cm.h             |  24 +++-
 include/rdma/restrack.h            |   4 +
 include/uapi/rdma/rdma_netlink.h   |  30 +++++
 7 files changed, 365 insertions(+), 30 deletions(-)

drivers/infiniband/core/cma.c index renamed from drivers/infiniband/core/cma.c +0 -0

72ad52b..51fbfa1 100644

drivers/infiniband/core/cma.c +3 -0

@@ -465,6 +465,9 @@ static void _cma_attach_to_dev(struct rdma_id_private
 *id_priv,
 	id_priv->id.route.addr.dev_addr.transport =
 		rdma_node_get_transport(cma_dev->device->node_type);
 	list_add_tail(&id_priv->list, &cma_dev->id_list);
+	id_priv->id.res.type = RDMA_RESTRACK_CM_ID;
+	id_priv->id.res.kern_name = id_priv->id.caller;
+	rdma_restrack_add(&id_priv->id.res);
 }
 

static void cma_attach_to_dev(struct rdma_id_private *id_priv, @@ -737,10
+740,10 @@ static void cma_deref_id(struct rdma_id_private *id_priv)

drivers/infiniband/core/cma.c +8 -5

 		complete(&id_priv->comp);
 }
 
-struct rdma_cm_id *rdma_create_id(struct net *net,
-				  rdma_cm_event_handler event_handler,
-				  void *context, enum rdma_port_space ps,
-				  enum ib_qp_type qp_type)
+struct rdma_cm_id *__rdma_create_id(struct net *net,
+				    rdma_cm_event_handler event_handler,
+				    void *context, enum rdma_port_space ps,
+				    enum ib_qp_type qp_type, const char *caller)
 {
 	struct rdma_id_private *id_priv;

@@ -748,7 +751,10 @@ struct rdma_cm_id *rdma_create_id(struct net *net,
 	if (!id_priv)
 		return ERR_PTR(-ENOMEM);
 
-	id_priv->owner = task_pid_nr(current);
+	if (caller)
+		id_priv->id.caller = caller;
+	else
+		id_priv->owner = task_pid_nr(current);
 	id_priv->state = RDMA_CM_IDLE;
 	id_priv->id.context = context;
 	id_priv->id.event_handler = event_handler; @@ -768,7 +774,7 @@

struct rdma_cm_id *rdma_create_id(struct net *net,

drivers/infiniband/core/cma.c +7 -5

 	return &id_priv->id;
 }
-EXPORT_SYMBOL(rdma_create_id);
+EXPORT_SYMBOL(__rdma_create_id);
 
 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)  {

@@ -1628,6 +1634,7 @@ void rdma_destroy_id(struct rdma_cm_id *id)
 	mutex_unlock(&id_priv->handler_mutex);
 
 	if (id_priv->cma_dev) {
+		rdma_restrack_del(&id_priv->id.res);
 		if (rdma_cap_ib_cm(id_priv->id.device, 1)) {
 			if (id_priv->cm_id.ib)
 				ib_destroy_cm_id(id_priv->cm_id.ib);

@@ -1786,9 +1793,10 @@ static struct rdma_id_private
 *cma_new_conn_id(struct rdma_cm_id *listen_id,
 		ib_event->param.req_rcvd.primary_path->service_id;
 	int ret;
 
-	id = rdma_create_id(listen_id->route.addr.dev_addr.net,
+	id = __rdma_create_id(listen_id->route.addr.dev_addr.net,
 			    listen_id->event_handler, listen_id->context,
-			    listen_id->ps, ib_event->param.req_rcvd.qp_type);
+			    listen_id->ps, ib_event->param.req_rcvd.qp_type,
+			    listen_id->caller);
 	if (IS_ERR(id))
 		return NULL;

@@ -1843,8 +1851,8 @@ static struct rdma_id_private
 *cma_new_udp_id(struct rdma_cm_id *listen_id,
 	struct net *net = listen_id->route.addr.dev_addr.net;
 	int ret;
 
-	id = rdma_create_id(net, listen_id->event_handler, listen_id->context,
-			    listen_id->ps, IB_QPT_UD);
+	id = __rdma_create_id(net, listen_id->event_handler, listen_id-
 >context,
+			      listen_id->ps, IB_QPT_UD, listen_id->caller);
 	if (IS_ERR(id))

drivers/infiniband/core/cma.c +5 -4

 		return NULL;

@@ -2110,10 +2118,11 @@ static int iw_conn_req_handler(struct iw_cm_id
 *cm_id,
 		goto out;
 
 	/* Create a new RDMA id for the new IW CM ID */
-	new_cm_id = rdma_create_id(listen_id->id.route.addr.dev_addr.net,
-				   listen_id->id.event_handler,
-				   listen_id->id.context,
-				   RDMA_PS_TCP, IB_QPT_RC);
+	new_cm_id = __rdma_create_id(listen_id->id.route.addr.dev_addr.net,
+				     listen_id->id.event_handler,
+				     listen_id->id.context,
+				     RDMA_PS_TCP, IB_QPT_RC,
+				     listen_id->id.caller);
 	if (IS_ERR(new_cm_id)) {
 		ret = -ENOMEM;

drivers/infiniband/core/cma.c +2 -2

 		goto out;

@@ -2238,8 +2247,8 @@ static void cma_listen_on_dev(struct rdma_id_private
 *id_priv,
 	if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev-
 >device, 1))
 		return;
 
-	id = rdma_create_id(net, cma_listen_handler, id_priv, id_priv->id.ps,
-			    id_priv->id.qp_type);
+	id = __rdma_create_id(net, cma_listen_handler, id_priv, id_priv->id.ps,
+			      id_priv->id.qp_type, id_priv->id.caller);
 	if (IS_ERR(id))

drivers/infiniband/core/cma.c +3 -1

 		return;

@@ -3347,8 +3356,10 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct
 sockaddr *addr)
 
 	return 0;
 err2:
-	if (id_priv->cma_dev)
+	if (id_priv->cma_dev) {
+		rdma_restrack_del(&id_priv->id.res);
 		cma_release_dev(id_priv);
+	}
 err1:
 	cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);

drivers/infiniband/core/cma.c +6 -2

 	return ret;

@@ -3731,14 +3742,18 @@ static int cma_send_sidr_rep(struct
 rdma_id_private *id_priv,
 	return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);  }
 
-int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param
 *conn_param)
+int __rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param
 *conn_param,
+		  const char *caller)
 {
 	struct rdma_id_private *id_priv;
 	int ret;
 
 	id_priv = container_of(id, struct rdma_id_private, id);
 
-	id_priv->owner = task_pid_nr(current);
+	if (caller)
+		id_priv->id.caller = caller;
+	else
+		id_priv->owner = task_pid_nr(current);
 

drivers/infiniband/core/cma.c +1 -1

 	if (!cma_comp(id_priv, RDMA_CM_CONNECT))
 		return -EINVAL;

@@ -3778,7 +3793,7 @@ int rdma_accept(struct rdma_cm_id *id, struct
 rdma_conn_param *conn_param)
 	rdma_reject(id, NULL, 0);
 	return ret;
 }
-EXPORT_SYMBOL(rdma_accept);
+EXPORT_SYMBOL(__rdma_accept);
 
 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)  { diff --git

a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index fa8655e..a4091b5 100644

drivers/infiniband/core/nldev.c +17 -0

@@ -34,6 +34,7 @@
 #include <linux/pid.h>
 #include <linux/pid_namespace.h>
 #include <net/netlink.h>
+#include <rdma/rdma_cm.h>
 #include <rdma/rdma_netlink.h>
 
 #include "core_priv.h"

@@ -71,6 +72,22 @@
 	[RDMA_NLDEV_ATTR_RES_PID]		= { .type = NLA_U32 },
 	[RDMA_NLDEV_ATTR_RES_KERN_NAME]		= { .type =
 NLA_NUL_STRING,
 						    .len = TASK_COMM_LEN },
+	[RDMA_NLDEV_ATTR_RES_CM_ID]		= { .type =
 NLA_NESTED },
+	[RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY]	= { .type =
 NLA_NESTED },
+	[RDMA_NLDEV_ATTR_RES_PS]		= { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_RES_IPV4_SADDR]	= {
+				.len = FIELD_SIZEOF(struct iphdr, saddr) },
+	[RDMA_NLDEV_ATTR_RES_IPV4_DADDR]	= {
+				.len = FIELD_SIZEOF(struct iphdr, saddr) },
+	[RDMA_NLDEV_ATTR_RES_IPV6_SADDR]	= {
+				.len = FIELD_SIZEOF(struct ipv6hdr, saddr) },
+	[RDMA_NLDEV_ATTR_RES_IPV6_DADDR]	= {
+				.len = FIELD_SIZEOF(struct ipv6hdr, saddr) },
+	[RDMA_NLDEV_ATTR_RES_IP_SPORT]		= { .type = NLA_U16 },
+	[RDMA_NLDEV_ATTR_RES_IP_DPORT]		= { .type = NLA_U16 },
+	[RDMA_NLDEV_ATTR_RES_DEV_TYPE]		= { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_RES_TRANSPORT_TYPE]	= { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_RES_NETWORK_TYPE]	= { .type = NLA_U8 },

};

static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device) @@ -
182,6 +199,7 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device)

Hi Steve,

Quoted 10 lines

-----Original Message-----
From: [email protected] [mailto:linux-rdma- [email protected]] On Behalf Of Steve Wise
Sent: Tuesday, January 30, 2018 10:59 AM
To: [email protected]; Jason Gunthorpe <jgg-VPRAkNaXOzVWk0Htik3J/[email protected]>
Cc: [email protected]; [email protected]
Subject: [PATCH RFC 1/2] RDMA/CM: move rdma_id_private into
include/rdma/rdma_cm.h

So the resource tracking services in core/nldev.c can see useful

Quoted 3 lines

about cm_ids.

There are other approaches. I just moved rdma_id_private to develop

Quoted 3 lines

prototype quickly, and it was simple. Other approaches include:

1) move the nldev cm_id dumpit functions into cma.c, and have nldev.c

Quoted 2 lines

This, however puts a ib_core->rdma_cm module dependency which makes the two modules interdependent in both directions. Thus, rdma_cm would have

Quoted 1 line

be merged into ib_core. This might not be a bad idea with all the

Quoted 3 lines

ULPs now using the rdma_cm.

2) move the specific attributes that are being dumped from the

rdma_id_private

Quoted 2 lines

struct to the rdma_cm_id struct, so nldev.c has access to them.

I prefer to see drivers/infiniband/core/cma_priv.h to contain rdma_cm_id_private structure.
This allows not to expose that to ULPs which includes

This also includes keeping cm_id state enum within cma_priv.h. Core files include this new cma_priv.h.

This approach is similar to core_priv.h where structures are not place in include/rdma/ib_*.h.
I actually have that cleanup patch as part of other patches that I have

doing.
But it was not worth enough as stand along patch, so didn't send it. But not that this RFC is present, I think it should be done in cma_priv.h.

Quoted 1 line

182,6 +199,7 @@ static int fill_res_info(struct sk_buff *msg, struct

drivers/infiniband/core/nldev.c static int fill_res_info(struct sk_buff *msg, struct ib_device *device) · line 202

Quoted 5 lines

*device)

drivers/infiniband/core/nldev.c +1 -0

 		[RDMA_RESTRACK_PD] = "pd",
 		[RDMA_RESTRACK_CQ] = "cq",
 		[RDMA_RESTRACK_QP] = "qp",
+		[RDMA_RESTRACK_CM_ID] = "cm_id",

May be rdmacm_id a better name to avoid confusion with ib_cm/iw_cm ids?

-----Original Message-----
From: [email protected] [mailto:linux-rdma- [email protected]] On Behalf Of Steve Wise
Sent: Wednesday, January 31, 2018 2:56 PM
To: Parav Pandit <parav-VPRAkNaXOzVWk0Htik3J/[email protected]>; [email protected]; Jason Gunthorpe <jgg-VPRAkNaXOzVWk0Htik3J/[email protected]>
Cc: [email protected]; [email protected]
Subject: RE: [PATCH RFC 2/2] RDMA/nldev: provide detailed CM_ID information

Quoted 1 line

Quoted 1 line

182,6 +199,7 @@ static int fill_res_info(struct sk_buff *msg, struct

ib_device

Quoted 7 lines

drivers/infiniband/core/nldev.c static int fill_res_info(struct sk_buff *msg, struct ib_device *device) · line 202

Quoted 5 lines

*device)

drivers/infiniband/core/nldev.c +1 -0

 		[RDMA_RESTRACK_PD] = "pd",
 		[RDMA_RESTRACK_CQ] = "cq",
 		[RDMA_RESTRACK_QP] = "qp",
+		[RDMA_RESTRACK_CM_ID] = "cm_id",

May be rdmacm_id a better name to avoid confusion with ib_cm/iw_cm ids?

Eventually, I could see the ib_cm and/or iw_cm info also getting dumped as part

of this. So perhaps the name "cm_id" is ok?

I prefer it also because its shorter (lazy typer! :) )

Quoted 21 lines

-----Original Message-----
From: [email protected] [mailto:linux-rdma- [email protected]] On Behalf Of Steve Wise
Sent: Wednesday, January 31, 2018 2:56 PM
To: Parav Pandit <parav-VPRAkNaXOzVWk0Htik3J/[email protected]>; [email protected]; Jason Gunthorpe <jgg-VPRAkNaXOzVWk0Htik3J/[email protected]>
Cc: [email protected]; [email protected]
Subject: RE: [PATCH RFC 2/2] RDMA/nldev: provide detailed CM_ID information

Quoted 1 line

Quoted 1 line

182,6 +199,7 @@ static int fill_res_info(struct sk_buff *msg, struct

ib_device

Quoted 7 lines

Review

Quoted 5 lines

*device)

(quoted diff) +1 -0

 		[RDMA_RESTRACK_PD] = "pd",
 		[RDMA_RESTRACK_CQ] = "cq",
 		[RDMA_RESTRACK_QP] = "qp",
+		[RDMA_RESTRACK_CM_ID] = "cm_id",

May be rdmacm_id a better name to avoid confusion with ib_cm/iw_cm ids?

Quoted 1 line

Eventually, I could see the ib_cm and/or iw_cm info also getting dumped as part

Yes. But it is still coming through id of rdmacm_id as transport id information.

Quoted 1 line

of this. So perhaps the name "cm_id" is ok?

It's just that longer name removes the ambiguity. I am not too particular about this particular naming.

Quoted 1 line

I prefer it also because its shorter (lazy typer! :) )

Sure, shorter names are useful to end user.
But I think this is handled well at iproute2 tool level.
For example below two commands achieves the same task.
#ip address show
#ip a s
So may be shorter naming at command level is better left at user space tool level.

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Implement RDMA nldev netlink interface to get detailed CM_ID information.

Because cm_id's are attached to rdma devices in various work queue contexts, the pid and task information at device-attach time is sometimes not useful. For example, an nvme/f host connection ends up being bound to a device in a work queue context and the resulting pid at attach time no longer exists after connection setup. So instead we mark all cm_id's created via the rdma_ucm as "user", and all others as "kernel". This required tweaking the restrack code a little. It also required wrapping some rdma_cm functions to allow passing the module name string.

Signed-off-by Steve Wise <[email protected]>

---
 drivers/infiniband/core/cma.c      |  55 ++++++---
 drivers/infiniband/core/nldev.c    | 245 +++++++++++++++++++++++++++++++++++++
 drivers/infiniband/core/restrack.c |  29 ++++-
 drivers/infiniband/core/ucma.c     |   8 +-
 include/rdma/rdma_cm.h             |  24 +++-
 include/rdma/restrack.h            |   4 +
 include/uapi/rdma/rdma_netlink.h   |  30 +++++
 7 files changed, 365 insertions(+), 30 deletions(-)

drivers/infiniband/core/cma.c +0 -0

index 72ad52b..51fbfa1 100644

drivers/infiniband/core/cma.c +5 -4

+	rdma_restrack_add(&id_priv->id.res);
 }
 
 static void cma_attach_to_dev(struct rdma_id_private *id_priv,

@@ -737,10 +740,10 @@ static void cma_deref_id(struct rdma_id_private *id_priv)
 		complete(&id_priv->comp);
 }
 
-struct rdma_cm_id *rdma_create_id(struct net *net,
-				  rdma_cm_event_handler event_handler,
-				  void *context, enum rdma_port_space ps,
-				  enum ib_qp_type qp_type)
+struct rdma_cm_id *__rdma_create_id(struct net *net,
+				    rdma_cm_event_handler event_handler,
+				    void *context, enum rdma_port_space ps,
+				    enum ib_qp_type qp_type, const char *caller)
 {
 	struct rdma_id_private *id_priv;
 

drivers/infiniband/core/cma.c +1 -1

@@ -768,7 +774,7 @@ struct rdma_cm_id *rdma_create_id(struct net *net,
 
 	return &id_priv->id;
 }
-EXPORT_SYMBOL(rdma_create_id);
+EXPORT_SYMBOL(__rdma_create_id);
 
 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 {

drivers/infiniband/core/cma.c +19 -12

 	if (IS_ERR(id))
 		return NULL;
 

@@ -1843,8 +1851,8 @@ static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
 	struct net *net = listen_id->route.addr.dev_addr.net;
 	int ret;
 
-	id = rdma_create_id(net, listen_id->event_handler, listen_id->context,
-			    listen_id->ps, IB_QPT_UD);
+	id = __rdma_create_id(net, listen_id->event_handler, listen_id->context,
+			      listen_id->ps, IB_QPT_UD, listen_id->caller);
 	if (IS_ERR(id))
 		return NULL;
 

@@ -2110,10 +2118,11 @@ static int iw_conn_req_handler(struct iw_cm_id *cm_id,
 		goto out;
 
 	/* Create a new RDMA id for the new IW CM ID */
-	new_cm_id = rdma_create_id(listen_id->id.route.addr.dev_addr.net,
-				   listen_id->id.event_handler,
-				   listen_id->id.context,
-				   RDMA_PS_TCP, IB_QPT_RC);
+	new_cm_id = __rdma_create_id(listen_id->id.route.addr.dev_addr.net,
+				     listen_id->id.event_handler,
+				     listen_id->id.context,
+				     RDMA_PS_TCP, IB_QPT_RC,
+				     listen_id->id.caller);
 	if (IS_ERR(new_cm_id)) {
 		ret = -ENOMEM;
 		goto out;

@@ -2238,8 +2247,8 @@ static void cma_listen_on_dev(struct rdma_id_private *id_priv,
 	if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev->device, 1))
 		return;
 
-	id = rdma_create_id(net, cma_listen_handler, id_priv, id_priv->id.ps,
-			    id_priv->id.qp_type);
+	id = __rdma_create_id(net, cma_listen_handler, id_priv, id_priv->id.ps,
+			      id_priv->id.qp_type, id_priv->id.caller);
 	if (IS_ERR(id))
 		return;
 

@@ -3347,8 +3356,10 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
 
 	return 0;
 err2:
-	if (id_priv->cma_dev)
+	if (id_priv->cma_dev) {
+		rdma_restrack_del(&id_priv->id.res);
 		cma_release_dev(id_priv);
+	}
 err1:
 	cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
 	return ret;

@@ -3731,14 +3742,18 @@ static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
 	return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
 }
 
-int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
+int __rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
+		  const char *caller)
 {
 	struct rdma_id_private *id_priv;
 	int ret;
 
 	id_priv = container_of(id, struct rdma_id_private, id);
 
-	id_priv->owner = task_pid_nr(current);
+	if (caller)
+		id_priv->id.caller = caller;
+	else
+		id_priv->owner = task_pid_nr(current);
 
 	if (!cma_comp(id_priv, RDMA_CM_CONNECT))
 		return -EINVAL;

@@ -3778,7 +3793,7 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
 	rdma_reject(id, NULL, 0);
 	return ret;
 }
-EXPORT_SYMBOL(rdma_accept);
+EXPORT_SYMBOL(__rdma_accept);
 
 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
 {

drivers/infiniband/core/nldev.c +1 -0

index fa8655e..a4091b5 100644
@@ -34,6 +34,7 @@
 #include <linux/pid.h>
 #include <linux/pid_namespace.h>
 #include <net/netlink.h>
+#include <rdma/rdma_cm.h>
 #include <rdma/rdma_netlink.h>
 
 #include "core_priv.h"

drivers/infiniband/core/nldev.c +132 -0

+	[RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY]	= { .type = NLA_NESTED },
+	[RDMA_NLDEV_ATTR_RES_PS]		= { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_RES_IPV4_SADDR]	= {
+				.len = FIELD_SIZEOF(struct iphdr, saddr) },
+	[RDMA_NLDEV_ATTR_RES_IPV4_DADDR]	= {
+				.len = FIELD_SIZEOF(struct iphdr, saddr) },
+	[RDMA_NLDEV_ATTR_RES_IPV6_SADDR]	= {
+				.len = FIELD_SIZEOF(struct ipv6hdr, saddr) },
+	[RDMA_NLDEV_ATTR_RES_IPV6_DADDR]	= {
+				.len = FIELD_SIZEOF(struct ipv6hdr, saddr) },
+	[RDMA_NLDEV_ATTR_RES_IP_SPORT]		= { .type = NLA_U16 },
+	[RDMA_NLDEV_ATTR_RES_IP_DPORT]		= { .type = NLA_U16 },
+	[RDMA_NLDEV_ATTR_RES_DEV_TYPE]		= { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_RES_TRANSPORT_TYPE]	= { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_RES_NETWORK_TYPE]	= { .type = NLA_U8 },
 };
 
 static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device)

@@ -182,6 +199,7 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device)
 		[RDMA_RESTRACK_PD] = "pd",
 		[RDMA_RESTRACK_CQ] = "cq",
 		[RDMA_RESTRACK_QP] = "qp",
+		[RDMA_RESTRACK_CM_ID] = "cm_id",
 	};
 
 	struct rdma_restrack_root *res = &device->res;

@@ -284,6 +302,99 @@ static int fill_res_qp_entry(struct sk_buff *msg,
 	return -EMSGSIZE;
 }
 
+static int fill_res_cm_id_entry(struct sk_buff *msg,
+				struct rdma_cm_id *cm_id, uint32_t port)
+{
+	struct rdma_id_private *id_priv;
+	struct nlattr *entry_attr;
+
+	if (port && port != cm_id->port_num)
+		return 0;
+
+	id_priv = container_of(cm_id, struct rdma_id_private, id);
+	entry_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY);
+	if (!entry_attr)
+		goto out;
+
+	if (cm_id->port_num &&
+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, cm_id->port_num))
+		goto err;
+
+	if (id_priv->qp_num &&
+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, id_priv->qp_num))
+		goto err;
+
+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PS, cm_id->ps))
+		goto err;
+
+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, cm_id->qp_type))
+		goto err;
+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, id_priv->state))
+		goto err;
+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_DEV_TYPE,
+		       id_priv->id.route.addr.dev_addr.dev_type))
+		goto err;
+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TRANSPORT_TYPE,
+		       id_priv->id.route.addr.dev_addr.transport))
+		goto err;
+
+	if (cm_id->route.addr.src_addr.ss_family == AF_INET) {
+		struct sockaddr_in *sin;
+
+		sin = (struct sockaddr_in *)&cm_id->route.addr.src_addr;
+		if (nla_put_in_addr(msg, RDMA_NLDEV_ATTR_RES_IPV4_SADDR,
+				    sin->sin_addr.s_addr))
+			goto err;
+		if (nla_put_net16(msg, RDMA_NLDEV_ATTR_RES_IP_SPORT,
+				  sin->sin_port))
+			goto err;
+
+		sin = (struct sockaddr_in *)&cm_id->route.addr.dst_addr;
+		if (nla_put_in_addr(msg, RDMA_NLDEV_ATTR_RES_IPV4_DADDR,
+				    sin->sin_addr.s_addr))
+			goto err;
+		if (nla_put_net16(msg, RDMA_NLDEV_ATTR_RES_IP_DPORT,
+				  sin->sin_port))
+			goto err;
+	} else {
+		struct sockaddr_in6 *sin6;
+
+		sin6 = (struct sockaddr_in6 *)&cm_id->route.addr.src_addr;
+		if (nla_put_in6_addr(msg, RDMA_NLDEV_ATTR_RES_IPV6_SADDR,
+				     &sin6->sin6_addr))
+			goto err;
+		if (nla_put_net16(msg, RDMA_NLDEV_ATTR_RES_IP_SPORT,
+				  sin6->sin6_port))
+			goto err;
+
+		sin6 = (struct sockaddr_in6 *)&cm_id->route.addr.dst_addr;
+		if (nla_put_in6_addr(msg, RDMA_NLDEV_ATTR_RES_IPV6_DADDR,
+				     &sin6->sin6_addr))
+			goto err;
+		if (nla_put_net16(msg, RDMA_NLDEV_ATTR_RES_IP_DPORT,
+				  sin6->sin6_port))
+			goto err;
+	}
+
+	if (id_priv->id.caller) {
+		if (nla_put_string(msg, RDMA_NLDEV_ATTR_RES_KERN_NAME,
+				   id_priv->id.caller))
+			goto err;
+	} else {
+		/* CMA keeps the owning pid. */
+		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PID, id_priv->owner))
+			goto err;
+	}
+
+	nla_nest_end(msg, entry_attr);
+	return 0;
+
+err:
+	nla_nest_cancel(msg, entry_attr);
+out:
+	return -EMSGSIZE;
+}
+
 static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
 			  struct netlink_ext_ack *extack)
 {

@@ -686,6 +797,137 @@ static int nldev_res_get_qp_dumpit(struct sk_buff *skb,
 	return ret;
 }
 
+static int nldev_res_get_cm_id_dumpit(struct sk_buff *skb,
+				      struct netlink_callback *cb)
+{
+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+	struct rdma_restrack_entry *res;
+	int err, ret = 0, idx = 0;
+	struct nlattr *table_attr;
+	struct ib_device *device;
+	int start = cb->args[0];
+	struct rdma_cm_id *cm_id = NULL;
+	struct nlmsghdr *nlh;
+	u32 index, port = 0;
+
+	err = nlmsg_parse(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
+			  nldev_policy, NULL);
+	/*
+	 * Right now, we are expecting the device index to get QP information,
+	 * but it is possible to extend this code to return all devices in
+	 * one shot by checking the existence of RDMA_NLDEV_ATTR_DEV_INDEX.
+	 * if it doesn't exist, we will iterate over all devices.
+	 *
+	 * But it is not needed for now.
+	 */

drivers/infiniband/core/nldev.c +90 -0

+		0, NLM_F_MULTI);
+
+	if (fill_nldev_handle(skb, device)) {
+		ret = -EMSGSIZE;
+		goto err;
+	}
+
+	table_attr = nla_nest_start(skb, RDMA_NLDEV_ATTR_RES_CM_ID);
+	if (!table_attr) {
+		ret = -EMSGSIZE;
+		goto err;
+	}
+
+	down_read(&device->res.rwsem);
+	hash_for_each_possible(device->res.hash, res, node,
+			       RDMA_RESTRACK_CM_ID) {
+		if (idx < start)
+			goto next;
+
+		if ((rdma_is_kernel_res(res) &&
+		     task_active_pid_ns(current) != &init_pid_ns) ||
+		    (!rdma_is_kernel_res(res) &&
+		     task_active_pid_ns(current) !=
+		     task_active_pid_ns(res->task)))
+			/*
+			 * 1. Kernel QPs should be visible in init namsapce only
+			 * 2. Preent only QPs visible in the current namespace
+			 */
+			goto next;
+
+		if (!rdma_restrack_get(res))
+			/*
+			 * Resource is under release now, but we are not
+			 * relesing lock now, so it will be released in
+			 * our next pass, once we will get ->next pointer.
+			 */
+			goto next;
+
+		cm_id = container_of(res, struct rdma_cm_id, res);
+
+		up_read(&device->res.rwsem);
+		ret = fill_res_cm_id_entry(skb, cm_id, port);
+		down_read(&device->res.rwsem);
+		/*
+		 * Return resource back, but it won't be released till
+		 * the &device->res.rwsem will be released for write.
+		 */
+		rdma_restrack_put(res);
+
+		if (ret == -EMSGSIZE)
+			/*
+			 * There is a chance to optimize here.
+			 * It can be done by using list_prepare_entry
+			 * and list_for_each_entry_continue afterwards.
+			 */
+			break;
+		if (ret)
+			goto res_err;
+next:		idx++;
+	}
+	up_read(&device->res.rwsem);
+
+	nla_nest_end(skb, table_attr);
+	nlmsg_end(skb, nlh);
+	cb->args[0] = idx;
+
+	/*
+	 * No more CM_IDs to fill, cancel the message and
+	 * return 0 to mark end of dumpit.
+	 */
+	if (!cm_id)
+		goto err;
+
+	put_device(&device->dev);
+	return skb->len;
+
+res_err:
+	nla_nest_cancel(skb, table_attr);
+	up_read(&device->res.rwsem);
+
+err:
+	nlmsg_cancel(skb, nlh);
+
+err_index:
+	put_device(&device->dev);
+	return ret;
+}
 static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
 	[RDMA_NLDEV_CMD_GET] = {
 		.doit = nldev_get_doit,

@@ -712,6 +954,9 @@ static int nldev_res_get_qp_dumpit(struct sk_buff *skb,
 		 * too.
 		 */
 	},
+	[RDMA_NLDEV_CMD_RES_CM_ID_GET] = {
+		.dump = nldev_res_get_cm_id_dumpit,
+	},
 };
 
 void __init nldev_init(void)

drivers/infiniband/core/restrack.c +28 -1

index 857637b..bb13169 100644
@@ -3,6 +3,7 @@
  * Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved.
  */
 
+#include <rdma/rdma_cm.h>
 #include <rdma/ib_verbs.h>
 #include <rdma/restrack.h>
 #include <linux/mutex.h>

@@ -67,6 +68,7 @@ static struct ib_device *res_to_dev(struct rdma_restrack_entry *res)
 	struct ib_pd *pd;
 	struct ib_cq *cq;
 	struct ib_qp *qp;
+	struct rdma_cm_id *cm_id;
 
 	switch (type) {
 	case RDMA_RESTRACK_PD:

@@ -85,6 +87,10 @@ static struct ib_device *res_to_dev(struct rdma_restrack_entry *res)
 		xrcd = container_of(res, struct ib_xrcd, res);
 		dev = xrcd->device;
 		break;
+	case RDMA_RESTRACK_CM_ID:
+		cm_id = container_of(res, struct rdma_cm_id, res);
+		dev = cm_id->device;
+		break;
 	default:
 		WARN_ONCE(true, "Wrong resource tracking type %u\n", type);
 		return NULL;

@@ -93,6 +99,27 @@ static struct ib_device *res_to_dev(struct rdma_restrack_entry *res)
 	return dev;
 }
 
+static bool res_is_user(struct rdma_restrack_entry *res)
+{
+	enum rdma_restrack_type type = res->type;
+	bool is_user;
+
+	switch (type) {
+	case RDMA_RESTRACK_CM_ID: {
+		struct rdma_cm_id *cm_id;
+
+		cm_id = container_of(res, struct rdma_cm_id, res);
+		is_user = !cm_id->caller;
+		break;
+	}
+	default:
+		is_user = !uaccess_kernel();
+		break;
+	}
+
+	return is_user;
+}
+
 void rdma_restrack_add(struct rdma_restrack_entry *res)
 {
 	struct ib_device *dev = res_to_dev(res);

@@ -100,7 +127,7 @@ void rdma_restrack_add(struct rdma_restrack_entry *res)
 	if (!dev)
 		return;
 
-	if (!uaccess_kernel()) {
+	if (res_is_user(res)) {
 		get_task_struct(current);
 		res->task = current;
 		res->kern_name = NULL;

drivers/infiniband/core/ucma.c +4 -4

index d67219d..f7f0282 100644
@@ -476,8 +476,8 @@ static ssize_t ucma_create_id(struct ucma_file *file, const char __user *inbuf,
 		return -ENOMEM;
 
 	ctx->uid = cmd.uid;
-	ctx->cm_id = rdma_create_id(current->nsproxy->net_ns,
-				    ucma_event_handler, ctx, cmd.ps, qp_type);
+	ctx->cm_id = __rdma_create_id(current->nsproxy->net_ns,
+			      ucma_event_handler, ctx, cmd.ps, qp_type, NULL);
 	if (IS_ERR(ctx->cm_id)) {
 		ret = PTR_ERR(ctx->cm_id);
 		goto err1;

@@ -1084,12 +1084,12 @@ static ssize_t ucma_accept(struct ucma_file *file, const char __user *inbuf,
 	if (cmd.conn_param.valid) {
 		ucma_copy_conn_param(ctx->cm_id, &conn_param, &cmd.conn_param);
 		mutex_lock(&file->mut);
-		ret = rdma_accept(ctx->cm_id, &conn_param);
+		ret = __rdma_accept(ctx->cm_id, &conn_param, NULL);
 		if (!ret)
 			ctx->uid = cmd.uid;
 		mutex_unlock(&file->mut);
 	} else
-		ret = rdma_accept(ctx->cm_id, NULL);
+		ret = __rdma_accept(ctx->cm_id, NULL, NULL);
 
 	ucma_put_ctx(ctx);
 	return ret;

include/rdma/rdma_cm.h +19 -5

index 5984225..73dac9b 100644
@@ -155,6 +155,12 @@ struct rdma_cm_id {
 	enum rdma_port_space	 ps;
 	enum ib_qp_type		 qp_type;
 	u8			 port_num;
+	const char *caller;
+
+	/*
+	 * Internal to RDMA/core, don't use in the drivers
+	 */
+	struct rdma_restrack_entry     res;
 };
 
 struct rdma_id_private {

@@ -198,6 +204,11 @@ struct rdma_id_private {
 	enum ib_gid_type	gid_type;
 };
 
+struct rdma_cm_id *__rdma_create_id(struct net *net,
+				  rdma_cm_event_handler event_handler,
+				  void *context, enum rdma_port_space ps,
+				  enum ib_qp_type qp_type, const char *caller);
+
 /**
  * rdma_create_id - Create an RDMA identifier.
  *

@@ -210,10 +221,9 @@ struct rdma_id_private {
  *
  * The id holds a reference on the network namespace until it is destroyed.
  */
-struct rdma_cm_id *rdma_create_id(struct net *net,
-				  rdma_cm_event_handler event_handler,
-				  void *context, enum rdma_port_space ps,
-				  enum ib_qp_type qp_type);
+#define rdma_create_id(net, event_handler, context, ps, qp_type) \
+	__rdma_create_id((net), (event_handler), (context), (ps), (qp_type), \
+			 KBUILD_MODNAME)
 
 /**
   * rdma_destroy_id - Destroys an RDMA identifier.

@@ -325,6 +335,9 @@ int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
  */
 int rdma_listen(struct rdma_cm_id *id, int backlog);
 
+int __rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
+		  const char *caller);
+
 /**
  * rdma_accept - Called to accept a connection request or response.
  * @id: Connection identifier associated with the request.

@@ -340,7 +353,8 @@ int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
  * state of the qp associated with the id is modified to error, such that any
  * previously posted receive buffers would be flushed.
  */
-int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param);
+#define rdma_accept(id, conn_param) \
+	__rdma_accept((id), (conn_param),  KBUILD_MODNAME)
 
 /**
  * rdma_notify - Notifies the RDMA CM of an asynchronous event that has

include/rdma/restrack.h +4 -0

index c2d8116..a794e0e 100644
@@ -33,6 +33,10 @@ enum rdma_restrack_type {
 	 */
 	RDMA_RESTRACK_XRCD,
 	/**
+	 * @RDMA_RESTRACK_CM_ID: Connection Manager ID (CM_ID)
+	 */
+	RDMA_RESTRACK_CM_ID,
+	/**
 	 * @RDMA_RESTRACK_MAX: Last entry, used for array dclarations
 	 */
 	RDMA_RESTRACK_MAX

include/uapi/rdma/rdma_netlink.h +7 -0

index 17e59be..4ed21ee 100644
@@ -240,6 +240,8 @@ enum rdma_nldev_command {
 
 	RDMA_NLDEV_CMD_RES_QP_GET, /* can dump */
 
+	RDMA_NLDEV_CMD_RES_CM_ID_GET, /* can dump */
+
 	RDMA_NLDEV_NUM_OPS
 };
 

@@ -352,6 +354,34 @@ enum rdma_nldev_attr {
 	 */
 	RDMA_NLDEV_ATTR_RES_KERN_NAME,		/* string */
 
+	RDMA_NLDEV_ATTR_RES_CM_ID,		/* nested table */
+	RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY,	/* nested table */
+	/*
+	 * rdma_cm_id port space.
+	 */

include/uapi/rdma/rdma_netlink.h +13 -1

+	/*
+	 * ARPHRD_INFINIBAND, ARPHRD_ETHER, ...
+	 */
+	RDMA_NLDEV_ATTR_RES_DEV_TYPE,		/* u8 */
+	/*
+	 * enum enum rdma_transport_type (IB, IWARP, ...)
+	 */
+	RDMA_NLDEV_ATTR_RES_TRANSPORT_TYPE,	/* u8 */
+	/*
+	 * enum rdma_network_type (IB, IPv4, IPv6,...)
+	 */
+	RDMA_NLDEV_ATTR_RES_NETWORK_TYPE,	/* u8 */
+
 	RDMA_NLDEV_ATTR_MAX
 };
 #endif /* _UAPI_RDMA_NETLINK_H */
--

1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 1 line

Implement RDMA nldev netlink interface to get detailed CM_ID

Quoted 2 lines

Because cm_id's are attached to rdma devices in various work queue

Quoted 1 line

the pid and task information at device-attach time is sometimes not

Quoted 2 lines

For example, an nvme/f host connection ends up being bound to a device in a work queue context and the resulting pid at attach time no longer

Quoted 9 lines

after connection setup. So instead we mark all cm_id's created via the rdma_ucm as "user", and all others as "kernel". This required tweaking the restrack code a little. It also required wrapping some rdma_cm functions to allow passing the module name string.

Signed-off-by Steve Wise <[email protected]>

---
 drivers/infiniband/core/cma.c      |  55 ++++++---
 drivers/infiniband/core/nldev.c    | 245

+++++++++++++++++++++++++++++++++++++

Review

Quoted 8 lines

 drivers/infiniband/core/restrack.c |  29 ++++-
 drivers/infiniband/core/ucma.c     |   8 +-
 include/rdma/rdma_cm.h             |  24 +++-
 include/rdma/restrack.h            |   4 +
 include/uapi/rdma/rdma_netlink.h   |  30 +++++
 7 files changed, 365 insertions(+), 30 deletions(-)

(quoted diff) +0 -0

drivers/infiniband/core/cma.c static void _cma_attach_to_dev(struct

Quoted 4 lines

index 72ad52b..51fbfa1 100644

drivers/infiniband/core/cma.c +0 -0

@@ -465,6 +465,9 @@ static void _cma_attach_to_dev(struct

rdma_id_private *id_priv,

drivers/infiniband/core/cma.c static void _cma_attach_to_dev(struct rdma_id_private *id_priv, · line 469

Quoted 5 lines

drivers/infiniband/core/cma.c +2 -0

 	id_priv->id.route.addr.dev_addr.transport =
 		rdma_node_get_transport(cma_dev->device->node_type);
 	list_add_tail(&id_priv->list, &cma_dev->id_list);
+	id_priv->id.res.type = RDMA_RESTRACK_CM_ID;
+	id_priv->id.res.kern_name = id_priv->id.caller;

Steve, I don't like it, I worked hard to hide it from the users of

and don't see reason why the same trick as with ib_create_cq/ib_create_pd won't
work here.

drivers/infiniband/core/cma.c static void cma_deref_id(struct rdma_id_private

Quoted 6 lines

drivers/infiniband/core/cma.c +1 -0

+	rdma_restrack_add(&id_priv->id.res);
 }
 
 static void cma_attach_to_dev(struct rdma_id_private *id_priv,

@@ -737,10 +740,10 @@ static void cma_deref_id(struct rdma_id_private

*id_priv)

Quoted 3 lines

drivers/infiniband/core/cma.c +0 -0

 		complete(&id_priv->comp);
 }
 

drivers/infiniband/core/cma.c static void cma_deref_id(struct rdma_id_private *id_priv)

Quoted 8 lines

drivers/infiniband/core/cma.c +4 -4

-struct rdma_cm_id *rdma_create_id(struct net *net,
-				  rdma_cm_event_handler event_handler,
-				  void *context, enum rdma_port_space ps,
-				  enum ib_qp_type qp_type)
+struct rdma_cm_id *__rdma_create_id(struct net *net,
+				    rdma_cm_event_handler event_handler,
+				    void *context, enum rdma_port_space ps,
+				    enum ib_qp_type qp_type, const char

Quoted 7 lines

drivers/infiniband/core/cma.c +0 -0

 {
 	struct rdma_id_private *id_priv;
 

@@ -748,7 +751,10 @@ struct rdma_cm_id *rdma_create_id(struct net *net,
 	if (!id_priv)
 		return ERR_PTR(-ENOMEM);
 

drivers/infiniband/core/cma.c struct rdma_cm_id *rdma_create_id(struct net *net, · line 757

Quoted 5 lines

drivers/infiniband/core/cma.c +4 -1

-	id_priv->owner = task_pid_nr(current);
+	if (caller)
+		id_priv->id.caller = caller;
+	else
+		id_priv->owner = task_pid_nr(current);

See the comment above

drivers/infiniband/core/cma.c struct rdma_cm_id *rdma_create_id(struct net *net, · line 760

Quoted 4 lines

drivers/infiniband/core/cma.c +0 -0

 	id_priv->state = RDMA_CM_IDLE;
 	id_priv->id.context = context;
 	id_priv->id.event_handler = event_handler;

Not saying that we need to do it now, but it is important to write, most probably we can drop certain initialization from rdma_create_id() adn reuse rdma_restrack_put/_get.

drivers/infiniband/core/cma.c struct rdma_cm_id *rdma_create_id(struct net *net, · line 779

Quoted 8 lines

drivers/infiniband/core/cma.c +1 -1

@@ -768,7 +774,7 @@ struct rdma_cm_id *rdma_create_id(struct net *net,
 
 	return &id_priv->id;
 }
-EXPORT_SYMBOL(rdma_create_id);
+EXPORT_SYMBOL(__rdma_create_id);
 
 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp

drivers/infiniband/core/cma.c void rdma_destroy_id(struct rdma_cm_id *id) · line 1637

Quoted 6 lines

drivers/infiniband/core/cma.c +1 -0

 {

@@ -1628,6 +1634,7 @@ void rdma_destroy_id(struct rdma_cm_id *id)
 	mutex_unlock(&id_priv->handler_mutex);
 
 	if (id_priv->cma_dev) {
+		rdma_restrack_del(&id_priv->id.res);

You should count all created cm_ids and not only binded.

drivers/infiniband/core/cma.c static struct rdma_id_private

Quoted 5 lines

drivers/infiniband/core/cma.c +0 -0

 		if (rdma_cap_ib_cm(id_priv->id.device, 1)) {
 			if (id_priv->cm_id.ib)
 				ib_destroy_cm_id(id_priv->cm_id.ib);

@@ -1786,9 +1793,10 @@ static struct rdma_id_private

*cma_new_conn_id(struct rdma_cm_id *listen_id,

drivers/infiniband/core/cma.c static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,

Quoted 7 lines

drivers/infiniband/core/cma.c +1 -2

 		ib_event->param.req_rcvd.primary_path->service_id;
 	int ret;
 
-	id = rdma_create_id(listen_id->route.addr.dev_addr.net,
+	id = __rdma_create_id(listen_id->route.addr.dev_addr.net,
 			    listen_id->event_handler, listen_id->context,
-			    listen_id->ps,

drivers/infiniband/core/cma.c static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id, · line 1799

Quoted 2 lines

drivers/infiniband/core/cma.c +2 -0

+			    listen_id->ps, ib_event->param.req_rcvd.qp_type,
+			    listen_id->caller);

I think the cleanest way will be to create some struct and pass pointer to

you can unfold all relevant data inside of __rdma_create_id().

drivers/infiniband/core/cma.c static struct rdma_id_private

Quoted 4 lines

drivers/infiniband/core/cma.c +0 -0

 	if (IS_ERR(id))
 		return NULL;
 

@@ -1843,8 +1851,8 @@ static struct rdma_id_private

*cma_new_udp_id(struct rdma_cm_id *listen_id,

drivers/infiniband/core/cma.c static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,

Quoted 4 lines

drivers/infiniband/core/cma.c +0 -1

 	struct net *net = listen_id->route.addr.dev_addr.net;
 	int ret;
 
-	id = rdma_create_id(net, listen_id->event_handler,

drivers/infiniband/core/cma.c static int iw_conn_req_handler(struct iw_cm_id

Quoted 8 lines

drivers/infiniband/core/cma.c +2 -1

-			    listen_id->ps, IB_QPT_UD);
+	id = __rdma_create_id(net, listen_id->event_handler, listen_id-
 context,
+			      listen_id->ps, IB_QPT_UD, listen_id->caller);
 	if (IS_ERR(id))
 		return NULL;

@@ -2110,10 +2118,11 @@ static int iw_conn_req_handler(struct iw_cm_id

*cm_id,

drivers/infiniband/core/cma.c static void cma_listen_on_dev(struct

Quoted 16 lines

drivers/infiniband/core/cma.c +5 -4

 		goto out;
 
 	/* Create a new RDMA id for the new IW CM ID */
-	new_cm_id = rdma_create_id(listen_id->id.route.addr.dev_addr.net,
-				   listen_id->id.event_handler,
-				   listen_id->id.context,
-				   RDMA_PS_TCP, IB_QPT_RC);
+	new_cm_id = __rdma_create_id(listen_id->id.route.addr.dev_addr.net,
+				     listen_id->id.event_handler,
+				     listen_id->id.context,
+				     RDMA_PS_TCP, IB_QPT_RC,
+				     listen_id->id.caller);
 	if (IS_ERR(new_cm_id)) {
 		ret = -ENOMEM;
 		goto out;

@@ -2238,8 +2247,8 @@ static void cma_listen_on_dev(struct

rdma_id_private *id_priv,

drivers/infiniband/core/cma.c static void cma_listen_on_dev(struct · line 2242

Quoted 5 lines

drivers/infiniband/core/cma.c +0 -1

 	if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev-
 device, 1))
 		return;
 
-	id = rdma_create_id(net, cma_listen_handler, id_priv,

drivers/infiniband/core/cma.c static void cma_listen_on_dev(struct · line 2251

Quoted 2 lines

drivers/infiniband/core/cma.c +1 -1

-			    id_priv->id.qp_type);
+	id = __rdma_create_id(net, cma_listen_handler, id_priv,

drivers/infiniband/core/cma.c int rdma_bind_addr(struct rdma_cm_id *id, struct

Quoted 5 lines

drivers/infiniband/core/cma.c +1 -0

+			      id_priv->id.qp_type, id_priv->id.caller);
 	if (IS_ERR(id))
 		return;

@@ -3347,8 +3356,10 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct

sockaddr *addr)

drivers/infiniband/core/cma.c static int cma_send_sidr_rep(struct

Quoted 12 lines

drivers/infiniband/core/cma.c +3 -1

 	return 0;
 err2:
-	if (id_priv->cma_dev)
+	if (id_priv->cma_dev) {
+		rdma_restrack_del(&id_priv->id.res);
 		cma_release_dev(id_priv);
+	}
 err1:
 	cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
 	return ret;

@@ -3731,14 +3742,18 @@ static int cma_send_sidr_rep(struct

rdma_id_private *id_priv,

drivers/infiniband/core/cma.c static int cma_send_sidr_rep(struct rdma_id_private *id_priv,

Quoted 4 lines

drivers/infiniband/core/cma.c +0 -1

 	return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
 }
 
-int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param

*conn_param)

drivers/infiniband/core/cma.c static int cma_send_sidr_rep(struct · line 3745

Quoted 1 line

drivers/infiniband/core/cma.c +1 -0

+int __rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param

*conn_param,

drivers/infiniband/core/cma.c int rdma_accept(struct rdma_cm_id *id, struct

Quoted 16 lines

drivers/infiniband/core/cma.c +5 -1

+		  const char *caller)
 {
 	struct rdma_id_private *id_priv;
 	int ret;
 
 	id_priv = container_of(id, struct rdma_id_private, id);
 
-	id_priv->owner = task_pid_nr(current);
+	if (caller)
+		id_priv->id.caller = caller;
+	else
+		id_priv->owner = task_pid_nr(current);
 
 	if (!cma_comp(id_priv, RDMA_CM_CONNECT))
 		return -EINVAL;

@@ -3778,7 +3793,7 @@ int rdma_accept(struct rdma_cm_id *id, struct

rdma_conn_param *conn_param)

Quoted 6 lines

drivers/infiniband/core/cma.c +1 -1

 	rdma_reject(id, NULL, 0);
 	return ret;
 }
-EXPORT_SYMBOL(rdma_accept);
+EXPORT_SYMBOL(__rdma_accept);
 

Review

Quoted 3 lines

drivers/infiniband/core/cma.c +0 -0

 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
 {

(quoted diff) +0 -0

Quoted 12 lines

index fa8655e..a4091b5 100644

drivers/infiniband/core/nldev.c +1 -0

@@ -34,6 +34,7 @@
 #include <linux/pid.h>
 #include <linux/pid_namespace.h>
 #include <net/netlink.h>
+#include <rdma/rdma_cm.h>
 #include <rdma/rdma_netlink.h>
 
 #include "core_priv.h"

drivers/infiniband/core/nldev.c line 73

Quoted 2 lines

drivers/infiniband/core/nldev.c +0 -0

@@ -71,6 +72,22 @@
 	[RDMA_NLDEV_ATTR_RES_PID]		= { .type = NLA_U32 },
 	[RDMA_NLDEV_ATTR_RES_KERN_NAME]		= { .type =

NLA_NUL_STRING,

drivers/infiniband/core/nldev.c line 75

Quoted 2 lines

drivers/infiniband/core/nldev.c +1 -0

 						    .len = TASK_COMM_LEN },
+	[RDMA_NLDEV_ATTR_RES_CM_ID]		= { .type =

NLA_NESTED },

I would like to use this opportunity. There is CM_ID, so users will be able to query nldev directly on this ID (once we implement .doit), but can we found a proper abstraction for other objects (PD, CQ, QP e.t.c.)?

I want to have that all resources will have something similar to

drivers/infiniband/core/nldev.c line 76

Quoted 2 lines

drivers/infiniband/core/nldev.c +1 -0

+	[RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY]	= { .type =

NLA_NESTED },

drivers/infiniband/core/nldev.c

Quoted 7 lines

drivers/infiniband/core/nldev.c +7 -0

+	[RDMA_NLDEV_ATTR_RES_PS]		= { .type = NLA_U32 },
+	[RDMA_NLDEV_ATTR_RES_IPV4_SADDR]	= {
+				.len = FIELD_SIZEOF(struct iphdr, saddr) },
+	[RDMA_NLDEV_ATTR_RES_IPV4_DADDR]	= {
+				.len = FIELD_SIZEOF(struct iphdr, saddr) },
+	[RDMA_NLDEV_ATTR_RES_IPV6_SADDR]	= {
+				.len = FIELD_SIZEOF(struct ipv6hdr, saddr)

drivers/infiniband/core/nldev.c line 85

Quoted 2 lines

drivers/infiniband/core/nldev.c +2 -0

+	[RDMA_NLDEV_ATTR_RES_IPV6_DADDR]	= {
+				.len = FIELD_SIZEOF(struct ipv6hdr, saddr)

drivers/infiniband/core/nldev.c

Quoted 8 lines

drivers/infiniband/core/nldev.c +5 -0

+	[RDMA_NLDEV_ATTR_RES_IP_SPORT]		= { .type = NLA_U16 },
+	[RDMA_NLDEV_ATTR_RES_IP_DPORT]		= { .type = NLA_U16 },
+	[RDMA_NLDEV_ATTR_RES_DEV_TYPE]		= { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_RES_TRANSPORT_TYPE]	= { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_RES_NETWORK_TYPE]	= { .type = NLA_U8 },
 };
 
 static int fill_nldev_handle(struct sk_buff *msg, struct ib_device

drivers/infiniband/core/nldev.c static int fill_res_info(struct sk_buff *msg, struct

Quoted 1 line

drivers/infiniband/core/nldev.c +0 -0

@@ -182,6 +199,7 @@ static int fill_res_info(struct sk_buff *msg, struct

ib_device *device)

Quoted 12 lines

drivers/infiniband/core/nldev.c +2 -0

 		[RDMA_RESTRACK_PD] = "pd",
 		[RDMA_RESTRACK_CQ] = "cq",
 		[RDMA_RESTRACK_QP] = "qp",
+		[RDMA_RESTRACK_CM_ID] = "cm_id",
 	};
 
 	struct rdma_restrack_root *res = &device->res;

@@ -284,6 +302,99 @@ static int fill_res_qp_entry(struct sk_buff *msg,
 	return -EMSGSIZE;
 }
 
+static int fill_res_cm_id_entry(struct sk_buff *msg,

drivers/infiniband/core/nldev.c static int fill_res_qp_entry(struct sk_buff *msg, · line 315

Quoted 10 lines

drivers/infiniband/core/nldev.c +10 -0

+				struct rdma_cm_id *cm_id, uint32_t port)
+{
+	struct rdma_id_private *id_priv;
+	struct nlattr *entry_attr;
+
+	if (port && port != cm_id->port_num)
+		return 0;
+
+	id_priv = container_of(cm_id, struct rdma_id_private, id);
+	entry_attr = nla_nest_start(msg,

RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY);

Quoted 22 lines

drivers/infiniband/core/nldev.c +20 -0

+	if (!entry_attr)
+		goto out;
+
+	if (cm_id->port_num &&
+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_PORT_INDEX, cm_id-
 port_num))
+		goto err;
+
+	if (id_priv->qp_num &&
+	    nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_LQPN, id_priv-
 qp_num))
+		goto err;
+
+	if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PS, cm_id->ps))
+		goto err;
+
+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TYPE, cm_id->qp_type))
+		goto err;
+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_STATE, id_priv->state))
+		goto err;
+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_DEV_TYPE,
+		       id_priv->id.route.addr.dev_addr.dev_type))

drivers/infiniband/core/nldev.c static int fill_res_qp_entry(struct sk_buff *msg,

Quoted 10 lines

drivers/infiniband/core/nldev.c +10 -0

+		goto err;
+	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_RES_TRANSPORT_TYPE,
+		       id_priv->id.route.addr.dev_addr.transport))
+		goto err;
+
+	if (cm_id->route.addr.src_addr.ss_family == AF_INET) {
+		struct sockaddr_in *sin;
+
+		sin = (struct sockaddr_in *)&cm_id->route.addr.src_addr;
+		if (nla_put_in_addr(msg,

RDMA_NLDEV_ATTR_RES_IPV4_SADDR,

drivers/infiniband/core/nldev.c static int fill_res_qp_entry(struct sk_buff *msg,

Quoted 8 lines

drivers/infiniband/core/nldev.c +8 -0

+				    sin->sin_addr.s_addr))
+			goto err;
+		if (nla_put_net16(msg, RDMA_NLDEV_ATTR_RES_IP_SPORT,
+				  sin->sin_port))
+			goto err;
+
+		sin = (struct sockaddr_in *)&cm_id->route.addr.dst_addr;
+		if (nla_put_in_addr(msg,

RDMA_NLDEV_ATTR_RES_IPV4_DADDR,

drivers/infiniband/core/nldev.c static int fill_res_qp_entry(struct sk_buff *msg,

Quoted 10 lines

drivers/infiniband/core/nldev.c +10 -0

+				    sin->sin_addr.s_addr))
+			goto err;
+		if (nla_put_net16(msg, RDMA_NLDEV_ATTR_RES_IP_DPORT,
+				  sin->sin_port))
+			goto err;
+	} else {
+		struct sockaddr_in6 *sin6;
+
+		sin6 = (struct sockaddr_in6 *)&cm_id->route.addr.src_addr;
+		if (nla_put_in6_addr(msg,

RDMA_NLDEV_ATTR_RES_IPV6_SADDR,

drivers/infiniband/core/nldev.c static int fill_res_qp_entry(struct sk_buff *msg,

Quoted 8 lines

drivers/infiniband/core/nldev.c +8 -0

+				     &sin6->sin6_addr))
+			goto err;
+		if (nla_put_net16(msg, RDMA_NLDEV_ATTR_RES_IP_SPORT,
+				  sin6->sin6_port))
+			goto err;
+
+		sin6 = (struct sockaddr_in6 *)&cm_id->route.addr.dst_addr;
+		if (nla_put_in6_addr(msg,

RDMA_NLDEV_ATTR_RES_IPV6_DADDR,

drivers/infiniband/core/nldev.c static int nldev_res_get_qp_dumpit(struct sk_buff

Quoted 31 lines

drivers/infiniband/core/nldev.c +26 -0

+				     &sin6->sin6_addr))
+			goto err;
+		if (nla_put_net16(msg, RDMA_NLDEV_ATTR_RES_IP_DPORT,
+				  sin6->sin6_port))
+			goto err;
+	}
+
+	if (id_priv->id.caller) {
+		if (nla_put_string(msg, RDMA_NLDEV_ATTR_RES_KERN_NAME,
+				   id_priv->id.caller))
+			goto err;
+	} else {
+		/* CMA keeps the owning pid. */
+		if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PID, id_priv-
 owner))
+			goto err;
+	}
+
+	nla_nest_end(msg, entry_attr);
+	return 0;
+
+err:
+	nla_nest_cancel(msg, entry_attr);
+out:
+	return -EMSGSIZE;
+}
+

static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,

drivers/infiniband/core/nldev.c +0 -0

 			  struct netlink_ext_ack *extack)
 {

@@ -686,6 +797,137 @@ static int nldev_res_get_qp_dumpit(struct sk_buff

*skb,

Quoted 12 lines

drivers/infiniband/core/nldev.c +9 -0

 	return ret;
 }
 
+static int nldev_res_get_cm_id_dumpit(struct sk_buff *skb,
+				      struct netlink_callback *cb)
+{
+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+	struct rdma_restrack_entry *res;
+	int err, ret = 0, idx = 0;
+	struct nlattr *table_attr;
+	struct ib_device *device;
+	int start = cb->args[0];

drivers/infiniband/core/nldev.c static int nldev_res_get_qp_dumpit(struct sk_buff *skb,

Quoted 8 lines

drivers/infiniband/core/nldev.c +8 -0

+	struct rdma_cm_id *cm_id = NULL;
+	struct nlmsghdr *nlh;
+	u32 index, port = 0;
+
+	err = nlmsg_parse(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
+			  nldev_policy, NULL);
+	/*
+	 * Right now, we are expecting the device index to get QP

drivers/infiniband/core/nldev.c static int nldev_res_get_qp_dumpit(struct sk_buff · line 818

Quoted 2 lines

drivers/infiniband/core/nldev.c +2 -0

+	 * but it is possible to extend this code to return all devices in
+	 * one shot by checking the existence of

RDMA_NLDEV_ATTR_DEV_INDEX.

Quoted 4 lines

drivers/infiniband/core/nldev.c +4 -0

+	 * if it doesn't exist, we will iterate over all devices.
+	 *
+	 * But it is not needed for now.
+	 */

drivers/infiniband/core/nldev.c static int nldev_res_get_qp_dumpit(struct sk_buff *skb,

Quoted 10 lines

drivers/infiniband/core/nldev.c +10 -0

+	if (err || !tb[RDMA_NLDEV_ATTR_DEV_INDEX])
+		return -EINVAL;
+
+	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
+	device = ib_device_get_by_index(index);
+	if (!device)
+		return -EINVAL;
+
+	/*
+	 * If no PORT_INDEX is supplied, we will return all QPs from that

QPs->CM_IDs

drivers/infiniband/core/nldev.c static int nldev_res_get_qp_dumpit(struct sk_buff *skb,

Quoted 11 lines

drivers/infiniband/core/nldev.c +11 -0

+	 */
+	if (tb[RDMA_NLDEV_ATTR_PORT_INDEX]) {
+		port = nla_get_u32(tb[RDMA_NLDEV_ATTR_PORT_INDEX]);
+		if (!rdma_is_port_valid(device, port)) {
+			ret = -EINVAL;
+			goto err_index;
+		}
+	}
+
+	nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
+		RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,

RDMA_NLDEV_CMD_RES_QP_GET),

RDMA_NLDEV_CMD_RES_QP_GET -> RDMA_NLDEV_CMD_RES_CM_ID_GET

Quoted 18 lines

drivers/infiniband/core/nldev.c +18 -0

+		0, NLM_F_MULTI);
+
+	if (fill_nldev_handle(skb, device)) {
+		ret = -EMSGSIZE;
+		goto err;
+	}
+
+	table_attr = nla_nest_start(skb, RDMA_NLDEV_ATTR_RES_CM_ID);
+	if (!table_attr) {
+		ret = -EMSGSIZE;
+		goto err;
+	}
+
+	down_read(&device->res.rwsem);
+	hash_for_each_possible(device->res.hash, res, node,
+			       RDMA_RESTRACK_CM_ID) {
+		if (idx < start)
+			goto next;

drivers/infiniband/core/nldev.c static int nldev_res_get_qp_dumpit(struct sk_buff *skb,

Quoted 8 lines

drivers/infiniband/core/nldev.c +8 -0

+
+		if ((rdma_is_kernel_res(res) &&
+		     task_active_pid_ns(current) != &init_pid_ns) ||
+		    (!rdma_is_kernel_res(res) &&
+		     task_active_pid_ns(current) !=
+		     task_active_pid_ns(res->task)))
+			/*
+			 * 1. Kernel QPs should be visible in init namsapce

drivers/infiniband/core/nldev.c static int nldev_res_get_qp_dumpit(struct sk_buff · line 870

Quoted 1 line

drivers/infiniband/core/nldev.c +1 -0

+			 * 2. Preent only QPs visible in the current

drivers/infiniband/core/nldev.c static int nldev_res_get_qp_dumpit(struct sk_buff

Quoted 64 lines

drivers/infiniband/core/nldev.c +60 -0

+			 */
+			goto next;
+
+		if (!rdma_restrack_get(res))
+			/*
+			 * Resource is under release now, but we are not
+			 * relesing lock now, so it will be released in
+			 * our next pass, once we will get ->next pointer.
+			 */
+			goto next;
+
+		cm_id = container_of(res, struct rdma_cm_id, res);
+
+		up_read(&device->res.rwsem);
+		ret = fill_res_cm_id_entry(skb, cm_id, port);
+		down_read(&device->res.rwsem);
+		/*
+		 * Return resource back, but it won't be released till
+		 * the &device->res.rwsem will be released for write.
+		 */
+		rdma_restrack_put(res);
+
+		if (ret == -EMSGSIZE)
+			/*
+			 * There is a chance to optimize here.
+			 * It can be done by using list_prepare_entry
+			 * and list_for_each_entry_continue afterwards.
+			 */
+			break;
+		if (ret)
+			goto res_err;
+next:		idx++;
+	}
+	up_read(&device->res.rwsem);
+
+	nla_nest_end(skb, table_attr);
+	nlmsg_end(skb, nlh);
+	cb->args[0] = idx;
+
+	/*
+	 * No more CM_IDs to fill, cancel the message and
+	 * return 0 to mark end of dumpit.
+	 */
+	if (!cm_id)
+		goto err;
+
+	put_device(&device->dev);
+	return skb->len;
+
+res_err:
+	nla_nest_cancel(skb, table_attr);
+	up_read(&device->res.rwsem);
+
+err:
+	nlmsg_cancel(skb, nlh);
+
+err_index:
+	put_device(&device->dev);
+	return ret;
+}
 static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
 	[RDMA_NLDEV_CMD_GET] = {
 		.doit = nldev_get_doit,

@@ -712,6 +954,9 @@ static int nldev_res_get_qp_dumpit(struct sk_buff

*skb,

Review

Quoted 10 lines

drivers/infiniband/core/nldev.c +3 -0

 		 * too.
 		 */
 	},
+	[RDMA_NLDEV_CMD_RES_CM_ID_GET] = {
+		.dump = nldev_res_get_cm_id_dumpit,
+	},
 };
 
 void __init nldev_init(void)

(quoted diff) +0 -0

b/drivers/infiniband/core/restrack.c

drivers/infiniband/core/restrack.c static struct ib_device *res_to_dev(struct

Quoted 12 lines

index 857637b..bb13169 100644

drivers/infiniband/core/restrack.c +1 -0

@@ -3,6 +3,7 @@
  * Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved.
  */
 
+#include <rdma/rdma_cm.h>
 #include <rdma/ib_verbs.h>
 #include <rdma/restrack.h>
 #include <linux/mutex.h>

@@ -67,6 +68,7 @@ static struct ib_device *res_to_dev(struct

rdma_restrack_entry *res)

drivers/infiniband/core/restrack.c static struct ib_device *res_to_dev(struct

Quoted 8 lines

drivers/infiniband/core/restrack.c +1 -0

 	struct ib_pd *pd;
 	struct ib_cq *cq;
 	struct ib_qp *qp;
+	struct rdma_cm_id *cm_id;
 
 	switch (type) {
 	case RDMA_RESTRACK_PD:

@@ -85,6 +87,10 @@ static struct ib_device *res_to_dev(struct

rdma_restrack_entry *res)

drivers/infiniband/core/restrack.c static struct ib_device *res_to_dev(struct

Quoted 11 lines

drivers/infiniband/core/restrack.c +4 -0

 		xrcd = container_of(res, struct ib_xrcd, res);
 		dev = xrcd->device;
 		break;
+	case RDMA_RESTRACK_CM_ID:
+		cm_id = container_of(res, struct rdma_cm_id, res);
+		dev = cm_id->device;
+		break;
 	default:
 		WARN_ONCE(true, "Wrong resource tracking type %u\n", type);
 		return NULL;

@@ -93,6 +99,27 @@ static struct ib_device *res_to_dev(struct

rdma_restrack_entry *res)

drivers/infiniband/core/restrack.c void rdma_restrack_add(struct rdma_restrack_entry

Quoted 28 lines

drivers/infiniband/core/restrack.c +21 -0

 	return dev;
 }
 
+static bool res_is_user(struct rdma_restrack_entry *res)
+{
+	enum rdma_restrack_type type = res->type;
+	bool is_user;
+
+	switch (type) {
+	case RDMA_RESTRACK_CM_ID: {
+		struct rdma_cm_id *cm_id;
+
+		cm_id = container_of(res, struct rdma_cm_id, res);
+		is_user = !cm_id->caller;
+		break;
+	}
+	default:
+		is_user = !uaccess_kernel();
+		break;
+	}
+
+	return is_user;
+}
+
 void rdma_restrack_add(struct rdma_restrack_entry *res)
 {
 	struct ib_device *dev = res_to_dev(res);

@@ -100,7 +127,7 @@ void rdma_restrack_add(struct rdma_restrack_entry

*res)

Quoted 3 lines

drivers/infiniband/core/restrack.c +0 -0

 	if (!dev)
 		return;
 

Review

Quoted 6 lines

drivers/infiniband/core/restrack.c +1 -1

-	if (!uaccess_kernel()) {
+	if (res_is_user(res)) {
 		get_task_struct(current);
 		res->task = current;
 		res->kern_name = NULL;

(quoted diff) +0 -0

drivers/infiniband/core/ucma.c static ssize_t ucma_create_id(struct ucma_file

Quoted 4 lines

index d67219d..f7f0282 100644

drivers/infiniband/core/ucma.c +0 -0

@@ -476,8 +476,8 @@ static ssize_t ucma_create_id(struct ucma_file

const char __user *inbuf,

drivers/infiniband/core/ucma.c static ssize_t ucma_create_id(struct ucma_file *file, const char __user *inbuf,

Quoted 5 lines

drivers/infiniband/core/ucma.c +0 -2

 		return -ENOMEM;
 
 	ctx->uid = cmd.uid;
-	ctx->cm_id = rdma_create_id(current->nsproxy->net_ns,
-				    ucma_event_handler, ctx, cmd.ps,

drivers/infiniband/core/ucma.c static ssize_t ucma_create_id(struct ucma_file · line 480

Quoted 2 lines

drivers/infiniband/core/ucma.c +2 -0

+	ctx->cm_id = __rdma_create_id(current->nsproxy->net_ns,
+			      ucma_event_handler, ctx, cmd.ps, qp_type,

drivers/infiniband/core/ucma.c static ssize_t ucma_accept(struct ucma_file

Quoted 4 lines

drivers/infiniband/core/ucma.c +0 -0

 	if (IS_ERR(ctx->cm_id)) {
 		ret = PTR_ERR(ctx->cm_id);
 		goto err1;

@@ -1084,12 +1084,12 @@ static ssize_t ucma_accept(struct ucma_file

const char __user *inbuf,

drivers/infiniband/core/ucma.c static ssize_t ucma_accept(struct ucma_file · line 1085

Quoted 2 lines

drivers/infiniband/core/ucma.c +0 -0

 	if (cmd.conn_param.valid) {
 		ucma_copy_conn_param(ctx->cm_id, &conn_param,

&cmd.conn_param);

Quoted 33 lines

drivers/infiniband/core/ucma.c +2 -2

 		mutex_lock(&file->mut);
-		ret = rdma_accept(ctx->cm_id, &conn_param);
+		ret = __rdma_accept(ctx->cm_id, &conn_param, NULL);
 		if (!ret)
 			ctx->uid = cmd.uid;
 		mutex_unlock(&file->mut);
 	} else
-		ret = rdma_accept(ctx->cm_id, NULL);
+		ret = __rdma_accept(ctx->cm_id, NULL, NULL);
 
 	ucma_put_ctx(ctx);
 	return ret;

include/rdma/rdma_cm.h +6 -0

index 5984225..73dac9b 100644
@@ -155,6 +155,12 @@ struct rdma_cm_id {
 	enum rdma_port_space	 ps;
 	enum ib_qp_type		 qp_type;
 	u8			 port_num;
+	const char *caller;
+
+	/*
+	 * Internal to RDMA/core, don't use in the drivers
+	 */
+	struct rdma_restrack_entry     res;
 };
 
 struct rdma_id_private {

@@ -198,6 +204,11 @@ struct rdma_id_private {
 	enum ib_gid_type	gid_type;
 };
 

include/rdma/rdma_cm.h struct rdma_id_private { · line 210

Quoted 4 lines

include/rdma/rdma_cm.h +4 -0

+struct rdma_cm_id *__rdma_create_id(struct net *net,
+				  rdma_cm_event_handler event_handler,
+				  void *context, enum rdma_port_space ps,
+				  enum ib_qp_type qp_type, const char

include/rdma/rdma_cm.h struct rdma_id_private { · line 222

Quoted 7 lines

include/rdma/rdma_cm.h +1 -0

+
 /**
  * rdma_create_id - Create an RDMA identifier.
  *

@@ -210,10 +221,9 @@ struct rdma_id_private {
  *
  * The id holds a reference on the network namespace until it is

include/rdma/rdma_cm.h struct rdma_id_private {

Quoted 7 lines

include/rdma/rdma_cm.h +2 -4

  */
-struct rdma_cm_id *rdma_create_id(struct net *net,
-				  rdma_cm_event_handler event_handler,
-				  void *context, enum rdma_port_space ps,
-				  enum ib_qp_type qp_type);
+#define rdma_create_id(net, event_handler, context, ps, qp_type) \
+	__rdma_create_id((net), (event_handler), (context), (ps), (qp_type),

include/rdma/rdma_cm.h int rdma_init_qp_attr(struct rdma_cm_id *id, struct

Quoted 5 lines

include/rdma/rdma_cm.h +1 -0

+			 KBUILD_MODNAME)
 
 /**
   * rdma_destroy_id - Destroys an RDMA identifier.

@@ -325,6 +335,9 @@ int rdma_init_qp_attr(struct rdma_cm_id *id, struct

ib_qp_attr *qp_attr,

include/rdma/rdma_cm.h int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,

Quoted 4 lines

include/rdma/rdma_cm.h +1 -0

  */
 int rdma_listen(struct rdma_cm_id *id, int backlog);
 
+int __rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param

*conn_param,

include/rdma/rdma_cm.h int rdma_init_qp_attr(struct rdma_cm_id *id, struct

Quoted 6 lines

include/rdma/rdma_cm.h +2 -0

+		  const char *caller);
+
 /**
  * rdma_accept - Called to accept a connection request or response.
  * @id: Connection identifier associated with the request.

@@ -340,7 +353,8 @@ int rdma_init_qp_attr(struct rdma_cm_id *id, struct

ib_qp_attr *qp_attr,

Quoted 1 line

include/rdma/rdma_cm.h +0 -0

  * state of the qp associated with the id is modified to error, such

include/rdma/rdma_cm.h int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,

Quoted 3 lines

include/rdma/rdma_cm.h +0 -1

  * previously posted receive buffers would be flushed.
  */
-int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param

*conn_param);

Review

Quoted 21 lines

include/rdma/rdma_cm.h +2 -0

+#define rdma_accept(id, conn_param) \
+	__rdma_accept((id), (conn_param),  KBUILD_MODNAME)
 
 /**
  * rdma_notify - Notifies the RDMA CM of an asynchronous event that has

include/rdma/restrack.h +4 -0

index c2d8116..a794e0e 100644
@@ -33,6 +33,10 @@ enum rdma_restrack_type {
 	 */
 	RDMA_RESTRACK_XRCD,
 	/**
+	 * @RDMA_RESTRACK_CM_ID: Connection Manager ID (CM_ID)
+	 */
+	RDMA_RESTRACK_CM_ID,
+	/**
 	 * @RDMA_RESTRACK_MAX: Last entry, used for array dclarations
 	 */
 	RDMA_RESTRACK_MAX

(quoted diff) +0 -0

b/include/uapi/rdma/rdma_netlink.h

Quoted 26 lines

index 17e59be..4ed21ee 100644

include/uapi/rdma/rdma_netlink.h +12 -0

@@ -240,6 +240,8 @@ enum rdma_nldev_command {
 
 	RDMA_NLDEV_CMD_RES_QP_GET, /* can dump */
 
+	RDMA_NLDEV_CMD_RES_CM_ID_GET, /* can dump */
+
 	RDMA_NLDEV_NUM_OPS
 };
 

@@ -352,6 +354,34 @@ enum rdma_nldev_attr {
 	 */
 	RDMA_NLDEV_ATTR_RES_KERN_NAME,		/* string */
 
+	RDMA_NLDEV_ATTR_RES_CM_ID,		/* nested table */
+	RDMA_NLDEV_ATTR_RES_CM_ID_ENTRY,	/* nested table */
+	/*
+	 * rdma_cm_id port space.
+	 */
+	RDMA_NLDEV_ATTR_RES_PS,			/* u32 */
+	/*
+	 * IP Addresses/port attributes.
+	 */
+	RDMA_NLDEV_ATTR_RES_IPV4_SADDR,		/* u8[4] */

include/uapi/rdma/rdma_netlink.h enum rdma_nldev_attr { · line 371

Quoted 5 lines

include/uapi/rdma/rdma_netlink.h +5 -0

+	RDMA_NLDEV_ATTR_RES_IPV4_DADDR,		/* u8[4] */
+	RDMA_NLDEV_ATTR_RES_IPV6_SADDR,		/* u8[16] */
+	RDMA_NLDEV_ATTR_RES_IPV6_DADDR,		/* u8[16] */
+	RDMA_NLDEV_ATTR_RES_IP_SPORT,		/* BE u16 */
+	RDMA_NLDEV_ATTR_RES_IP_DPORT,		/* BE u16 */

Can you please document the meaning of S (source) and D (destination) in regards of this netlink output? It is needed to remove ambiguity.

Quoted 11 lines

include/uapi/rdma/rdma_netlink.h +10 -0

+	/*
+	 * ARPHRD_INFINIBAND, ARPHRD_ETHER, ...
+	 */
+	RDMA_NLDEV_ATTR_RES_DEV_TYPE,		/* u8 */
+	/*
+	 * enum enum rdma_transport_type (IB, IWARP, ...)
+	 */
+	RDMA_NLDEV_ATTR_RES_TRANSPORT_TYPE,	/* u8 */
+	/*
+	 * enum rdma_network_type (IB, IPv4, IPv6,...)

Quoted 8 lines

include/uapi/rdma/rdma_netlink.h +3 -1

+	 */
+	RDMA_NLDEV_ATTR_RES_NETWORK_TYPE,	/* u8 */
+
 	RDMA_NLDEV_ATTR_MAX
 };
 #endif /* _UAPI_RDMA_NETLINK_H */
--

1.8.3.1

Regarding the name, I personally think that cm_id is better because it is general and applicable both to ib_cm and iw_cm. The separation between them can be done with introduction of new netlink attribute, e.g. cm_id_type.

include/uapi/rdma/rdma_netlink.h enum rdma_nldev_attr { · line 371

Quoted 6 lines

include/uapi/rdma/rdma_netlink.h +6 -0

+	RDMA_NLDEV_ATTR_RES_IPV4_SADDR,		/* u8[4] */
+	RDMA_NLDEV_ATTR_RES_IPV4_DADDR,		/* u8[4] */
+	RDMA_NLDEV_ATTR_RES_IPV6_SADDR,		/* u8[16] */
+	RDMA_NLDEV_ATTR_RES_IPV6_DADDR,		/* u8[16] */
+	RDMA_NLDEV_ATTR_RES_IP_SPORT,		/* BE u16 */
+	RDMA_NLDEV_ATTR_RES_IP_DPORT,		/* BE u16 */

Can you please document the meaning of S (source) and D (destination) in regards of this netlink output? It is needed to remove ambiguity.

Quoted 4 lines

Regarding the name, I personally think that cm_id is better because it is general and applicable both to ib_cm and iw_cm. The separation between them can be done with introduction of new netlink attribute,

Quoted 1 line

cm_id_type.

In netlink the attributes should be self-describing, or intrinsically related to something mandatory and fundamental about their container (eg AF_ family in rtnl)

What information do we actually need from the cm_id in various protocol families?

-----Original Message-----
From: Jason Gunthorpe [mailto:jgg-uk2M96/[email protected]]
Sent: Thursday, February 01, 2018 11:54 AM
To: Leon Romanovsky
Cc: Steve Wise; [email protected]; [email protected]
Subject: Re: [PATCH RFC 2/2] RDMA/nldev: provide detailed CM_ID

Quoted 10 lines

include/uapi/rdma/rdma_netlink.h enum rdma_nldev_attr { · line 371

Quoted 7 lines

include/uapi/rdma/rdma_netlink.h +6 -0

+	RDMA_NLDEV_ATTR_RES_IPV4_SADDR,		/* u8[4] */
+	RDMA_NLDEV_ATTR_RES_IPV4_DADDR,		/* u8[4] */
+	RDMA_NLDEV_ATTR_RES_IPV6_SADDR,		/* u8[16] */
+	RDMA_NLDEV_ATTR_RES_IPV6_DADDR,		/* u8[16] */
+	RDMA_NLDEV_ATTR_RES_IP_SPORT,		/* BE u16 */
+	RDMA_NLDEV_ATTR_RES_IP_DPORT,		/* BE u16 */

Can you please document the meaning of S (source) and D (destination) in regards of this netlink output? It is needed to remove ambiguity.

And no on BE's in netlink, I think.

Quoted 7 lines

From: Jason Gunthorpe [mailto:jgg-uk2M96/[email protected]]
Sent: Thursday, February 01, 2018 11:54 AM
To: Leon Romanovsky
Cc: Steve Wise; [email protected]; [email protected]
Subject: Re: [PATCH RFC 2/2] RDMA/nldev: provide detailed CM_ID

information

Quoted 13 lines · Leon Romanovsky

Quoted 10 lines

Review

Quoted 7 lines

(quoted diff) +6 -0

+	RDMA_NLDEV_ATTR_RES_IPV4_SADDR,		/* u8[4] */
+	RDMA_NLDEV_ATTR_RES_IPV4_DADDR,		/* u8[4] */
+	RDMA_NLDEV_ATTR_RES_IPV6_SADDR,		/* u8[16] */
+	RDMA_NLDEV_ATTR_RES_IPV6_DADDR,		/* u8[16] */
+	RDMA_NLDEV_ATTR_RES_IP_SPORT,		/* BE u16 */
+	RDMA_NLDEV_ATTR_RES_IP_DPORT,		/* BE u16 */

Can you please document the meaning of S (source) and D (destination) in regards of this netlink output? It is needed to remove ambiguity.

Quoted 1 line

And no on BE's in netlink, I think.

Sure there are, see nla_put_be32() for example. I'm using nla_put_net16() to store the ports. Perhaps I need to change the above comment to ne16 instead of BE u16?

Quoted 10 lines

Quoted 10 lines · Leon Romanovsky

Quoted 9 lines

Review

Quoted 7 lines

(quoted diff) +6 -0

+	RDMA_NLDEV_ATTR_RES_IPV4_SADDR,		/* u8[4] */
+	RDMA_NLDEV_ATTR_RES_IPV4_DADDR,		/* u8[4] */
+	RDMA_NLDEV_ATTR_RES_IPV6_SADDR,		/* u8[16] */
+	RDMA_NLDEV_ATTR_RES_IPV6_DADDR,		/* u8[16] */
+	RDMA_NLDEV_ATTR_RES_IP_SPORT,		/* BE u16 */
+	RDMA_NLDEV_ATTR_RES_IP_DPORT,		/* BE u16 */

Can you please document the meaning of S (source) and D

Quoted 6 lines

Quoted 3 lines

Quoted 1 line

in regards of this netlink output? It is needed to remove ambiguity.

And no on BE's in netlink, I think.

Sure there are, see nla_put_be32() for example. I'm using

Quoted 1 line

store the ports. Perhaps I need to change the above comment to ne16

Quoted 1 line

of BE u16?

There is no reason thes ports should be be, so don't make it be.

Quoted 10 lines

Quoted 10 lines

Quoted 10 lines · Leon Romanovsky

Quoted 9 lines

Review

Quoted 7 lines

(quoted diff) +6 -0

+	RDMA_NLDEV_ATTR_RES_IPV4_SADDR,		/* u8[4] */
+	RDMA_NLDEV_ATTR_RES_IPV4_DADDR,		/* u8[4] */
+	RDMA_NLDEV_ATTR_RES_IPV6_SADDR,		/* u8[16] */
+	RDMA_NLDEV_ATTR_RES_IPV6_DADDR,		/* u8[16] */
+	RDMA_NLDEV_ATTR_RES_IP_SPORT,		/* BE u16 */
+	RDMA_NLDEV_ATTR_RES_IP_DPORT,		/* BE u16 */

Can you please document the meaning of S (source) and D

(destination)

Quoted 6 lines

Quoted 6 lines

Quoted 3 lines

Quoted 1 line

in regards of this netlink output? It is needed to remove ambiguity.

And no on BE's in netlink, I think.

Sure there are, see nla_put_be32() for example. I'm using

nla_put_net16() to

Quoted 1 line

Quoted 1 line

store the ports. Perhaps I need to change the above comment to ne16

instead

Quoted 3 lines

Quoted 1 line

of BE u16?

There is no reason thes ports should be be, so don't make it be.

Well, they're BE on the wire. That's the reason. And other APIs return them to user space in their NBO, like getsockname().

You would rather the nldev code puts it in HBO before sending it up?

Quoted 4 lines · Steve Wise

Quoted 1 line

Implement RDMA nldev netlink interface to get detailed CM_ID

information.

Quoted 2 lines

Quoted 2 lines

Because cm_id's are attached to rdma devices in various work queue

contexts,

Quoted 1 line

Quoted 1 line

the pid and task information at device-attach time is sometimes not

useful.

Quoted 2 lines

Quoted 2 lines

For example, an nvme/f host connection ends up being bound to a device in a work queue context and the resulting pid at attach time no longer

exists

Quoted 10 lines

Quoted 9 lines

after connection setup. So instead we mark all cm_id's created via the rdma_ucm as "user", and all others as "kernel". This required tweaking the restrack code a little. It also required wrapping some rdma_cm functions to allow passing the module name string.

Signed-off-by Steve Wise <[email protected]>

---
 drivers/infiniband/core/cma.c      |  55 ++++++---
 drivers/infiniband/core/nldev.c    | 245

+++++++++++++++++++++++++++++++++++++

Quoted 8 lines

Review

Quoted 8 lines

 drivers/infiniband/core/restrack.c |  29 ++++-
 drivers/infiniband/core/ucma.c     |   8 +-
 include/rdma/rdma_cm.h             |  24 +++-
 include/rdma/restrack.h            |   4 +
 include/uapi/rdma/rdma_netlink.h   |  30 +++++
 7 files changed, 365 insertions(+), 30 deletions(-)

(quoted diff) +0 -0

b/drivers/infiniband/core/cma.c

Quoted 5 lines

drivers/infiniband/core/cma.c static void _cma_attach_to_dev(struct

Quoted 4 lines

index 72ad52b..51fbfa1 100644

drivers/infiniband/core/cma.c +0 -0

@@ -465,6 +465,9 @@ static void _cma_attach_to_dev(struct

rdma_id_private *id_priv,

Quoted 7 lines

drivers/infiniband/core/cma.c static void _cma_attach_to_dev(struct rdma_id_private *id_priv, · line 469

Quoted 5 lines

drivers/infiniband/core/cma.c +2 -0

 	id_priv->id.route.addr.dev_addr.transport =
 		rdma_node_get_transport(cma_dev->device->node_type);
 	list_add_tail(&id_priv->list, &cma_dev->id_list);
+	id_priv->id.res.type = RDMA_RESTRACK_CM_ID;
+	id_priv->id.res.kern_name = id_priv->id.caller;

Steve, I don't like it, I worked hard to hide it from the users of

restrack,

Quoted 3 lines

and don't see reason why the same trick as with ib_create_cq/ib_create_pd won't
work here.

I am doing the same trick, no? rdma_create_id() is a static inline that passes KBUILD_MODNAME. The issue is that at the time the rdma_cm_id is created, it is not associated with any ib_device. That only happens at cma_attach time. So how can the resource be added if there is no device?

Quoted 10 lines

drivers/infiniband/core/cma.c static void cma_deref_id(struct rdma_id_private

Quoted 6 lines

drivers/infiniband/core/cma.c +1 -0

+	rdma_restrack_add(&id_priv->id.res);
 }
 
 static void cma_attach_to_dev(struct rdma_id_private *id_priv,

@@ -737,10 +740,10 @@ static void cma_deref_id(struct rdma_id_private

*id_priv)

Quoted 3 lines

drivers/infiniband/core/cma.c +0 -0

 		complete(&id_priv->comp);
 }
 

Quoted 8 lines

drivers/infiniband/core/cma.c static void cma_deref_id(struct rdma_id_private *id_priv)

Quoted 8 lines

drivers/infiniband/core/cma.c +4 -4

-struct rdma_cm_id *rdma_create_id(struct net *net,
-				  rdma_cm_event_handler event_handler,
-				  void *context, enum rdma_port_space ps,
-				  enum ib_qp_type qp_type)
+struct rdma_cm_id *__rdma_create_id(struct net *net,
+				    rdma_cm_event_handler event_handler,
+				    void *context, enum rdma_port_space ps,
+				    enum ib_qp_type qp_type, const char

*caller)

Quoted 7 lines

Quoted 7 lines

drivers/infiniband/core/cma.c +0 -0

 {
 	struct rdma_id_private *id_priv;
 

@@ -748,7 +751,10 @@ struct rdma_cm_id *rdma_create_id(struct net *net,
 	if (!id_priv)
 		return ERR_PTR(-ENOMEM);
 

Quoted 7 lines

drivers/infiniband/core/cma.c struct rdma_cm_id *rdma_create_id(struct net *net, · line 757

Quoted 5 lines

drivers/infiniband/core/cma.c +4 -1

-	id_priv->owner = task_pid_nr(current);
+	if (caller)
+		id_priv->id.caller = caller;
+	else
+		id_priv->owner = task_pid_nr(current);

See the comment above

There is no ib_device at this point, so caller (and owner) must be saved until the cm_id is bound to a device (or possibly devices for listening ids).

Quoted 6 lines

Quoted 5 lines

drivers/infiniband/core/cma.c +0 -0

 	id_priv->state = RDMA_CM_IDLE;
 	id_priv->id.context = context;
 	id_priv->id.event_handler = event_handler;

Quoted 4 lines

Not saying that we need to do it now, but it is important to write, most probably we can drop certain initialization from rdma_create_id() adn reuse rdma_restrack_put/_get.

I don't understand this comment. Can you please elaborate?

Quoted 9 lines

drivers/infiniband/core/cma.c struct rdma_cm_id *rdma_create_id(struct net *net, · line 779

Quoted 9 lines

drivers/infiniband/core/cma.c +1 -1

@@ -768,7 +774,7 @@ struct rdma_cm_id *rdma_create_id(struct net *net,
 
 	return &id_priv->id;
 }
-EXPORT_SYMBOL(rdma_create_id);
+EXPORT_SYMBOL(__rdma_create_id);
 
 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp

*qp)

Quoted 8 lines

drivers/infiniband/core/cma.c void rdma_destroy_id(struct rdma_cm_id *id) · line 1637

Quoted 6 lines

drivers/infiniband/core/cma.c +1 -0

 {

@@ -1628,6 +1634,7 @@ void rdma_destroy_id(struct rdma_cm_id *id)
 	mutex_unlock(&id_priv->handler_mutex);
 
 	if (id_priv->cma_dev) {
+		rdma_restrack_del(&id_priv->id.res);

You should count all created cm_ids and not only binded.

No ib_device if they aren't bound.

Quoted 6 lines

drivers/infiniband/core/cma.c static struct rdma_id_private

Quoted 5 lines

drivers/infiniband/core/cma.c +0 -0

 		if (rdma_cap_ib_cm(id_priv->id.device, 1)) {
 			if (id_priv->cm_id.ib)
 				ib_destroy_cm_id(id_priv->cm_id.ib);

@@ -1786,9 +1793,10 @@ static struct rdma_id_private

*cma_new_conn_id(struct rdma_cm_id *listen_id,

Quoted 7 lines

drivers/infiniband/core/cma.c static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,

Quoted 7 lines

drivers/infiniband/core/cma.c +1 -2

 		ib_event->param.req_rcvd.primary_path->service_id;
 	int ret;
 
-	id = rdma_create_id(listen_id->route.addr.dev_addr.net,
+	id = __rdma_create_id(listen_id->route.addr.dev_addr.net,
 			    listen_id->event_handler, listen_id->context,
-			    listen_id->ps,

ib_event->param.req_rcvd.qp_type);

Quoted 4 lines

drivers/infiniband/core/cma.c static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id, · line 1799

Quoted 2 lines

drivers/infiniband/core/cma.c +2 -0

+			    listen_id->ps, ib_event->param.req_rcvd.qp_type,
+			    listen_id->caller);

I think the cleanest way will be to create some struct and pass pointer to

it so

Quoted 2 lines

you can unfold all relevant data inside of __rdma_create_id().

Why is that cleaner? Marshall up the data into a struct, pass a ptr, unmarshall it all...

Quoted 11 lines

drivers/infiniband/core/cma.c static struct rdma_id_private

Quoted 6 lines

drivers/infiniband/core/cma.c +0 -0

 	if (IS_ERR(id))
 		return NULL;
 

@@ -1843,8 +1851,8 @@ static struct rdma_id_private

*cma_new_udp_id(struct rdma_cm_id *listen_id,

drivers/infiniband/core/cma.c static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,

Quoted 4 lines

drivers/infiniband/core/cma.c +0 -1

 	struct net *net = listen_id->route.addr.dev_addr.net;
 	int ret;
 
-	id = rdma_create_id(net, listen_id->event_handler,

listen_id->context,

Quoted 26 lines

drivers/infiniband/core/cma.c static int iw_conn_req_handler(struct iw_cm_id

Quoted 8 lines

drivers/infiniband/core/cma.c +2 -1

-			    listen_id->ps, IB_QPT_UD);
+	id = __rdma_create_id(net, listen_id->event_handler, listen_id-
 context,
+			      listen_id->ps, IB_QPT_UD, listen_id->caller);
 	if (IS_ERR(id))
 		return NULL;

@@ -2110,10 +2118,11 @@ static int iw_conn_req_handler(struct iw_cm_id

*cm_id,

drivers/infiniband/core/cma.c static void cma_listen_on_dev(struct

Quoted 16 lines

drivers/infiniband/core/cma.c +5 -4

 		goto out;
 
 	/* Create a new RDMA id for the new IW CM ID */
-	new_cm_id = rdma_create_id(listen_id->id.route.addr.dev_addr.net,
-				   listen_id->id.event_handler,
-				   listen_id->id.context,
-				   RDMA_PS_TCP, IB_QPT_RC);
+	new_cm_id = __rdma_create_id(listen_id->id.route.addr.dev_addr.net,
+				     listen_id->id.event_handler,
+				     listen_id->id.context,
+				     RDMA_PS_TCP, IB_QPT_RC,
+				     listen_id->id.caller);
 	if (IS_ERR(new_cm_id)) {
 		ret = -ENOMEM;
 		goto out;

@@ -2238,8 +2247,8 @@ static void cma_listen_on_dev(struct

rdma_id_private *id_priv,

Quoted 5 lines

drivers/infiniband/core/cma.c static void cma_listen_on_dev(struct · line 2242

Quoted 5 lines

drivers/infiniband/core/cma.c +0 -1

 	if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev-
 device, 1))
 		return;
 
-	id = rdma_create_id(net, cma_listen_handler, id_priv,

id_priv->id.ps,

Quoted 2 lines

drivers/infiniband/core/cma.c static void cma_listen_on_dev(struct · line 2251

Quoted 2 lines

drivers/infiniband/core/cma.c +1 -1

-			    id_priv->id.qp_type);
+	id = __rdma_create_id(net, cma_listen_handler, id_priv,

id_priv->id.ps,

Quoted 49 lines

drivers/infiniband/core/cma.c int rdma_bind_addr(struct rdma_cm_id *id, struct

Quoted 5 lines

drivers/infiniband/core/cma.c +1 -0

+			      id_priv->id.qp_type, id_priv->id.caller);
 	if (IS_ERR(id))
 		return;

@@ -3347,8 +3356,10 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct

sockaddr *addr)

drivers/infiniband/core/cma.c static int cma_send_sidr_rep(struct

Quoted 12 lines

drivers/infiniband/core/cma.c +3 -1

 	return 0;
 err2:
-	if (id_priv->cma_dev)
+	if (id_priv->cma_dev) {
+		rdma_restrack_del(&id_priv->id.res);
 		cma_release_dev(id_priv);
+	}
 err1:
 	cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
 	return ret;

@@ -3731,14 +3742,18 @@ static int cma_send_sidr_rep(struct

rdma_id_private *id_priv,

drivers/infiniband/core/cma.c static int cma_send_sidr_rep(struct rdma_id_private *id_priv,

Quoted 4 lines

drivers/infiniband/core/cma.c +0 -1

 	return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
 }
 
-int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param

*conn_param)

drivers/infiniband/core/cma.c static int cma_send_sidr_rep(struct · line 3745

Quoted 1 line

drivers/infiniband/core/cma.c +1 -0

+int __rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param

*conn_param,

drivers/infiniband/core/cma.c int rdma_accept(struct rdma_cm_id *id, struct

Quoted 16 lines

drivers/infiniband/core/cma.c +5 -1

+		  const char *caller)
 {
 	struct rdma_id_private *id_priv;
 	int ret;
 
 	id_priv = container_of(id, struct rdma_id_private, id);
 
-	id_priv->owner = task_pid_nr(current);
+	if (caller)
+		id_priv->id.caller = caller;
+	else
+		id_priv->owner = task_pid_nr(current);
 
 	if (!cma_comp(id_priv, RDMA_CM_CONNECT))
 		return -EINVAL;

@@ -3778,7 +3793,7 @@ int rdma_accept(struct rdma_cm_id *id, struct

rdma_conn_param *conn_param)

Quoted 6 lines

drivers/infiniband/core/cma.c +1 -1

 	rdma_reject(id, NULL, 0);
 	return ret;
 }
-EXPORT_SYMBOL(rdma_accept);
+EXPORT_SYMBOL(__rdma_accept);
 

Quoted 3 lines

Review

Quoted 3 lines

drivers/infiniband/core/cma.c +0 -0

 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
 {

(quoted diff) +0 -0

b/drivers/infiniband/core/nldev.c

Quoted 23 lines

Quoted 12 lines

index fa8655e..a4091b5 100644

drivers/infiniband/core/nldev.c +1 -0

@@ -34,6 +34,7 @@
 #include <linux/pid.h>
 #include <linux/pid_namespace.h>
 #include <net/netlink.h>
+#include <rdma/rdma_cm.h>
 #include <rdma/rdma_netlink.h>
 
 #include "core_priv.h"

drivers/infiniband/core/nldev.c line 73

Quoted 2 lines

drivers/infiniband/core/nldev.c +0 -0

@@ -71,6 +72,22 @@
 	[RDMA_NLDEV_ATTR_RES_PID]		= { .type = NLA_U32 },
 	[RDMA_NLDEV_ATTR_RES_KERN_NAME]		= { .type =

NLA_NUL_STRING,

drivers/infiniband/core/nldev.c line 75

Quoted 2 lines

drivers/infiniband/core/nldev.c +1 -0

 						    .len = TASK_COMM_LEN },
+	[RDMA_NLDEV_ATTR_RES_CM_ID]		= { .type =

NLA_NESTED },

I would like to use this opportunity. There is CM_ID, so users will be able to query nldev directly on this ID (once we implement .doit), but can we found a proper abstraction for other objects (PD, CQ, QP e.t.c.)?