Skip to contentSkip to conversation

ib_core: Refine device personality from node type to port type

PATCH New Eli Cohen 1 participant 1 message Nov 16, 2009

Subsystems (5)
Networking Network drivers NFS client NFS server +1 more subsystems
lore
Share & tools
lore

Eli Cohen

author
#0 lore raw

PATCH v6 1/10 ib_core: Refine device personality from node type to port type

As a preparation to devices that, in general, support a different transport protocol for each port, specifically RDMAoE, this patch defines a transport type for each of a device's ports. As a result, rdma_node_get_transport() has been unexported and is used internally by the implementation of the new API, rdma_port_get_transport(), which gives the transport protocol of the queried port. rdma_is_transport_supported() is also added to be used for verifying if a given device supports a given protocol on any of its ports. All references to rdma_node_get_transport() are changed to use the new APIs. Also, ib_port_attr is extended to contain enum rdma_transport_type.

Signed-off-by Eli Cohen <eli-VPRAkNaXOzVS1MOuV/[email protected]>
---
 drivers/infiniband/core/cm.c              |   25 +++++++++----
 drivers/infiniband/core/cma.c             |   54 +++++++++++++++--------------
 drivers/infiniband/core/mad.c             |   49 ++++++++++++++++---------
 drivers/infiniband/core/multicast.c       |    4 +-
 drivers/infiniband/core/sa_query.c        |   39 +++++++++++++--------
 drivers/infiniband/core/ucm.c             |    8 +++-
 drivers/infiniband/core/ucma.c            |    2 +-
 drivers/infiniband/core/user_mad.c        |    6 +++-
 drivers/infiniband/core/verbs.c           |   25 ++++++++++++-
 drivers/infiniband/ulp/ipoib/ipoib_main.c |   12 +++---
 include/rdma/ib_verbs.h                   |   11 ++++--
 net/sunrpc/xprtrdma/svc_rdma_recvfrom.c   |    3 +-
 net/sunrpc/xprtrdma/svc_rdma_transport.c  |    2 +-
 13 files changed, 153 insertions(+), 87 deletions(-)
drivers/infiniband/core/cm.c +17 -8
index 5130fc5..d082f59 100644
@@ -3678,8 +3678,9 @@ static void cm_add_one(struct ib_device *ib_device)
 	unsigned long flags;
 	int ret;
 	u8 i;
+	enum rdma_transport_type tt;
 
-	if (rdma_node_get_transport(ib_device->node_type) != RDMA_TRANSPORT_IB)
+	if (!rdma_is_transport_supported(ib_device, RDMA_TRANSPORT_IB))
 		return;
 
 	cm_dev = kzalloc(sizeof(*cm_dev) + sizeof(*port) *

@@ -3700,6 +3701,10 @@ static void cm_add_one(struct ib_device *ib_device)
 
 	set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
 	for (i = 1; i <= ib_device->phys_port_cnt; i++) {
+		tt = rdma_port_get_transport(ib_device, i);
+		if (tt != RDMA_TRANSPORT_IB)
+			continue;
+
 		port = kzalloc(sizeof *port, GFP_KERNEL);
 		if (!port)
 			goto error1;

@@ -3742,9 +3747,11 @@ error1:
 	port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
 	while (--i) {
 		port = cm_dev->port[i-1];
-		ib_modify_port(ib_device, port->port_num, 0, &port_modify);
-		ib_unregister_mad_agent(port->mad_agent);
-		cm_remove_port_fs(port);
+		if (port) {
+			ib_modify_port(ib_device, port->port_num, 0, &port_modify);
+			ib_unregister_mad_agent(port->mad_agent);
+			cm_remove_port_fs(port);
+		}
 	}
 	device_unregister(cm_dev->device);
 	kfree(cm_dev);

@@ -3770,10 +3777,12 @@ static void cm_remove_one(struct ib_device *ib_device)
 
 	for (i = 1; i <= ib_device->phys_port_cnt; i++) {
 		port = cm_dev->port[i-1];
-		ib_modify_port(ib_device, port->port_num, 0, &port_modify);
-		ib_unregister_mad_agent(port->mad_agent);
-		flush_workqueue(cm.wq);
-		cm_remove_port_fs(port);
+		if (port) {
+			ib_modify_port(ib_device, port->port_num, 0, &port_modify);
+			ib_unregister_mad_agent(port->mad_agent);
+			flush_workqueue(cm.wq);
+			cm_remove_port_fs(port);
+		}
 	}
 	device_unregister(cm_dev->device);
 	kfree(cm_dev);
drivers/infiniband/core/cma.c +28 -26

122 lines not on this page · raw message

drivers/infiniband/core/mad.c +31 -18

85 lines not on this page · raw message

drivers/infiniband/core/multicast.c +2 -2
index 8d82ba1..7d9887d 100644
@@ -796,10 +796,10 @@ static void mcast_add_one(struct ib_device *device)
 	struct mcast_port *port;
 	int i;
 
-	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
+	if (!rdma_is_transport_supported(device, RDMA_TRANSPORT_IB))
 		return;
 
-	dev = kmalloc(sizeof *dev + device->phys_port_cnt * sizeof *port,
+	dev = kzalloc(sizeof *dev + device->phys_port_cnt * sizeof *port,
 		      GFP_KERNEL);
 	if (!dev)
 		return;
drivers/infiniband/core/sa_query.c +24 -15
index 7e1ffd8..18b09c4 100644
@@ -416,14 +416,16 @@ static void ib_sa_event(struct ib_event_handler *handler, struct ib_event *event
 		struct ib_sa_port *port =
 			&sa_dev->port[event->element.port_num - sa_dev->start_port];
 
-		spin_lock_irqsave(&port->ah_lock, flags);
-		if (port->sm_ah)
-			kref_put(&port->sm_ah->ref, free_sm_ah);
-		port->sm_ah = NULL;
-		spin_unlock_irqrestore(&port->ah_lock, flags);
-
-		schedule_work(&sa_dev->port[event->element.port_num -
-					    sa_dev->start_port].update_task);
+		if (rdma_port_get_transport(handler->device, port->port_num) == RDMA_TRANSPORT_IB) {
+			spin_lock_irqsave(&port->ah_lock, flags);
+			if (port->sm_ah)
+				kref_put(&port->sm_ah->ref, free_sm_ah);
+			port->sm_ah = NULL;
+			spin_unlock_irqrestore(&port->ah_lock, flags);
+
+			schedule_work(&sa_dev->port[event->element.port_num -
+						    sa_dev->start_port].update_task);
+		}
 	}
 }
 

@@ -997,7 +999,7 @@ static void ib_sa_add_one(struct ib_device *device)
 	struct ib_sa_device *sa_dev;
 	int s, e, i;
 
-	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
+	if (!rdma_is_transport_supported(device, RDMA_TRANSPORT_IB))
 		return;
 
 	if (device->node_type == RDMA_NODE_IB_SWITCH)

@@ -1007,7 +1009,7 @@ static void ib_sa_add_one(struct ib_device *device)
 		e = device->phys_port_cnt;
 	}
 
-	sa_dev = kmalloc(sizeof *sa_dev +
+	sa_dev = kzalloc(sizeof *sa_dev +
 			 (e - s + 1) * sizeof (struct ib_sa_port),
 			 GFP_KERNEL);
 	if (!sa_dev)

@@ -1017,6 +1019,9 @@ static void ib_sa_add_one(struct ib_device *device)
 	sa_dev->end_port   = e;
 
 	for (i = 0; i <= e - s; ++i) {
+		if (rdma_port_get_transport(device, i + 1) != RDMA_TRANSPORT_IB)
+			continue;
+
 		sa_dev->port[i].sm_ah    = NULL;
 		sa_dev->port[i].port_num = i + s;
 		spin_lock_init(&sa_dev->port[i].ah_lock);

@@ -1045,13 +1050,15 @@ static void ib_sa_add_one(struct ib_device *device)
 		goto err;
 
 	for (i = 0; i <= e - s; ++i)
-		update_sm_ah(&sa_dev->port[i].update_task);
+		if (rdma_port_get_transport(device, i + 1) == RDMA_TRANSPORT_IB)
+			update_sm_ah(&sa_dev->port[i].update_task);
 
 	return;
 
 err:
 	while (--i >= 0)
-		ib_unregister_mad_agent(sa_dev->port[i].agent);
+		if (rdma_port_get_transport(device, i + 1) == RDMA_TRANSPORT_IB)
+			ib_unregister_mad_agent(sa_dev->port[i].agent);
 
 	kfree(sa_dev);
 

@@ -1071,9 +1078,11 @@ static void ib_sa_remove_one(struct ib_device *device)
 	flush_scheduled_work();
 
 	for (i = 0; i <= sa_dev->end_port - sa_dev->start_port; ++i) {
-		ib_unregister_mad_agent(sa_dev->port[i].agent);
-		if (sa_dev->port[i].sm_ah)
-			kref_put(&sa_dev->port[i].sm_ah->ref, free_sm_ah);
+		if (rdma_port_get_transport(device, i + 1) == RDMA_TRANSPORT_IB) {
+			ib_unregister_mad_agent(sa_dev->port[i].agent);
+			if (sa_dev->port[i].sm_ah)
+				kref_put(&sa_dev->port[i].sm_ah->ref, free_sm_ah);
+		}
 	}
 
 	kfree(sa_dev);
drivers/infiniband/core/ucm.c +6 -2
index f504c9b..43700b4 100644
@@ -1240,11 +1240,15 @@ static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
 static void ib_ucm_add_one(struct ib_device *device)
 {
 	struct ib_ucm_device *ucm_dev;
+	int i;
 
-	if (!device->alloc_ucontext ||
-	    rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
+	if (!device->alloc_ucontext)
 		return;
 
+	for (i = 1; i <= device->phys_port_cnt; ++i)
+		if (rdma_port_get_transport(device, i) != RDMA_TRANSPORT_IB)
+			return;
+
 	ucm_dev = kzalloc(sizeof *ucm_dev, GFP_KERNEL);
 	if (!ucm_dev)
 		return;
drivers/infiniband/core/ucma.c +1 -1
index f1cbd26..fd831fd 100644
@@ -616,7 +616,7 @@ static ssize_t ucma_query_route(struct ucma_file *file,
 
 	resp.node_guid = (__force __u64) ctx->cm_id->device->node_guid;
 	resp.port_num = ctx->cm_id->port_num;
-	switch (rdma_node_get_transport(ctx->cm_id->device->node_type)) {
+	switch (rdma_port_get_transport(ctx->cm_id->device, ctx->cm_id->port_num)) {
 	case RDMA_TRANSPORT_IB:
 		ucma_copy_ib_route(&resp, &ctx->cm_id->route);
 		break;
drivers/infiniband/core/user_mad.c +5 -1
index 7de0296..6d6795d 100644
@@ -1114,7 +1114,7 @@ static void ib_umad_add_one(struct ib_device *device)
 	struct ib_umad_device *umad_dev;
 	int s, e, i;
 
-	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
+	if (!rdma_is_transport_supported(device, RDMA_TRANSPORT_IB))
 		return;
 
 	if (device->node_type == RDMA_NODE_IB_SWITCH)

@@ -1124,6 +1124,10 @@ static void ib_umad_add_one(struct ib_device *device)
 		e = device->phys_port_cnt;
 	}
 
+	for (i = s; i <= e; ++i)
+		if (rdma_port_get_transport(device, i) != RDMA_TRANSPORT_IB)
+			return;
+
 	umad_dev = kzalloc(sizeof *umad_dev +
 			   (e - s + 1) * sizeof (struct ib_umad_port),
 			   GFP_KERNEL);
drivers/infiniband/core/verbs.c +23 -2
index a7da9be..d81e217 100644
@@ -77,7 +77,7 @@ enum ib_rate mult_to_ib_rate(int mult)
 }
 EXPORT_SYMBOL(mult_to_ib_rate);
 
-enum rdma_transport_type
+static enum rdma_transport_type
 rdma_node_get_transport(enum rdma_node_type node_type)
 {
 	switch (node_type) {

@@ -92,7 +92,28 @@ rdma_node_get_transport(enum rdma_node_type node_type)
 		return 0;
 	}
 }
-EXPORT_SYMBOL(rdma_node_get_transport);
+
+enum rdma_transport_type rdma_port_get_transport(struct ib_device *device,
+						 u8 port_num)
+{
+	return device->get_port_transport ?
+		device->get_port_transport(device, port_num) :
+		rdma_node_get_transport(device->node_type);
+}
+EXPORT_SYMBOL(rdma_port_get_transport);
+
+int rdma_is_transport_supported(struct ib_device *device,
+				enum rdma_transport_type transport)
+{
+	int i;
+
+	for (i = 1; i <= device->phys_port_cnt; ++i)
+		if (rdma_port_get_transport(device, i) == transport)
+			return 1;
+
+	return 0;
+}
+EXPORT_SYMBOL(rdma_is_transport_supported);
 
 /* Protection domains */
drivers/infiniband/ulp/ipoib/ipoib_main.c +6 -6
index 2bf5116..de89e80 100644
@@ -1343,9 +1343,6 @@ static void ipoib_add_one(struct ib_device *device)
 	struct ipoib_dev_priv *priv;
 	int s, e, p;
 
-	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
-		return;
-
 	dev_list = kmalloc(sizeof *dev_list, GFP_KERNEL);
 	if (!dev_list)
 		return;

@@ -1361,6 +1358,9 @@ static void ipoib_add_one(struct ib_device *device)
 	}
 
 	for (p = s; p <= e; ++p) {
+		if (rdma_port_get_transport(device, p) != RDMA_TRANSPORT_IB)
+			continue;
+
 		dev = ipoib_add_port("ib%d", device, p);
 		if (!IS_ERR(dev)) {
 			priv = netdev_priv(dev);

@@ -1376,12 +1376,12 @@ static void ipoib_remove_one(struct ib_device *device)
 	struct ipoib_dev_priv *priv, *tmp;
 	struct list_head *dev_list;
 
-	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
-		return;
-
 	dev_list = ib_get_client_data(device, &ipoib_client);
 
 	list_for_each_entry_safe(priv, tmp, dev_list, list) {
+		if (rdma_port_get_transport(device, priv->port) != RDMA_TRANSPORT_IB)
+			continue;
+
 		ib_unregister_event_handler(&priv->event_handler);
 
 		rtnl_lock();
include/rdma/ib_verbs.h +8 -3
index c179318..4cf42f3 100644
@@ -72,9 +72,6 @@ enum rdma_transport_type {
 	RDMA_TRANSPORT_IWARP
 };
 
-enum rdma_transport_type
-rdma_node_get_transport(enum rdma_node_type node_type) __attribute_const__;
-
 enum ib_device_cap_flags {
 	IB_DEVICE_RESIZE_MAX_WR		= 1,
 	IB_DEVICE_BAD_PKEY_CNTR		= (1<<1),

@@ -298,6 +295,7 @@ struct ib_port_attr {
 	u8			active_width;
 	u8			active_speed;
 	u8                      phys_state;
+	enum rdma_transport_type	transport;
 };
 
 enum ib_device_modify_flags {

@@ -1003,6 +1001,8 @@ struct ib_device {
 	int		           (*query_port)(struct ib_device *device,
 						 u8 port_num,
 						 struct ib_port_attr *port_attr);
+	enum rdma_transport_type   (*get_port_transport)(struct ib_device *device,
+							 u8 port_num);
 	int		           (*query_gid)(struct ib_device *device,
 						u8 port_num, int index,
 						union ib_gid *gid);

@@ -1213,6 +1213,11 @@ int ib_query_device(struct ib_device *device,
 int ib_query_port(struct ib_device *device,
 		  u8 port_num, struct ib_port_attr *port_attr);
 
+enum rdma_transport_type rdma_port_get_transport(struct ib_device *device,
+						 u8 port_num);
+int rdma_is_transport_supported(struct ib_device *device,
+				enum rdma_transport_type transport);
+
 int ib_query_gid(struct ib_device *device,
 		 u8 port_num, int index, union ib_gid *gid);
net/sunrpc/xprtrdma/svc_rdma_recvfrom.c +1 -2
index 9e88438..1131c74 100644
@@ -338,8 +338,7 @@ static int rdma_set_ctxt_sge(struct svcxprt_rdma *xprt,
 static int rdma_read_max_sge(struct svcxprt_rdma *xprt, int sge_count)
 {
 	if ((RDMA_TRANSPORT_IWARP ==
-	     rdma_node_get_transport(xprt->sc_cm_id->
-				     device->node_type))
+	     rdma_port_get_transport(xprt->sc_cm_id->device, xprt->sc_cm_id->port_num))
 	    && sge_count > 1)
 		return 1;
 	else
net/sunrpc/xprtrdma/svc_rdma_transport.c +1 -1
index 3fa5751..4e286f7 100644
@@ -977,7 +977,7 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
 	/*
 	 * Determine if a DMA MR is required and if so, what privs are required
 	 */
-	switch (rdma_node_get_transport(newxprt->sc_cm_id->device->node_type)) {
+	switch (rdma_port_get_transport(newxprt->sc_cm_id->device, newxprt->sc_cm_id->port_num)) {
 	case RDMA_TRANSPORT_IWARP:
 		newxprt->sc_dev_caps |= SVCRDMA_DEVCAP_READ_W_INV;
 		if (!(newxprt->sc_dev_caps & SVCRDMA_DEVCAP_FAST_REG)) {
Signature · 1 line
1.6.5.2