Skip to contentSkip to conversation

ib_core: RoCEE support only QP1

PATCH Superseded Eli Cohen 3 participants 3 messages Jan 5, 2010 – Jan 6, 2010 1 message not in the archive InfiniBand

v7 of 8 Superseded Latest is v8 →

lore
Share & tools
lore

Eli Cohen

author
#0 lore raw

PATCH v7 2/9 ib_core: RoCEE support only QP1

Since RoCEE is using Ethernet as its link layer, there is no need for QP0. QP1 is still needed since it handles communications between CM agents. This patch will create only QP1 for RoCEE ports.

---
 drivers/infiniband/core/agent.c     |   37 ++++++++++++-------
 drivers/infiniband/core/mad.c       |   66 ++++++++++++++++++++++++-----------
 drivers/infiniband/core/multicast.c |   25 +++++++++++--
 drivers/infiniband/core/sa_query.c  |   39 +++++++++++++-------
 4 files changed, 114 insertions(+), 53 deletions(-)
drivers/infiniband/core/agent.c +23 -14
index ae7c288..964f4fb 100644
@@ -48,6 +48,8 @@
 struct ib_agent_port_private {
 	struct list_head port_list;
 	struct ib_mad_agent *agent[2];
+	struct ib_device    *device;
+	u8		     port_num;
 };
 
 static DEFINE_SPINLOCK(ib_agent_port_list_lock);
@@ -58,11 +60,10 @@ __ib_get_agent_port(struct ib_device *device, int port_num)
 {
 	struct ib_agent_port_private *entry;
 
-	list_for_each_entry(entry, &ib_agent_port_list, port_list) {
-		if (entry->agent[0]->device == device &&
-		    entry->agent[0]->port_num == port_num)
+	list_for_each_entry(entry, &ib_agent_port_list, port_list)
+		if (entry->device == device && entry->port_num == port_num)
 			return entry;
-	}
+
 	return NULL;
 }
 
@@ -155,14 +156,16 @@ int ib_agent_port_open(struct ib_device *device, int port_num)
 		goto error1;
 	}
 
-	/* Obtain send only MAD agent for SMI QP */
-	port_priv->agent[0] = ib_register_mad_agent(device, port_num,
-						    IB_QPT_SMI, NULL, 0,
-						    &agent_send_handler,
-						    NULL, NULL);
-	if (IS_ERR(port_priv->agent[0])) {
-		ret = PTR_ERR(port_priv->agent[0]);
-		goto error2;
+	if (rdma_port_link_layer(device, port_num) == IB_LINK_LAYER_INFINIBAND) {
+		/* Obtain send only MAD agent for SMI QP */
+		port_priv->agent[0] = ib_register_mad_agent(device, port_num,
+							    IB_QPT_SMI, NULL, 0,
+							    &agent_send_handler,
+							    NULL, NULL);
+		if (IS_ERR(port_priv->agent[0])) {
+			ret = PTR_ERR(port_priv->agent[0]);
+			goto error2;
+		}
 	}
 
 	/* Obtain send only MAD agent for GSI QP */
@@ -175,6 +178,9 @@ int ib_agent_port_open(struct ib_device *device, int port_num)
 		goto error3;
 	}
 
+	port_priv->device = device;
+	port_priv->port_num = port_num;
+
 	spin_lock_irqsave(&ib_agent_port_list_lock, flags);
 	list_add_tail(&port_priv->port_list, &ib_agent_port_list);
 	spin_unlock_irqrestore(&ib_agent_port_list_lock, flags);
@@ -182,7 +188,8 @@ int ib_agent_port_open(struct ib_device *device, int port_num)
 	return 0;
 
 error3:
-	ib_unregister_mad_agent(port_priv->agent[0]);
+	if (rdma_port_link_layer(device, port_num) == IB_LINK_LAYER_INFINIBAND)
+		ib_unregister_mad_agent(port_priv->agent[0]);
 error2:
 	kfree(port_priv);
 error1:
@@ -205,7 +212,9 @@ int ib_agent_port_close(struct ib_device *device, int port_num)
 	spin_unlock_irqrestore(&ib_agent_port_list_lock, flags);
 
 	ib_unregister_mad_agent(port_priv->agent[1]);
-	ib_unregister_mad_agent(port_priv->agent[0]);
+	if (rdma_port_link_layer(device, port_num) == IB_LINK_LAYER_INFINIBAND)
+		ib_unregister_mad_agent(port_priv->agent[0]);
+
 	kfree(port_priv);
 	return 0;
 }
drivers/infiniband/core/mad.c +45 -21

125 lines not on this page · raw message

drivers/infiniband/core/multicast.c +21 -4
index 8d82ba1..55f24ff 100644
@@ -773,6 +773,10 @@ static void mcast_event_handler(struct ib_event_handler *handler,
 	int index;
 
 	dev = container_of(handler, struct mcast_device, event_handler);
+	if (rdma_port_link_layer(dev->device, event->element.port_num) !=
+	    IB_LINK_LAYER_INFINIBAND)
+		return;
+
 	index = event->element.port_num - dev->start_port;
 
 	switch (event->event) {
@@ -795,11 +799,12 @@ static void mcast_add_one(struct ib_device *device)
 	struct mcast_device *dev;
 	struct mcast_port *port;
 	int i;
+	int count = 0;
 
 	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
 		return;
 
-	dev = kmalloc(sizeof *dev + device->phys_port_cnt * sizeof *port,
+	dev = kzalloc(sizeof *dev + device->phys_port_cnt * sizeof *port,
 		      GFP_KERNEL);
 	if (!dev)
 		return;
@@ -812,6 +817,9 @@ static void mcast_add_one(struct ib_device *device)
 	}
 
 	for (i = 0; i <= dev->end_port - dev->start_port; i++) {
+		if (rdma_port_link_layer(device, dev->start_port + i) !=
+		    IB_LINK_LAYER_INFINIBAND)
+			continue;
 		port = &dev->port[i];
 		port->dev = dev;
 		port->port_num = dev->start_port + i;
@@ -819,6 +827,12 @@ static void mcast_add_one(struct ib_device *device)
 		port->table = RB_ROOT;
 		init_completion(&port->comp);
 		atomic_set(&port->refcount, 1);
+		++count;
+	}
+
+	if (!count) {
+		kfree(dev);
+		return;
 	}
 
 	dev->device = device;
@@ -842,9 +856,12 @@ static void mcast_remove_one(struct ib_device *device)
 	flush_workqueue(mcast_wq);
 
 	for (i = 0; i <= dev->end_port - dev->start_port; i++) {
-		port = &dev->port[i];
-		deref_port(port);
-		wait_for_completion(&port->comp);
+		if (rdma_port_link_layer(device, dev->start_port + i) ==
+		    IB_LINK_LAYER_INFINIBAND) {
+			port = &dev->port[i];
+			deref_port(port);
+			wait_for_completion(&port->comp);
+		}
 	}
 
 	kfree(dev);
drivers/infiniband/core/sa_query.c +25 -14
index 7e1ffd8..824e29b 100644
@@ -416,14 +416,17 @@ static void ib_sa_event(struct ib_event_handler *handler, struct ib_event *event
 		struct ib_sa_port *port =
 			&sa_dev->port[event->element.port_num - sa_dev->start_port];
 
-		spin_lock_irqsave(&port->ah_lock, flags);
-		if (port->sm_ah)
-			kref_put(&port->sm_ah->ref, free_sm_ah);
-		port->sm_ah = NULL;
-		spin_unlock_irqrestore(&port->ah_lock, flags);
-
-		schedule_work(&sa_dev->port[event->element.port_num -
-					    sa_dev->start_port].update_task);
+		if (rdma_port_link_layer(handler->device, port->port_num) == IB_LINK_LAYER_INFINIBAND) {
+			spin_lock_irqsave(&port->ah_lock, flags);
+			if (port->sm_ah)
+				kref_put(&port->sm_ah->ref, free_sm_ah);
+			port->sm_ah = NULL;
+			spin_unlock_irqrestore(&port->ah_lock, flags);
+
+			schedule_work(&sa_dev->port[event->element.port_num -
+				      sa_dev->start_port].update_task);
+		}
+
 	}
 }
 
@@ -1007,7 +1010,7 @@ static void ib_sa_add_one(struct ib_device *device)
 		e = device->phys_port_cnt;
 	}
 
-	sa_dev = kmalloc(sizeof *sa_dev +
+	sa_dev = kzalloc(sizeof *sa_dev +
 			 (e - s + 1) * sizeof (struct ib_sa_port),
 			 GFP_KERNEL);
 	if (!sa_dev)
@@ -1017,6 +1020,9 @@ static void ib_sa_add_one(struct ib_device *device)
 	sa_dev->end_port   = e;
 
 	for (i = 0; i <= e - s; ++i) {
+		if (rdma_port_link_layer(device, i + 1) != IB_LINK_LAYER_INFINIBAND)
+			continue;
+
 		sa_dev->port[i].sm_ah    = NULL;
 		sa_dev->port[i].port_num = i + s;
 		spin_lock_init(&sa_dev->port[i].ah_lock);
@@ -1045,13 +1051,15 @@ static void ib_sa_add_one(struct ib_device *device)
 		goto err;
 
 	for (i = 0; i <= e - s; ++i)
-		update_sm_ah(&sa_dev->port[i].update_task);
+		if (rdma_port_link_layer(device, i + 1) == IB_LINK_LAYER_INFINIBAND)
+			update_sm_ah(&sa_dev->port[i].update_task);
 
 	return;
 
 err:
 	while (--i >= 0)
-		ib_unregister_mad_agent(sa_dev->port[i].agent);
+		if (rdma_port_link_layer(device, i + 1) == IB_LINK_LAYER_INFINIBAND)
+			ib_unregister_mad_agent(sa_dev->port[i].agent);
 
 	kfree(sa_dev);
 
@@ -1071,9 +1079,12 @@ static void ib_sa_remove_one(struct ib_device *device)
 	flush_scheduled_work();
 
 	for (i = 0; i <= sa_dev->end_port - sa_dev->start_port; ++i) {
-		ib_unregister_mad_agent(sa_dev->port[i].agent);
-		if (sa_dev->port[i].sm_ah)
-			kref_put(&sa_dev->port[i].sm_ah->ref, free_sm_ah);
+		if (rdma_port_link_layer(device, i + 1) == IB_LINK_LAYER_INFINIBAND) {
+			ib_unregister_mad_agent(sa_dev->port[i].agent);
+			if (sa_dev->port[i].sm_ah)
+				kref_put(&sa_dev->port[i].sm_ah->ref, free_sm_ah);
+		}
+
 	}
 
 	kfree(sa_dev);
Signature · 6 lines
1.6.6

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
#1 lore raw

Review static void ib_mad_init_device(struct ib_device

Quoted 1 line
(quoted diff) +0 -0
@@ -2941,21 +2960,24 @@ static void ib_mad_init_device(struct ib_device

This is at the top of ib_mad_init_device():

        if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
                return;

...

drivers/infiniband/core/mad.c static void ib_mad_init_device(struct ib_device *device) · line 2963

Quoted 8 lines

*device)

	return;

error_agent:

drivers/infiniband/core/mad.c +1 -3
-	if (ib_mad_port_close(device, i))
-		printk(KERN_ERR PFX "Couldn't close %s port %d\n",
-		       device->name, i);
+	if (rdma_node_get_transport(device->node_type) == RDMA_TRANSPORT_IB)

... so this if statement will always be true. Using rdma_port_link_layer() doesn't seem quite right for QP 1 support, so I'm not sure that you want this check at all.

Quoted 16 lines
drivers/infiniband/core/mad.c +3 -0
+		if (ib_mad_port_close(device, i))
+			printk(KERN_ERR PFX "Couldn't close %s port %d\n",
+			       device->name, i);

error:

	i--;

	while (i >= start) {
drivers/infiniband/core/mad.c +1 -7
-		if (ib_agent_port_close(device, i))
-			printk(KERN_ERR PFX "Couldn't close %s port %d "
-			       "for agents\n",
-			       device->name, i);
-		if (ib_mad_port_close(device, i))
-			printk(KERN_ERR PFX "Couldn't close %s port %d\n",
-			       device->name, i);
+		if (rdma_node_get_transport(device->node_type) ==
Quoted 1 line

RDMA_TRANSPORT_IB) {

Same here.

drivers/infiniband/core/mad.c static void ib_mad_init_device(struct ib_device *device)

Quoted 6 lines
drivers/infiniband/core/mad.c +6 -0
+			if (ib_agent_port_close(device, i))
+				printk(KERN_ERR PFX "Couldn't close %s port %d "
+				       "for agents\n",
+				       device->name, i);
+			if (ib_mad_port_close(device, i))
+				printk(KERN_ERR PFX "Couldn't close %s port

%d\n",

Quoted 9 lines
(quoted diff) +2 -0
+				       device->name, i);
+		}
		i--;
	}
}
drivers/infiniband/core/mad.c +0 -0
@@ -2972,13 +2994,15 @@ static void ib_mad_remove_device(struct ib_device
 *device)
 		cur_port = 1;
 	}

drivers/infiniband/core/mad.c static void ib_mad_remove_device(struct ib_device · line 2999

Quoted 10 lines
drivers/infiniband/core/mad.c +1 -7
 	for (i = 0; i < num_ports; i++, cur_port++) {
-		if (ib_agent_port_close(device, cur_port))
-			printk(KERN_ERR PFX "Couldn't close %s port %d "
-			       "for agents\n",
-			       device->name, cur_port);
-		if (ib_mad_port_close(device, cur_port))
-			printk(KERN_ERR PFX "Couldn't close %s port %d\n",
-			       device->name, cur_port);
+		if (rdma_node_get_transport(device->node_type) ==
 RDMA_TRANSPORT_IB) {

It would be more efficient to move this check outside of the for loop, similar to the check in ib_mad_init_device().

drivers/infiniband/core/mad.c static void ib_mad_remove_device(struct ib_device *device)

Quoted 6 lines
drivers/infiniband/core/mad.c +6 -0
+			if (ib_agent_port_close(device, cur_port))
+				printk(KERN_ERR PFX "Couldn't close %s port %d "
+				       "for agents\n",
+				       device->name, cur_port);
+			if (ib_mad_port_close(device, cur_port))
+				printk(KERN_ERR PFX "Couldn't close %s port

%d\n",

Quoted 5 lines
drivers/infiniband/core/mad.c +2 -0
+				       device->name, cur_port);
+		}
 	}

}

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): 3C9FA4CD26174188832FDD339D658C66-Zpru7NauK7drdx17CPfAsdBPR1lH4CV8@public.gmane.org

#3 lore raw

Sean, you're right. These conditionals are not required as RoCEE and IB have the same node transport type. All that needs to be done is remove them. I'll fix that.

Quoted 78 lines · Sean Hefty

Review static void ib_mad_init_device(struct ib_device

Quoted 1 line
(quoted diff) +0 -0
@@ -2941,21 +2960,24 @@ static void ib_mad_init_device(struct ib_device

This is at the top of ib_mad_init_device():

        if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
                return;

...

drivers/infiniband/core/mad.c static void ib_mad_init_device(struct ib_device *device) · line 2963

Quoted 8 lines

*device)

	return;

error_agent:

drivers/infiniband/core/mad.c +1 -3
-	if (ib_mad_port_close(device, i))
-		printk(KERN_ERR PFX "Couldn't close %s port %d\n",
-		       device->name, i);
+	if (rdma_node_get_transport(device->node_type) == RDMA_TRANSPORT_IB)

... so this if statement will always be true. Using rdma_port_link_layer() doesn't seem quite right for QP 1 support, so I'm not sure that you want this check at all.

Quoted 16 lines
drivers/infiniband/core/mad.c +3 -0
+		if (ib_mad_port_close(device, i))
+			printk(KERN_ERR PFX "Couldn't close %s port %d\n",
+			       device->name, i);

error:

	i--;

	while (i >= start) {
drivers/infiniband/core/mad.c +1 -7
-		if (ib_agent_port_close(device, i))
-			printk(KERN_ERR PFX "Couldn't close %s port %d "
-			       "for agents\n",
-			       device->name, i);
-		if (ib_mad_port_close(device, i))
-			printk(KERN_ERR PFX "Couldn't close %s port %d\n",
-			       device->name, i);
+		if (rdma_node_get_transport(device->node_type) ==
Quoted 1 line

RDMA_TRANSPORT_IB) {

Same here.

drivers/infiniband/core/mad.c static void ib_mad_init_device(struct ib_device *device)

Quoted 6 lines
drivers/infiniband/core/mad.c +6 -0
+			if (ib_agent_port_close(device, i))
+				printk(KERN_ERR PFX "Couldn't close %s port %d "
+				       "for agents\n",
+				       device->name, i);
+			if (ib_mad_port_close(device, i))
+				printk(KERN_ERR PFX "Couldn't close %s port

%d\n",

Quoted 9 lines
(quoted diff) +2 -0
+				       device->name, i);
+		}
		i--;
	}
}
drivers/infiniband/core/mad.c +0 -0
@@ -2972,13 +2994,15 @@ static void ib_mad_remove_device(struct ib_device
 *device)
 		cur_port = 1;
 	}

drivers/infiniband/core/mad.c static void ib_mad_remove_device(struct ib_device · line 2999

Quoted 10 lines
drivers/infiniband/core/mad.c +1 -7
 	for (i = 0; i < num_ports; i++, cur_port++) {
-		if (ib_agent_port_close(device, cur_port))
-			printk(KERN_ERR PFX "Couldn't close %s port %d "
-			       "for agents\n",
-			       device->name, cur_port);
-		if (ib_mad_port_close(device, cur_port))
-			printk(KERN_ERR PFX "Couldn't close %s port %d\n",
-			       device->name, cur_port);
+		if (rdma_node_get_transport(device->node_type) ==
 RDMA_TRANSPORT_IB) {

It would be more efficient to move this check outside of the for loop, similar to the check in ib_mad_init_device().

drivers/infiniband/core/mad.c static void ib_mad_remove_device(struct ib_device *device)

Quoted 6 lines
drivers/infiniband/core/mad.c +6 -0
+			if (ib_agent_port_close(device, cur_port))
+				printk(KERN_ERR PFX "Couldn't close %s port %d "
+				       "for agents\n",
+				       device->name, cur_port);
+			if (ib_mad_port_close(device, cur_port))
+				printk(KERN_ERR PFX "Couldn't close %s port

%d\n",

Quoted 10 lines
Quoted 5 lines
drivers/infiniband/core/mad.c +2 -0
+				       device->name, cur_port);
+		}
 	}

}

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html