Skip to contentSkip to conversation

ib_core: IBoE support only QP1

PATCH In review Eli Cohen 4 participants 10 messages Feb 18, 2010 – May 17, 2010 7 messages not in the archive InfiniBand

lore
Share & tools
lore

Eli Cohen

author
#0 lore raw

PATCH v8 2/11 ib_core: IBoE support only QP1

Since IBoE is using Ethernet as its link layer, there is no central management entity so there is need for QP0. QP1 is still needed since it handles communications between CM agents. This patch will create only QP1 for IBoE ports.

---
Changes from v7:

1. Remove "always true" code
2. Fix failure to initialize port ah_lock in ib_sa_add_one

 drivers/infiniband/core/agent.c     |   37 +++++++++++++++++++------------
 drivers/infiniband/core/mad.c       |   27 +++++++++++++++++++---
 drivers/infiniband/core/multicast.c |   25 ++++++++++++++++++---
 drivers/infiniband/core/sa_query.c  |   41 ++++++++++++++++++++++------------
 4 files changed, 93 insertions(+), 37 deletions(-)
drivers/infiniband/core/agent.c +23 -14
index ae7c288..964f4fb 100644
@@ -48,6 +48,8 @@
 struct ib_agent_port_private {
 	struct list_head port_list;
 	struct ib_mad_agent *agent[2];
+	struct ib_device    *device;
+	u8		     port_num;
 };
 
 static DEFINE_SPINLOCK(ib_agent_port_list_lock);
@@ -58,11 +60,10 @@ __ib_get_agent_port(struct ib_device *device, int port_num)
 {
 	struct ib_agent_port_private *entry;
 
-	list_for_each_entry(entry, &ib_agent_port_list, port_list) {
-		if (entry->agent[0]->device == device &&
-		    entry->agent[0]->port_num == port_num)
+	list_for_each_entry(entry, &ib_agent_port_list, port_list)
+		if (entry->device == device && entry->port_num == port_num)
 			return entry;
-	}
+
 	return NULL;
 }
 
@@ -155,14 +156,16 @@ int ib_agent_port_open(struct ib_device *device, int port_num)
 		goto error1;
 	}
 
-	/* Obtain send only MAD agent for SMI QP */
-	port_priv->agent[0] = ib_register_mad_agent(device, port_num,
-						    IB_QPT_SMI, NULL, 0,
-						    &agent_send_handler,
-						    NULL, NULL);
-	if (IS_ERR(port_priv->agent[0])) {
-		ret = PTR_ERR(port_priv->agent[0]);
-		goto error2;
+	if (rdma_port_link_layer(device, port_num) == IB_LINK_LAYER_INFINIBAND) {
+		/* Obtain send only MAD agent for SMI QP */
+		port_priv->agent[0] = ib_register_mad_agent(device, port_num,
+							    IB_QPT_SMI, NULL, 0,
+							    &agent_send_handler,
+							    NULL, NULL);
+		if (IS_ERR(port_priv->agent[0])) {
+			ret = PTR_ERR(port_priv->agent[0]);
+			goto error2;
+		}
 	}
 
 	/* Obtain send only MAD agent for GSI QP */
@@ -175,6 +178,9 @@ int ib_agent_port_open(struct ib_device *device, int port_num)
 		goto error3;
 	}
 
+	port_priv->device = device;
+	port_priv->port_num = port_num;
+
 	spin_lock_irqsave(&ib_agent_port_list_lock, flags);
 	list_add_tail(&port_priv->port_list, &ib_agent_port_list);
 	spin_unlock_irqrestore(&ib_agent_port_list_lock, flags);
@@ -182,7 +188,8 @@ int ib_agent_port_open(struct ib_device *device, int port_num)
 	return 0;
 
 error3:
-	ib_unregister_mad_agent(port_priv->agent[0]);
+	if (rdma_port_link_layer(device, port_num) == IB_LINK_LAYER_INFINIBAND)
+		ib_unregister_mad_agent(port_priv->agent[0]);
 error2:
 	kfree(port_priv);
 error1:
@@ -205,7 +212,9 @@ int ib_agent_port_close(struct ib_device *device, int port_num)
 	spin_unlock_irqrestore(&ib_agent_port_list_lock, flags);
 
 	ib_unregister_mad_agent(port_priv->agent[1]);
-	ib_unregister_mad_agent(port_priv->agent[0]);
+	if (rdma_port_link_layer(device, port_num) == IB_LINK_LAYER_INFINIBAND)
+		ib_unregister_mad_agent(port_priv->agent[0]);
+
 	kfree(port_priv);
 	return 0;
 }
drivers/infiniband/core/mad.c +23 -4
index 7522008..f546ab7 100644
@@ -2610,6 +2610,9 @@ static void cleanup_recv_queue(struct ib_mad_qp_info *qp_info)
 	struct ib_mad_private *recv;
 	struct ib_mad_list_head *mad_list;
 
+	if (!qp_info->qp)
+		return;
+
 	while (!list_empty(&qp_info->recv_queue.list)) {
 
 		mad_list = list_entry(qp_info->recv_queue.list.next,
@@ -2651,6 +2654,9 @@ static int ib_mad_port_start(struct ib_mad_port_private *port_priv)
 
 	for (i = 0; i < IB_MAD_QPS_CORE; i++) {
 		qp = port_priv->qp_info[i].qp;
+		if (!qp)
+			continue;
+
 		/*
 		 * PKey index for QP1 is irrelevant but
 		 * one is needed for the Reset to Init transition
@@ -2692,6 +2698,9 @@ static int ib_mad_port_start(struct ib_mad_port_private *port_priv)
 	}
 
 	for (i = 0; i < IB_MAD_QPS_CORE; i++) {
+		if (!port_priv->qp_info[i].qp)
+			continue;
+
 		ret = ib_mad_post_receive_mads(&port_priv->qp_info[i], NULL);
 		if (ret) {
 			printk(KERN_ERR PFX "Couldn't post receive WRs\n");
@@ -2770,6 +2779,9 @@ error:
 
 static void destroy_mad_qp(struct ib_mad_qp_info *qp_info)
 {
+	if (!qp_info->qp)
+		return;
+
 	ib_destroy_qp(qp_info->qp);
 	kfree(qp_info->snoop_table);
 }
@@ -2785,6 +2797,7 @@ static int ib_mad_port_open(struct ib_device *device,
 	struct ib_mad_port_private *port_priv;
 	unsigned long flags;
 	char name[sizeof "ib_mad123"];
+	int has_smi;
 
 	/* Create new device info */
 	port_priv = kzalloc(sizeof *port_priv, GFP_KERNEL);
@@ -2800,7 +2813,11 @@ static int ib_mad_port_open(struct ib_device *device,
 	init_mad_qp(port_priv, &port_priv->qp_info[0]);
 	init_mad_qp(port_priv, &port_priv->qp_info[1]);
 
-	cq_size = (mad_sendq_size + mad_recvq_size) * 2;
+	cq_size = mad_sendq_size + mad_recvq_size;
+	has_smi = rdma_port_link_layer(device, port_num) == IB_LINK_LAYER_INFINIBAND;
+	if (has_smi)
+		cq_size *= 2;
+
 	port_priv->cq = ib_create_cq(port_priv->device,
 				     ib_mad_thread_completion_handler,
 				     NULL, port_priv, cq_size, 0);
@@ -2824,9 +2841,11 @@ static int ib_mad_port_open(struct ib_device *device,
 		goto error5;
 	}
 
-	ret = create_mad_qp(&port_priv->qp_info[0], IB_QPT_SMI);
-	if (ret)
-		goto error6;
+	if (has_smi) {
+		ret = create_mad_qp(&port_priv->qp_info[0], IB_QPT_SMI);
+		if (ret)
+			goto error6;
+	}
 	ret = create_mad_qp(&port_priv->qp_info[1], IB_QPT_GSI);
 	if (ret)
 		goto error7;
drivers/infiniband/core/multicast.c +21 -4
index 8d82ba1..55f24ff 100644
@@ -773,6 +773,10 @@ static void mcast_event_handler(struct ib_event_handler *handler,
 	int index;
 
 	dev = container_of(handler, struct mcast_device, event_handler);
+	if (rdma_port_link_layer(dev->device, event->element.port_num) !=
+	    IB_LINK_LAYER_INFINIBAND)
+		return;
+
 	index = event->element.port_num - dev->start_port;
 
 	switch (event->event) {
@@ -795,11 +799,12 @@ static void mcast_add_one(struct ib_device *device)
 	struct mcast_device *dev;
 	struct mcast_port *port;
 	int i;
+	int count = 0;
 
 	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
 		return;
 
-	dev = kmalloc(sizeof *dev + device->phys_port_cnt * sizeof *port,
+	dev = kzalloc(sizeof *dev + device->phys_port_cnt * sizeof *port,
 		      GFP_KERNEL);
 	if (!dev)
 		return;
@@ -812,6 +817,9 @@ static void mcast_add_one(struct ib_device *device)
 	}
 
 	for (i = 0; i <= dev->end_port - dev->start_port; i++) {
+		if (rdma_port_link_layer(device, dev->start_port + i) !=
+		    IB_LINK_LAYER_INFINIBAND)
+			continue;
 		port = &dev->port[i];
 		port->dev = dev;
 		port->port_num = dev->start_port + i;
@@ -819,6 +827,12 @@ static void mcast_add_one(struct ib_device *device)
 		port->table = RB_ROOT;
 		init_completion(&port->comp);
 		atomic_set(&port->refcount, 1);
+		++count;
+	}
+
+	if (!count) {
+		kfree(dev);
+		return;
 	}
 
 	dev->device = device;
@@ -842,9 +856,12 @@ static void mcast_remove_one(struct ib_device *device)
 	flush_workqueue(mcast_wq);
 
 	for (i = 0; i <= dev->end_port - dev->start_port; i++) {
-		port = &dev->port[i];
-		deref_port(port);
-		wait_for_completion(&port->comp);
+		if (rdma_port_link_layer(device, dev->start_port + i) ==
+		    IB_LINK_LAYER_INFINIBAND) {
+			port = &dev->port[i];
+			deref_port(port);
+			wait_for_completion(&port->comp);
+		}
 	}
 
 	kfree(dev);
drivers/infiniband/core/sa_query.c +26 -15

78 lines not on this page · raw message

Signature · 6 lines
1.7.0

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
#1 lore raw
Quoted 13 lines
drivers/infiniband/core/multicast.c +2 -1
@@ -795,11 +799,12 @@ static void mcast_add_one(struct ib_device *device)
 	struct mcast_device *dev;
 	struct mcast_port *port;
 	int i;
+	int count = 0;
 
 	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
 		return;
 
-	dev = kmalloc(sizeof *dev + device->phys_port_cnt * sizeof *port,
+	dev = kzalloc(sizeof *dev + device->phys_port_cnt * sizeof *port,
 

drivers/infiniband/core/multicast.c static void ib_sa_add_one(struct ib_device *device) · line 1013

Quoted 5 lines
drivers/infiniband/core/multicast.c +1 -1
@@ -1007,7 +1010,7 @@ static void ib_sa_add_one(struct ib_device *device)
 		e = device->phys_port_cnt;
 	}
 
-	sa_dev = kmalloc(sizeof *sa_dev +
+	sa_dev = kzalloc(sizeof *sa_dev +

Do you happen to remember why you needed these kmalloc -> kzalloc conversions?

Signature · 6 lines
Roland Dreier <[email protected]> || For corporate legal information go to:
http://www.cisco.com/web/about/doing_business/legal/cri/index.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): ada1vdqaqpc.fsf-BjVyx320WGW9gfZ95n9DRSW4+XlvGpQz@public.gmane.org

#3 lore raw
Quoted 12 lines · Roland Dreier
Quoted 11 lines
drivers/infiniband/core/multicast.c +2 -1
@@ -795,11 +799,12 @@ static void mcast_add_one(struct ib_device *device)
 	struct mcast_device *dev;
 	struct mcast_port *port;
 	int i;
+	int count = 0;
 
 	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
 		return;
 
-	dev = kmalloc(sizeof *dev + device->phys_port_cnt * sizeof *port,
+	dev = kzalloc(sizeof *dev + device->phys_port_cnt * sizeof *port,
Quoted 8 lines

drivers/infiniband/core/sa_query.c static void ib_sa_add_one(struct ib_device *device) · line 1013

Quoted 6 lines
drivers/infiniband/core/sa_query.c +1 -1
@@ -1007,7 +1010,7 @@ static void ib_sa_add_one(struct ib_device *device)
 		e = device->phys_port_cnt;
 	}
 
-	sa_dev = kmalloc(sizeof *sa_dev +
+	sa_dev = kzalloc(sizeof *sa_dev +

Do you happen to remember why you needed these kmalloc -> kzalloc conversions?

I can't remember why. I do have this habbit of prefering kzalloc over kmalloc because it saves troubles sometimes.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): 20100506142849.GC29792-8YAHvHwT2UEvbXDkjdHOrw/[email protected]

#5 lore raw
Quoted 10 lines · Eli Cohen
Quoted 6 lines · Roland Dreier

drivers/infiniband/core/sa_query.c static void ib_sa_add_one(struct ib_device *device) · line 1010

Quoted 3 lines
drivers/infiniband/core/sa_query.c +1 -1
@@ -1007,7 +1010,7 @@ static void ib_sa_add_one(struct ib_device *device)
-	sa_dev = kmalloc(sizeof *sa_dev +
+	sa_dev = kzalloc(sizeof *sa_dev +

Do you happen to remember why you needed these kmalloc -> kzalloc conversions?

I can't remember why. I do have this habbit of prefering kzalloc over kmalloc because it saves troubles sometimes.

Hi Eli, just a friendly comment, best if such cleanup is done in a separate patch, else later someone attempting to debug/bisect (who might be yourself btw) could spend a hell of time wondering why it was done here and in the framework of this patch...

Or.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

#6 lore raw
In reply to Eli Cohen

drivers/infiniband/core/sa_query.c static void ib_sa_add_one(struct ib_device *device) · line 1025

Quoted 7 lines
drivers/infiniband/core/sa_query.c +3 -0
@@ -1017,9 +1020,12 @@ static void ib_sa_add_one(struct ib_device *device)
 	sa_dev->end_port   = e;
 
 	for (i = 0; i <= e - s; ++i) {
+		spin_lock_init(&sa_dev->port[i].ah_lock);
+		if (rdma_port_link_layer(device, i + 1) != IB_LINK_LAYER_INFINIBAND)
+			continue;

Not sure I understand why you move the initialization of the spinlock up here? It seems we ignore everything that might have to do with spinlock if this is an IBoE port.

But the larger issue is what if someone calls one of the ib_sa_XXX_query functions on an IBoE port? Seems we just crash on uninitialized structures. I guess we're assuming that the kernel is smart enough not to do that?

Also I'm wondering why you did the "count" stuff to ignore IBoE-only devices in multicast.c but not sa_query.c?

- R.

Signature · 6 lines
Roland Dreier <[email protected]> || For corporate legal information go to:
http://www.cisco.com/web/about/doing_business/legal/cri/index.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): ada632svqph.fsf-BjVyx320WGW9gfZ95n9DRSW4+XlvGpQz@public.gmane.org

#8 lore raw
Quoted 11 lines · Roland Dreier

drivers/infiniband/core/sa_query.c static void ib_sa_add_one(struct ib_device *device) · line 1025

Quoted 7 lines
drivers/infiniband/core/sa_query.c +3 -0
@@ -1017,9 +1020,12 @@ static void ib_sa_add_one(struct ib_device *device)
 	sa_dev->end_port   = e;
 
 	for (i = 0; i <= e - s; ++i) {
+		spin_lock_init(&sa_dev->port[i].ah_lock);
+		if (rdma_port_link_layer(device, i + 1) != IB_LINK_LAYER_INFINIBAND)
+			continue;

Not sure I understand why you move the initialization of the spinlock up here? It seems we ignore everything that might have to do with spinlock if this is an IBoE port.

We need the spinlock initialized for get_src_path_mask() which is called by ib_init_ah_from_path() which in turn is called for IBoE ports as well.

Quoted 5 lines

But the larger issue is what if someone calls one of the ib_sa_XXX_query functions on an IBoE port? Seems we just crash on uninitialized structures. I guess we're assuming that the kernel is smart enough not to do that?

Yes, we're not calling the SA for IBoE.

Quoted 4 lines

Also I'm wondering why you did the "count" stuff to ignore IBoE-only devices in multicast.c but not sa_query.c?

It seems to me the right place to put this logic as the mutlicast code registers as an IB client and the add_one implemntation is multicast.c.

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): 20100513065917.GA16073-8YAHvHwT2UEvbXDkjdHOrw/[email protected]

#10 lore raw
Quoted 6 lines
Quoted 2 lines

Also I'm wondering why you did the "count" stuff to ignore IBoE-only devices in multicast.c but not sa_query.c?

It seems to me the right place to put this logic as the mutlicast code registers as an IB client and the add_one implemntation is multicast.c.

Right, I'm not saying it shouldn't be in multicast.c. I'm just wondering why you don't have the same thing for sa_query.c.

- R.

Signature · 6 lines
Roland Dreier <[email protected]> || For corporate legal information go to:
http://www.cisco.com/web/about/doing_business/legal/cri/index.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): adatyqbsszt.fsf-BjVyx320WGW9gfZ95n9DRSW4+XlvGpQz@public.gmane.org

#12 lore raw
Quoted 4 lines · Roland Dreier

Right, I'm not saying it shouldn't be in multicast.c. I'm just wondering why you don't have the same thing for sa_query.c.

One reason is that get_src_path_mask() may access ah_lock.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): 20100513165201.GA19438-8YAHvHwT2UEvbXDkjdHOrw/[email protected]

#14 lore raw
Quoted 1 line

One reason is that get_src_path_mask() may access ah_lock.

I don't see that:

static u8 get_src_path_mask(struct ib_device *device, u8 port_num)

{
	struct ib_sa_device *sa_dev;
	struct ib_sa_port   *port;
	unsigned long flags;
	u8 src_path_mask;

	sa_dev = ib_get_client_data(device, &sa_client);
	if (!sa_dev)
		return 0x7f;

so if we never add the sa_client structure it just returns.

- R.

Signature · 6 lines
Roland Dreier <[email protected]> || For corporate legal information go to:
http://www.cisco.com/web/about/doing_business/legal/cri/index.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): adahbmbso19.fsf-BjVyx320WGW9gfZ95n9DRSW4+XlvGpQz@public.gmane.org

#16 lore raw
Quoted 15 lines · Roland Dreier
Quoted 1 line

One reason is that get_src_path_mask() may access ah_lock.

I don't see that:

static u8 get_src_path_mask(struct ib_device *device, u8 port_num)

{
	struct ib_sa_device *sa_dev;
	struct ib_sa_port   *port;
	unsigned long flags;
	u8 src_path_mask;

	sa_dev = ib_get_client_data(device, &sa_client);
	if (!sa_dev)
		return 0x7f;
Quoted 2 lines

so if we never add the sa_client structure it just returns.

I see... so I have no good reason to not treat sa_query.c in the same manner I did for multicast.c.
By the way, how do you prefer to work out all these comments. Would you prefer a new patch set or would you prefer to push the fixed versions to your tree?
Also, there are a few fixes that I pushed to OFED that were pushed after I sent V8. How would you like me to communicate them?
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html