Skip to contentSkip to conversation

IPoIB: reinitialize path struct's completion for every query

PATCH New Roland Dreier 1 participant 8 messages Nov 30, 2005 InfiniBand

lore
Share & tools
lore
#0 lore raw

PATCH 1/8 IPoIB: reinitialize path struct's completion for every query

It's possible that IPoIB will issue multiple SA queries for the same path struct. Therefore the struct's completion needs to be initialized for each query rather than only once when the struct is allocated, or else we might not wait long enough for later queries to finish and free the path struct too soon.

---

 drivers/infiniband/ulp/ipoib/ipoib_main.c |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

applies-to: 9fd732ebc6b85090b64862c4ee3af7078ba1f822 65c7eddaba33995e013ef3c04718f6dc8fdf2335

drivers/infiniband/ulp/ipoib/ipoib_main.c +2 -1
index 2fa3075..cd58b3d 100644
@@ -428,7 +428,6 @@ static struct ipoib_path *path_rec_creat
 	skb_queue_head_init(&path->queue);
 
 	INIT_LIST_HEAD(&path->neigh_list);
-	init_completion(&path->done);
 
 	memcpy(path->pathrec.dgid.raw, gid->raw, sizeof (union ib_gid));
 	path->pathrec.sgid      = priv->local_gid;
@@ -446,6 +445,8 @@ static int path_rec_start(struct net_dev
 	ipoib_dbg(priv, "Start path record lookup for " IPOIB_GID_FMT "\n",
 		  IPOIB_GID_ARG(path->pathrec.dgid));
 
+	init_completion(&path->done);
+
 	path->query_id =
 		ib_sa_path_rec_get(priv->ca, priv->port,
 				   &path->pathrec,

---
0.99.9k

#1 lore raw

PATCH 2/8 IPoIB: always set path->query to NULL when query finishes

Always set path->query to NULL when the SA path record query completes, rather than only when we don't have an address handle.

---

 drivers/infiniband/ulp/ipoib/ipoib_main.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

applies-to: 5f96a0797d23e7cee4f2a6c4770bacadee31a261 5872a9fc28e6cd3a4e51479a50970d19a01573b3

drivers/infiniband/ulp/ipoib/ipoib_main.c +2 -2
index cd58b3d..826d7a7 100644
@@ -398,9 +398,9 @@ static void path_rec_completion(int stat
 			while ((skb = __skb_dequeue(&neigh->queue)))
 				__skb_queue_tail(&skqueue, skb);
 		}
-	} else
-		path->query = NULL;
+	}
 
+	path->query = NULL;
 	complete(&path->done);
 
 	spin_unlock_irqrestore(&priv->lock, flags);

---
0.99.9k

#2 lore raw

PATCH 3/8 IPoIB: reinitialize mcast structs' completions for every query

Make sure mcast->done is initialized to uncompleted value before we submit a new query, so that it's safe to wait on.

---

 drivers/infiniband/ulp/ipoib/ipoib_multicast.c |    6 ++++--
 1 files changed, 4 insertions(+), 2 deletions(-)

applies-to: 993e77f7c00b7bc296e96f0cec1c98ea28a0436a de922487890936470660e89f9095aee980637989

drivers/infiniband/ulp/ipoib/ipoib_multicast.c +4 -2
index c33ed87..10404e0 100644
@@ -135,8 +135,6 @@ static struct ipoib_mcast *ipoib_mcast_a
 	if (!mcast)
 		return NULL;
 
-	init_completion(&mcast->done);
-
 	mcast->dev = dev;
 	mcast->created = jiffies;
 	mcast->backoff = 1;
@@ -350,6 +348,8 @@ static int ipoib_mcast_sendonly_join(str
 	rec.port_gid = priv->local_gid;
 	rec.pkey     = cpu_to_be16(priv->pkey);
 
+	init_completion(&mcast->done);
+
 	ret = ib_sa_mcmember_rec_set(priv->ca, priv->port, &rec,
 				     IB_SA_MCMEMBER_REC_MGID		|
 				     IB_SA_MCMEMBER_REC_PORT_GID	|
@@ -469,6 +469,8 @@ static void ipoib_mcast_join(struct net_
 		rec.traffic_class = priv->broadcast->mcmember.traffic_class;
 	}
 
+	init_completion(&mcast->done);
+
 	ret = ib_sa_mcmember_rec_set(priv->ca, priv->port, &rec, comp_mask,
 				     mcast->backoff * 1000, GFP_ATOMIC,
 				     ipoib_mcast_join_complete,

---
0.99.9k

#3 lore raw

PATCH 4/8 IPoIB: don't zero members after we allocate with kzalloc

ipoib_mcast_alloc() uses kzalloc(), so there's no need to zero out members of the mcast struct after it's allocated.

---

 drivers/infiniband/ulp/ipoib/ipoib_multicast.c |    4 ----
 1 files changed, 0 insertions(+), 4 deletions(-)

applies-to: bbb88a18ee78fa43c0f887c138011a055a9c8045 2e86541ec878de9ec5771600a77f451a80bebfc4

drivers/infiniband/ulp/ipoib/ipoib_multicast.c +0 -4
index 10404e0..ef3ee03 100644
@@ -138,15 +138,11 @@ static struct ipoib_mcast *ipoib_mcast_a
 	mcast->dev = dev;
 	mcast->created = jiffies;
 	mcast->backoff = 1;
-	mcast->logcount = 0;
 
 	INIT_LIST_HEAD(&mcast->list);
 	INIT_LIST_HEAD(&mcast->neigh_list);
 	skb_queue_head_init(&mcast->pkt_queue);
 
-	mcast->ah    = NULL;
-	mcast->query = NULL;
-
 	return mcast;
 }
 

---
0.99.9k

#4 lore raw

PATCH 5/8 IPoIB: protect child list in ipoib_ib_dev_flush

race condition: ipoib_ib_dev_flush is accessing child list without locks.

---

 drivers/infiniband/ulp/ipoib/ipoib_ib.c |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)

applies-to: e58f418dd8e64675b1dbaa6db92d7c1e606d1506 4f71055a45a503273c039d80db8ba9b13cb17549

drivers/infiniband/ulp/ipoib/ipoib_ib.c +4 -0
index 54ef2fe..2388580 100644
@@ -608,9 +608,13 @@ void ipoib_ib_dev_flush(void *_dev)
 	if (test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags))
 		ipoib_ib_dev_up(dev);
 
+	down(&priv->vlan_mutex);
+
 	/* Flush any child interfaces too */
 	list_for_each_entry(cpriv, &priv->child_intfs, list)
 		ipoib_ib_dev_flush(&cpriv->dev);
+
+	up(&priv->vlan_mutex);
 }
 
 void ipoib_ib_dev_cleanup(struct net_device *dev)

---
0.99.9k

#5 lore raw

PATCH 6/8 IPoIB: fix error handling in ipoib_open

If ipoib_ib_dev_up() fails after ipoib_ib_dev_open() is called, then ipoib_ib_dev_stop() needs to be called to clean up.

---

 drivers/infiniband/ulp/ipoib/ipoib_main.c |    4 +++-
 1 files changed, 3 insertions(+), 1 deletions(-)

applies-to: bb4b6f10197addff1af91368f916904eb4404edf 267ee88ed34c76dc527eeb3d95f9f9558ac99973

drivers/infiniband/ulp/ipoib/ipoib_main.c +3 -1
index 826d7a7..475d98f 100644
@@ -94,8 +94,10 @@ int ipoib_open(struct net_device *dev)
 	if (ipoib_ib_dev_open(dev))
 		return -EINVAL;
 
-	if (ipoib_ib_dev_up(dev))
+	if (ipoib_ib_dev_up(dev)) {
+		ipoib_ib_dev_stop(dev);
 		return -EINVAL;
+	}
 
 	if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
 		struct ipoib_dev_priv *cpriv;

---
0.99.9k

#6 lore raw

PATCH 7/8 IB/mthca: fix posting of send lists of length >= 255 on mem-free HCAs

On mem-free HCAs, when posting a long list of send requests, a doorbell must be rung every 255 requests. Add code to handle this.

---

 drivers/infiniband/hw/mthca/mthca_qp.c  |   31 +++++++++++++++++++++++++++++--
 drivers/infiniband/hw/mthca/mthca_wqe.h |    3 ++-
 2 files changed, 31 insertions(+), 3 deletions(-)

applies-to: 1f53cd0db55372192cc088788dadbed102845a17 e0ae9ecf469fdd3c1ad999efbf4fe6b782f49900

drivers/infiniband/hw/mthca/mthca_qp.c +29 -2
index f9c8eb9..7450550 100644
@@ -1822,6 +1822,7 @@ int mthca_arbel_post_send(struct ib_qp *
 {
 	struct mthca_dev *dev = to_mdev(ibqp->device);
 	struct mthca_qp *qp = to_mqp(ibqp);
+	__be32 doorbell[2];
 	void *wqe;
 	void *prev_wqe;
 	unsigned long flags;
@@ -1841,6 +1842,34 @@ int mthca_arbel_post_send(struct ib_qp *
 	ind = qp->sq.head & (qp->sq.max - 1);
 
 	for (nreq = 0; wr; ++nreq, wr = wr->next) {
+		if (unlikely(nreq == MTHCA_ARBEL_MAX_WQES_PER_SEND_DB)) {
+			nreq = 0;
+
+			doorbell[0] = cpu_to_be32((MTHCA_ARBEL_MAX_WQES_PER_SEND_DB << 24) |
+						  ((qp->sq.head & 0xffff) << 8) |
+						  f0 | op0);
+			doorbell[1] = cpu_to_be32((qp->qpn << 8) | size0);
+
+			qp->sq.head += MTHCA_ARBEL_MAX_WQES_PER_SEND_DB;
+			size0 = 0;
+
+			/*
+			 * Make sure that descriptors are written before
+			 * doorbell record.
+			 */
+			wmb();
+			*qp->sq.db = cpu_to_be32(qp->sq.head & 0xffff);
+
+			/*
+			 * Make sure doorbell record is written before we
+			 * write MMIO send doorbell.
+			 */
+			wmb();
+			mthca_write64(doorbell,
+				      dev->kar + MTHCA_SEND_DOORBELL,
+				      MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
+		}
+
 		if (mthca_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
 			mthca_err(dev, "SQ %06x full (%u head, %u tail,"
 					" %d max, %d nreq)\n", qp->qpn,
@@ -2017,8 +2046,6 @@ int mthca_arbel_post_send(struct ib_qp *
 
 out:
 	if (likely(nreq)) {
-		__be32 doorbell[2];
-
 		doorbell[0] = cpu_to_be32((nreq << 24)                  |
 					  ((qp->sq.head & 0xffff) << 8) |
 					  f0 | op0);
drivers/infiniband/hw/mthca/mthca_wqe.h +2 -1
index 73f1c0b..e7d2c1e 100644
@@ -50,7 +50,8 @@ enum {
 
 enum {
 	MTHCA_INVAL_LKEY			= 0x100,
-	MTHCA_TAVOR_MAX_WQES_PER_RECV_DB	= 256
+	MTHCA_TAVOR_MAX_WQES_PER_RECV_DB	= 256,
+	MTHCA_ARBEL_MAX_WQES_PER_SEND_DB	= 255
 };
 
 struct mthca_next_seg {

---
0.99.9k

#7 lore raw

PATCH 8/8 IB/uverbs: track multicast group membership for userspace QPs

uverbs needs to track which multicast groups is each qp attached to, in order to properly detach when cleanup is performed on device file close.

---

 drivers/infiniband/core/uverbs.h      |   11 ++++
 drivers/infiniband/core/uverbs_cmd.c  |   90 ++++++++++++++++++++++++++-------
 drivers/infiniband/core/uverbs_main.c |   21 ++++++--
 3 files changed, 99 insertions(+), 23 deletions(-)

applies-to: 9896d3c1093ded78c62da6b9a52b71e282c763e0 f4e401562c11c7ca65592ebd749353cf0b19af7b

drivers/infiniband/core/uverbs.h +11 -0
index ecb8301..7114e3f 100644
@@ -105,12 +105,23 @@ struct ib_uverbs_event {
 	u32				       *counter;
 };
 
+struct ib_uverbs_mcast_entry {
+	struct list_head	list;
+	union ib_gid 		gid;
+	u16 			lid;
+};
+
 struct ib_uevent_object {
 	struct ib_uobject	uobject;
 	struct list_head	event_list;
 	u32			events_reported;
 };
 
+struct ib_uqp_object {
+	struct ib_uevent_object	uevent;
+	struct list_head 	mcast_list;
+};
+
 struct ib_ucq_object {
 	struct ib_uobject	uobject;
 	struct ib_uverbs_file  *uverbs_file;
drivers/infiniband/core/uverbs_cmd.c +71 -19

180 lines not on this page · raw message

drivers/infiniband/core/uverbs_main.c +17 -4
index de6581d..81737bd 100644
@@ -160,6 +160,18 @@ void ib_uverbs_release_uevent(struct ib_
 	spin_unlock_irq(&file->async_file->lock);
 }
 
+static void ib_uverbs_detach_umcast(struct ib_qp *qp,
+				    struct ib_uqp_object *uobj)
+{
+	struct ib_uverbs_mcast_entry *mcast, *tmp;
+
+	list_for_each_entry_safe(mcast, tmp, &uobj->mcast_list, list) {
+		ib_detach_mcast(qp, &mcast->gid, mcast->lid);
+		list_del(&mcast->list);
+		kfree(mcast);
+	}
+}
+
 static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file,
 				      struct ib_ucontext *context)
 {
@@ -180,13 +192,14 @@ static int ib_uverbs_cleanup_ucontext(st
 
 	list_for_each_entry_safe(uobj, tmp, &context->qp_list, list) {
 		struct ib_qp *qp = idr_find(&ib_uverbs_qp_idr, uobj->id);
-		struct ib_uevent_object *uevent =
-			container_of(uobj, struct ib_uevent_object, uobject);
+		struct ib_uqp_object *uqp =
+			container_of(uobj, struct ib_uqp_object, uevent.uobject);
 		idr_remove(&ib_uverbs_qp_idr, uobj->id);
+		ib_uverbs_detach_umcast(qp, uqp);
 		ib_destroy_qp(qp);
 		list_del(&uobj->list);
-		ib_uverbs_release_uevent(file, uevent);
-		kfree(uevent);
+		ib_uverbs_release_uevent(file, &uqp->uevent);
+		kfree(uqp);
 	}
 
 	list_for_each_entry_safe(uobj, tmp, &context->cq_list, list) {

---
0.99.9k