Skip to contentSkip to conversation

ipath debug header

RFC PATCH NAK Roland Dreier 15 participants 63 messages Dec 16, 2005 – Dec 26, 2005 1 message not in the archive InfiniBand Page 2 of 2

lore
Share & tools
lore
#12 lore raw

RFC PATCH 11/13 ipath verbs, part 2

In reply to Roland Dreier

Second half of ipath verbs

---

 drivers/infiniband/hw/ipath/ipath_verbs.c | 2931 +++++++++++++++++++++++++++++
 1 files changed, 2931 insertions(+), 0 deletions(-)

3f617d81354835f183e089849cca09e295b2df0a

drivers/infiniband/hw/ipath/ipath_verbs.c +2931 -0
index 808326e..25d738d 100644
@@ -3242,3 +3242,2934 @@ static int get_rwqe(struct ipath_qp *qp,
 		spin_unlock(&rq->lock);
 	return 1;
 }
+
+/*
+ * This is called from ipath_qp_rcv() to process an incomming UC packet
+ * for the given QP.
+ * Called at interrupt level.
+ */
+static void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
+			 int has_grh, void *data, u32 tlen, struct ipath_qp *qp)
+{
+	struct ipath_other_headers *ohdr;
+	int opcode;
+	u32 hdrsize;
+	u32 psn;
+	u32 pad;
+	unsigned long flags;
+	struct ib_wc wc;
+	u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);
+	struct ib_reth *reth;
+
+	/* Check for GRH */
+	if (!has_grh) {
+		ohdr = &hdr->u.oth;
+		hdrsize = 8 + 12;	/* LRH + BTH */
+		psn = be32_to_cpu(ohdr->bth[2]);
+	} else {
+		ohdr = &hdr->u.l.oth;
+		hdrsize = 8 + 40 + 12;	/* LRH + GRH + BTH */
+		/*
+		 * The header with GRH is 60 bytes and the
+		 * core driver sets the eager header buffer
+		 * size to 56 bytes so the last 4 bytes of
+		 * the BTH header (PSN) is in the data buffer.
+		 */
+		psn = be32_to_cpu(((u32 *) data)[0]);
+		data += sizeof(u32);
+	}
+	/*
+	 * The opcode is in the low byte when its in network order
+	 * (top byte when in host order).
+	 */
+	opcode = *(u8 *) (&ohdr->bth[0]);
+
+	wc.imm_data = 0;
+	wc.wc_flags = 0;
+
+	spin_lock_irqsave(&qp->r_rq.lock, flags);
+
+	/* Compare the PSN verses the expected PSN. */
+	if (unlikely(cmp24(psn, qp->r_psn) != 0)) {
+		/*
+		 * Handle a sequence error.
+		 * Silently drop any current message.
+		 */
+		qp->r_psn = psn;
+	      inv:
+		qp->r_state = IB_OPCODE_UC_SEND_LAST;
+		switch (opcode) {
+		case IB_OPCODE_UC_SEND_FIRST:
+		case IB_OPCODE_UC_SEND_ONLY:
+		case IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE:
+			goto send_first;
+
+		case IB_OPCODE_UC_RDMA_WRITE_FIRST:
+		case IB_OPCODE_UC_RDMA_WRITE_ONLY:
+		case IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE:
+			goto rdma_first;
+
+		default:
+			dev->n_pkt_drops++;
+			goto done;
+		}
+	}
+
+	/* Check for opcode sequence errors. */
+	switch (qp->r_state) {
+	case IB_OPCODE_UC_SEND_FIRST:
+	case IB_OPCODE_UC_SEND_MIDDLE:
+		if (opcode == IB_OPCODE_UC_SEND_MIDDLE ||
+		    opcode == IB_OPCODE_UC_SEND_LAST ||
+		    opcode == IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE)
+			break;
+		goto inv;
+
+	case IB_OPCODE_UC_RDMA_WRITE_FIRST:
+	case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:
+		if (opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE ||
+		    opcode == IB_OPCODE_UC_RDMA_WRITE_LAST ||
+		    opcode == IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE)
+			break;
+		goto inv;
+
+	default:
+		if (opcode == IB_OPCODE_UC_SEND_FIRST ||
+		    opcode == IB_OPCODE_UC_SEND_ONLY ||
+		    opcode == IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE ||
+		    opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST ||
+		    opcode == IB_OPCODE_UC_RDMA_WRITE_ONLY ||
+		    opcode == IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE)
+			break;
+		goto inv;
+	}
+
+	/* OK, process the packet. */
+	switch (opcode) {
+	case IB_OPCODE_UC_SEND_FIRST:
+	case IB_OPCODE_UC_SEND_ONLY:
+	case IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE:
+	      send_first:
+		if (qp->r_reuse_sge) {
+			qp->r_reuse_sge = 0;
+			qp->r_sge = qp->s_rdma_sge;
+		} else if (!get_rwqe(qp, 0)) {
+			dev->n_pkt_drops++;
+			goto done;
+		}
+		/* Save the WQE so we can reuse it in case of an error. */
+		qp->s_rdma_sge = qp->r_sge;
+		qp->r_rcv_len = 0;
+		if (opcode == IB_OPCODE_UC_SEND_ONLY)
+			goto send_last;
+		else if (opcode == IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE)
+			goto send_last_imm;
+		/* FALLTHROUGH */
+	case IB_OPCODE_UC_SEND_MIDDLE:
+		/* Check for invalid length PMTU or posted rwqe len. */
+		if (unlikely(tlen != (hdrsize + pmtu + 4))) {
+			qp->r_reuse_sge = 1;
+			dev->n_pkt_drops++;
+			goto done;
+		}
+		qp->r_rcv_len += pmtu;
+		if (unlikely(qp->r_rcv_len > qp->r_len)) {
+			qp->r_reuse_sge = 1;
+			dev->n_pkt_drops++;
+			goto done;
+		}
+		copy_sge(&qp->r_sge, data, pmtu);
+		break;
+
+	case IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE:
+	      send_last_imm:
+		if (has_grh) {
+			wc.imm_data = *(u32 *) data;
+			data += sizeof(u32);
+		} else {
+			/* Immediate data comes after BTH */
+			wc.imm_data = ohdr->u.imm_data;
+		}
+		hdrsize += 4;
+		wc.wc_flags = IB_WC_WITH_IMM;
+		/* FALLTHROUGH */
+	case IB_OPCODE_UC_SEND_LAST:
+	      send_last:
+		/* Get the number of bytes the message was padded by. */
+		pad = (ohdr->bth[0] >> 12) & 3;
+		/* Check for invalid length. */
+		/* XXX LAST len should be >= 1 */
+		if (unlikely(tlen < (hdrsize + pad + 4))) {
+			qp->r_reuse_sge = 1;
+			dev->n_pkt_drops++;
+			goto done;
+		}
+		/* Don't count the CRC. */
+		tlen -= (hdrsize + pad + 4);
+		wc.byte_len = tlen + qp->r_rcv_len;
+		if (unlikely(wc.byte_len > qp->r_len)) {
+			qp->r_reuse_sge = 1;
+			dev->n_pkt_drops++;
+			goto done;
+		}
+		/* XXX Need to free SGEs */
+	      last_imm:
+		copy_sge(&qp->r_sge, data, tlen);
+		wc.wr_id = qp->r_wr_id;
+		wc.status = IB_WC_SUCCESS;
+		wc.opcode = IB_WC_RECV;
+		wc.vendor_err = 0;
+		wc.qp_num = qp->ibqp.qp_num;
+		wc.src_qp = qp->remote_qpn;
+		wc.pkey_index = 0;
+		wc.slid = qp->remote_ah_attr.dlid;
+		wc.sl = qp->remote_ah_attr.sl;
+		wc.dlid_path_bits = 0;
+		wc.port_num = 0;
+		/* Signal completion event if the solicited bit is set. */
+		ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,
+			       ohdr->bth[0] & __constant_cpu_to_be32(1 << 23));
+		break;
+
+	case IB_OPCODE_UC_RDMA_WRITE_FIRST:
+	case IB_OPCODE_UC_RDMA_WRITE_ONLY:
+	case IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE: /* consume RWQE */
+	      rdma_first:
+		/* RETH comes after BTH */
+		if (!has_grh)
+			reth = &ohdr->u.rc.reth;
+		else {
+			reth = (struct ib_reth *)data;
+			data += sizeof(*reth);
+		}
+		hdrsize += sizeof(*reth);
+		qp->r_len = be32_to_cpu(reth->length);
+		qp->r_rcv_len = 0;
+		if (qp->r_len != 0) {
+			u32 rkey = be32_to_cpu(reth->rkey);
+			u64 vaddr = be64_to_cpu(reth->vaddr);
+
+			/* Check rkey */
+			if (unlikely(!ipath_rkey_ok(dev, &qp->r_sge, qp->r_len,
+						    vaddr, rkey,
+						    IB_ACCESS_REMOTE_WRITE))) {
+				dev->n_pkt_drops++;
+				goto done;
+			}
+		} else {
+			qp->r_sge.sg_list = NULL;
+			qp->r_sge.sge.mr = NULL;
+			qp->r_sge.sge.vaddr = NULL;
+			qp->r_sge.sge.length = 0;
+			qp->r_sge.sge.sge_length = 0;
+		}
+		if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) {
+			dev->n_pkt_drops++;
+			goto done;
+		}
+		if (opcode == IB_OPCODE_UC_RDMA_WRITE_ONLY)
+			goto rdma_last;
+		else if (opcode == IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE)
+			goto rdma_last_imm;
+		/* FALLTHROUGH */
+	case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:
+		/* Check for invalid length PMTU or posted rwqe len. */
+		if (unlikely(tlen != (hdrsize + pmtu + 4))) {
+			dev->n_pkt_drops++;
+			goto done;
+		}
+		qp->r_rcv_len += pmtu;
+		if (unlikely(qp->r_rcv_len > qp->r_len)) {
+			dev->n_pkt_drops++;
+			goto done;
+		}
+		copy_sge(&qp->r_sge, data, pmtu);
+		break;
+
+	case IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
+	      rdma_last_imm:
+		/* Get the number of bytes the message was padded by. */
+		pad = (ohdr->bth[0] >> 12) & 3;
+		/* Check for invalid length. */
+		/* XXX LAST len should be >= 1 */
+		if (unlikely(tlen < (hdrsize + pad + 4))) {
+			dev->n_pkt_drops++;
+			goto done;
+		}
+		/* Don't count the CRC. */
+		tlen -= (hdrsize + pad + 4);
+		if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) {
+			dev->n_pkt_drops++;
+			goto done;
+		}
+		if (qp->r_reuse_sge) {
+			qp->r_reuse_sge = 0;
+		} else if (!get_rwqe(qp, 1)) {
+			dev->n_pkt_drops++;
+			goto done;
+		}
+		if (has_grh) {
+			wc.imm_data = *(u32 *) data;
+			data += sizeof(u32);
+		} else {
+			/* Immediate data comes after BTH */
+			wc.imm_data = ohdr->u.imm_data;
+		}
+		hdrsize += 4;
+		wc.wc_flags = IB_WC_WITH_IMM;
+		wc.byte_len = 0;
+		goto last_imm;
+
+	case IB_OPCODE_UC_RDMA_WRITE_LAST:
+	      rdma_last:
+		/* Get the number of bytes the message was padded by. */
+		pad = (ohdr->bth[0] >> 12) & 3;
+		/* Check for invalid length. */
+		/* XXX LAST len should be >= 1 */
+		if (unlikely(tlen < (hdrsize + pad + 4))) {
+			dev->n_pkt_drops++;
+			goto done;
+		}
+		/* Don't count the CRC. */
+		tlen -= (hdrsize + pad + 4);
+		if (unlikely(tlen + qp->r_rcv_len != qp->r_len)) {
+			dev->n_pkt_drops++;
+			goto done;
+		}
+		copy_sge(&qp->r_sge, data, tlen);
+		break;
+
+	default:
+		/* Drop packet for unknown opcodes. */
+		spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+		dev->n_pkt_drops++;
+		return;
+	}
+	qp->r_psn++;
+	qp->r_state = opcode;
+done:
+	spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+}
+
+/*
+ * Put this QP on the RNR timeout list for the device.
+ * XXX Use a simple list for now.  We might need a priority
+ * queue if we have lots of QPs waiting for RNR timeouts
+ * but that should be rare.
+ */
+static void insert_rnr_queue(struct ipath_qp *qp)
+{
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->pending_lock, flags);
+	if (list_empty(&dev->rnrwait))
+		list_add(&qp->timerwait, &dev->rnrwait);
+	else {
+		struct list_head *l = &dev->rnrwait;
+		struct ipath_qp *nqp = list_entry(l->next, struct ipath_qp,
+						  timerwait);
+
+		while (qp->s_rnr_timeout >= nqp->s_rnr_timeout) {
+			qp->s_rnr_timeout -= nqp->s_rnr_timeout;
+			l = l->next;
+			if (l->next == &dev->rnrwait)
+				break;
+			nqp = list_entry(l->next, struct ipath_qp, timerwait);
+		}
+		list_add(&qp->timerwait, l);
+	}
+	spin_unlock_irqrestore(&dev->pending_lock, flags);
+}
+
+/*
+ * This is called from do_uc_send() or do_rc_send() to forward a WQE addressed
+ * to the same HCA.
+ * Note that although we are single threaded due to the tasklet, we still
+ * have to protect against post_send().  We don't have to worry about
+ * receive interrupts since this is a connected protocol and all packets
+ * will pass through here.
+ */
+static void ipath_ruc_loopback(struct ipath_qp *sqp, struct ib_wc *wc)
+{
+	struct ipath_ibdev *dev = to_idev(sqp->ibqp.device);
+	struct ipath_qp *qp;
+	struct ipath_swqe *wqe;
+	struct ipath_sge *sge;
+	unsigned long flags;
+	u64 sdata;
+
+	qp = ipath_lookup_qpn(&dev->qp_table, sqp->remote_qpn);
+	if (!qp) {
+		dev->n_pkt_drops++;
+		return;
+	}
+
+again:
+	spin_lock_irqsave(&sqp->s_lock, flags);
+
+	if (!(state_ops[sqp->state] & IPATH_PROCESS_SEND_OK)) {
+		spin_unlock_irqrestore(&sqp->s_lock, flags);
+		goto done;
+	}
+
+	/* Get the next send request. */
+	if (sqp->s_last == sqp->s_head) {
+		/* Send work queue is empty. */
+		spin_unlock_irqrestore(&sqp->s_lock, flags);
+		goto done;
+	}
+
+	/*
+	 * We can rely on the entry not changing without the s_lock
+	 * being held until we update s_last.
+	 */
+	wqe = get_swqe_ptr(sqp, sqp->s_last);
+	spin_unlock_irqrestore(&sqp->s_lock, flags);
+
+	wc->wc_flags = 0;
+	wc->imm_data = 0;
+
+	sqp->s_sge.sge = wqe->sg_list[0];
+	sqp->s_sge.sg_list = wqe->sg_list + 1;
+	sqp->s_sge.num_sge = wqe->wr.num_sge;
+	sqp->s_len = wqe->length;
+	switch (wqe->wr.opcode) {
+	case IB_WR_SEND_WITH_IMM:
+		wc->wc_flags = IB_WC_WITH_IMM;
+		wc->imm_data = wqe->wr.imm_data;
+		/* FALLTHROUGH */
+	case IB_WR_SEND:
+		spin_lock_irqsave(&qp->r_rq.lock, flags);
+		if (!get_rwqe(qp, 0)) {
+		      rnr_nak:
+			spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+			/* Handle RNR NAK */
+			if (qp->ibqp.qp_type == IB_QPT_UC)
+				goto send_comp;
+			if (sqp->s_rnr_retry == 0) {
+				wc->status = IB_WC_RNR_RETRY_EXC_ERR;
+				goto err;
+			}
+			if (sqp->s_rnr_retry_cnt < 7)
+				sqp->s_rnr_retry--;
+			dev->n_rnr_naks++;
+			sqp->s_rnr_timeout = rnr_table[sqp->s_min_rnr_timer];
+			insert_rnr_queue(sqp);
+			goto done;
+		}
+		spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+		break;
+
+	case IB_WR_RDMA_WRITE_WITH_IMM:
+		wc->wc_flags = IB_WC_WITH_IMM;
+		wc->imm_data = wqe->wr.imm_data;
+		spin_lock_irqsave(&qp->r_rq.lock, flags);
+		if (!get_rwqe(qp, 1))
+			goto rnr_nak;
+		spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+		/* FALLTHROUGH */
+	case IB_WR_RDMA_WRITE:
+		if (wqe->length == 0)
+			break;
+		if (unlikely(!ipath_rkey_ok(dev, &qp->r_sge, wqe->length,
+					    wqe->wr.wr.rdma.remote_addr,
+					    wqe->wr.wr.rdma.rkey,
+					    IB_ACCESS_REMOTE_WRITE))) {
+		      acc_err:
+			wc->status = IB_WC_REM_ACCESS_ERR;
+		      err:
+			wc->wr_id = wqe->wr.wr_id;
+			wc->opcode = wc_opcode[wqe->wr.opcode];
+			wc->vendor_err = 0;
+			wc->byte_len = 0;
+			wc->qp_num = sqp->ibqp.qp_num;
+			wc->src_qp = sqp->remote_qpn;
+			wc->pkey_index = 0;
+			wc->slid = sqp->remote_ah_attr.dlid;
+			wc->sl = sqp->remote_ah_attr.sl;
+			wc->dlid_path_bits = 0;
+			wc->port_num = 0;
+			ipath_sqerror_qp(sqp, wc);
+			goto done;
+		}
+		break;
+
+	case IB_WR_RDMA_READ:
+		if (unlikely(!ipath_rkey_ok(dev, &sqp->s_sge, wqe->length,
+					    wqe->wr.wr.rdma.remote_addr,
+					    wqe->wr.wr.rdma.rkey,
+					    IB_ACCESS_REMOTE_READ))) {
+			goto acc_err;
+		}
+		if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
+			goto acc_err;
+		qp->r_sge.sge = wqe->sg_list[0];
+		qp->r_sge.sg_list = wqe->sg_list + 1;
+		qp->r_sge.num_sge = wqe->wr.num_sge;
+		break;
+
+	case IB_WR_ATOMIC_CMP_AND_SWP:
+	case IB_WR_ATOMIC_FETCH_AND_ADD:
+		if (unlikely(!ipath_rkey_ok(dev, &qp->r_sge, sizeof(u64),
+					    wqe->wr.wr.rdma.remote_addr,
+					    wqe->wr.wr.rdma.rkey,
+					    IB_ACCESS_REMOTE_ATOMIC))) {
+			goto acc_err;
+		}
+		/* Perform atomic OP and save result. */
+		sdata = wqe->wr.wr.atomic.swap;
+		spin_lock_irqsave(&dev->pending_lock, flags);
+		qp->r_atomic_data = *(u64 *) qp->r_sge.sge.vaddr;
+		if (wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
+			*(u64 *) qp->r_sge.sge.vaddr =
+			    qp->r_atomic_data + sdata;
+		} else if (qp->r_atomic_data == wqe->wr.wr.atomic.compare_add) {
+			*(u64 *) qp->r_sge.sge.vaddr = sdata;
+		}
+		spin_unlock_irqrestore(&dev->pending_lock, flags);
+		*(u64 *) sqp->s_sge.sge.vaddr = qp->r_atomic_data;
+		goto send_comp;
+
+	default:
+		goto done;
+	}
+
+	sge = &sqp->s_sge.sge;
+	while (sqp->s_len) {
+		u32 len = sqp->s_len;
+
+		if (len > sge->length)
+			len = sge->length;
+		BUG_ON(len == 0);
+		copy_sge(&qp->r_sge, sge->vaddr, len);
+		sge->vaddr += len;
+		sge->length -= len;
+		sge->sge_length -= len;
+		if (sge->sge_length == 0) {
+			if (--sqp->s_sge.num_sge)
+				*sge = *sqp->s_sge.sg_list++;
+		} else if (sge->length == 0 && sge->mr != NULL) {
+			if (++sge->n >= IPATH_SEGSZ) {
+				if (++sge->m >= sge->mr->mapsz)
+					break;
+				sge->n = 0;
+			}
+			sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
+			sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
+		}
+		sqp->s_len -= len;
+	}
+
+	if (wqe->wr.opcode == IB_WR_RDMA_WRITE ||
+	    wqe->wr.opcode == IB_WR_RDMA_READ)
+		goto send_comp;
+
+	if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM)
+		wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
+	else
+		wc->opcode = IB_WC_RECV;
+	wc->wr_id = qp->r_wr_id;
+	wc->status = IB_WC_SUCCESS;
+	wc->vendor_err = 0;
+	wc->byte_len = wqe->length;
+	wc->qp_num = qp->ibqp.qp_num;
+	wc->src_qp = qp->remote_qpn;
+	/* XXX do we know which pkey matched? Only needed for GSI. */
+	wc->pkey_index = 0;
+	wc->slid = qp->remote_ah_attr.dlid;
+	wc->sl = qp->remote_ah_attr.sl;
+	wc->dlid_path_bits = 0;
+	/* Signal completion event if the solicited bit is set. */
+	ipath_cq_enter(to_icq(qp->ibqp.recv_cq), wc,
+		       wqe->wr.send_flags & IB_SEND_SOLICITED);
+
+send_comp:
+	sqp->s_rnr_retry = sqp->s_rnr_retry_cnt;
+
+	if (!test_bit(IPATH_S_SIGNAL_REQ_WR, &sqp->s_flags) ||
+	    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
+		wc->wr_id = wqe->wr.wr_id;
+		wc->status = IB_WC_SUCCESS;
+		wc->opcode = wc_opcode[wqe->wr.opcode];
+		wc->vendor_err = 0;
+		wc->byte_len = wqe->length;
+		wc->qp_num = sqp->ibqp.qp_num;
+		wc->src_qp = 0;
+		wc->pkey_index = 0;
+		wc->slid = 0;
+		wc->sl = 0;
+		wc->dlid_path_bits = 0;
+		wc->port_num = 0;
+		ipath_cq_enter(to_icq(sqp->ibqp.send_cq), wc, 0);
+	}
+
+	/* Update s_last now that we are finished with the SWQE */
+	spin_lock_irqsave(&sqp->s_lock, flags);
+	if (++sqp->s_last >= sqp->s_size)
+		sqp->s_last = 0;
+	spin_unlock_irqrestore(&sqp->s_lock, flags);
+	goto again;
+
+done:
+	if (atomic_dec_and_test(&qp->refcount))
+		wake_up(&qp->wait);
+}
+
+/*
+ * Flush send work queue.
+ * The QP s_lock should be held.
+ */
+static void ipath_get_credit(struct ipath_qp *qp, u32 aeth)
+{
+	u32 credit = (aeth >> 24) & 0x1F;
+
+	/*
+	 * If credit == 0x1F, credit is invalid and we can send
+	 * as many packets as we like.  Otherwise, we have to
+	 * honor the credit field.
+	 */
+	if (credit == 0x1F) {
+		qp->s_lsn = (u32) -1;
+	} else if (qp->s_lsn != (u32) -1) {
+		/* Compute new LSN (i.e., MSN + credit) */
+		credit = (aeth + credit_table[credit]) & 0xFFFFFF;
+		if (cmp24(credit, qp->s_lsn) > 0)
+			qp->s_lsn = credit;
+	}
+
+	/* Restart sending if it was blocked due to lack of credits. */
+	if (qp->s_cur != qp->s_head &&
+	    (qp->s_lsn == (u32) -1 ||
+	     cmp24(get_swqe_ptr(qp, qp->s_cur)->ssn, qp->s_lsn + 1) <= 0)) {
+		tasklet_schedule(&qp->s_task);
+	}
+}
+
+/*
+ * This is called from ipath_rc_rcv() to process an incomming RC ACK
+ * for the given QP.
+ * Called at interrupt level with the QP s_lock held.
+ * Returns 1 if OK, 0 if current operation should be aborted (NAK).
+ */
+static int do_rc_ack(struct ipath_qp *qp, u32 aeth, u32 psn, int opcode)
+{
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ib_wc wc;
+	struct ipath_swqe *wqe;
+
+	/*
+	 * Remove the QP from the timeout queue (or RNR timeout queue).
+	 * If ipath_ib_timer() has already removed it,
+	 * it's OK since we hold the QP s_lock and ipath_restart_rc()
+	 * just won't find anything to restart if we ACK everything.
+	 */
+	spin_lock(&dev->pending_lock);
+	if (qp->timerwait.next != LIST_POISON1)
+		list_del(&qp->timerwait);
+	spin_unlock(&dev->pending_lock);
+
+	/*
+	 * Note that NAKs implicitly ACK outstanding SEND and
+	 * RDMA write requests and implicitly NAK RDMA read and
+	 * atomic requests issued before the NAK'ed request.
+	 * The MSN won't include the NAK'ed request but will include
+	 * an ACK'ed request(s).
+	 */
+	wqe = get_swqe_ptr(qp, qp->s_last);
+
+	/* Nothing is pending to ACK/NAK. */
+	if (qp->s_last == qp->s_tail)
+		return 0;
+
+	/*
+	 * The MSN might be for a later WQE than the PSN indicates so
+	 * only complete WQEs that the PSN finishes.
+	 */
+	while (cmp24(psn, wqe->lpsn) >= 0) {
+		/* If we are ACKing a WQE, the MSN should be >= the SSN. */
+		if (cmp24(aeth, wqe->ssn) < 0)
+			break;
+		/*
+		 * If this request is a RDMA read or atomic, and the ACK is
+		 * for a later operation, this ACK NAKs the RDMA read or atomic.
+		 * In other words, only a RDMA_READ_LAST or ONLY can ACK
+		 * a RDMA read and likewise for atomic ops.
+		 * Note that the NAK case can only happen if relaxed ordering
+		 * is used and requests are sent after an RDMA read
+		 * or atomic is sent but before the response is received.
+		 */
+		if ((wqe->wr.opcode == IB_WR_RDMA_READ &&
+		     opcode != IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST) ||
+		    ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
+		      wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) &&
+		     (opcode != IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE ||
+		      cmp24(wqe->psn, psn) != 0))) {
+			/* The last valid PSN seen is the previous request's. */
+			qp->s_last_psn = wqe->psn - 1;
+			/* Retry this request. */
+			ipath_restart_rc(qp, wqe->psn, &wc);
+			/*
+			 * No need to process the ACK/NAK since we are
+			 * restarting an earlier request.
+			 */
+			return 0;
+		}
+		/* Post a send completion queue entry if requested. */
+		if (!test_bit(IPATH_S_SIGNAL_REQ_WR, &qp->s_flags) ||
+		    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
+			wc.wr_id = wqe->wr.wr_id;
+			wc.status = IB_WC_SUCCESS;
+			wc.opcode = wc_opcode[wqe->wr.opcode];
+			wc.vendor_err = 0;
+			wc.byte_len = wqe->length;
+			wc.qp_num = qp->ibqp.qp_num;
+			wc.src_qp = qp->remote_qpn;
+			wc.pkey_index = 0;
+			wc.slid = qp->remote_ah_attr.dlid;
+			wc.sl = qp->remote_ah_attr.sl;
+			wc.dlid_path_bits = 0;
+			wc.port_num = 0;
+			ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0);
+		}
+		qp->s_retry = qp->s_retry_cnt;
+		/*
+		 * If we are completing a request which is in the process
+		 * of being resent, we can stop resending it since we know
+		 * the responder has already seen it.
+		 */
+		if (qp->s_last == qp->s_cur) {
+			if (++qp->s_cur >= qp->s_size)
+				qp->s_cur = 0;
+			wqe = get_swqe_ptr(qp, qp->s_cur);
+			qp->s_state = IB_OPCODE_RC_SEND_LAST;
+			qp->s_psn = wqe->psn;
+		}
+		if (++qp->s_last >= qp->s_size)
+			qp->s_last = 0;
+		wqe = get_swqe_ptr(qp, qp->s_last);
+		if (qp->s_last == qp->s_tail)
+			break;
+	}
+
+	switch (aeth >> 29) {
+	case 0:		/* ACK */
+		dev->n_rc_acks++;
+		/* If this is a partial ACK, reset the retransmit timer. */
+		if (qp->s_last != qp->s_tail) {
+			spin_lock(&dev->pending_lock);
+			list_add_tail(&qp->timerwait,
+				      &dev->pending[dev->pending_index]);
+			spin_unlock(&dev->pending_lock);
+		}
+		ipath_get_credit(qp, aeth);
+		qp->s_rnr_retry = qp->s_rnr_retry_cnt;
+		qp->s_retry = qp->s_retry_cnt;
+		qp->s_last_psn = psn;
+		return 1;
+
+	case 1:		/* RNR NAK */
+		dev->n_rnr_naks++;
+		if (qp->s_rnr_retry == 0) {
+			if (qp->s_last == qp->s_tail)
+				return 0;
+
+			wc.status = IB_WC_RNR_RETRY_EXC_ERR;
+			goto class_b;
+		}
+		if (qp->s_rnr_retry_cnt < 7)
+			qp->s_rnr_retry--;
+		if (qp->s_last == qp->s_tail)
+			return 0;
+
+		/* The last valid PSN seen is the previous request's. */
+		qp->s_last_psn = wqe->psn - 1;
+
+		/* Restart this request after the RNR timeout. */
+		wqe = get_swqe_ptr(qp, qp->s_last);
+
+		dev->n_rc_resends += (int)qp->s_psn - (int)psn;
+
+		/*
+		 * If we are starting the request from the beginning, let the
+		 * normal send code handle initialization.
+		 */
+		qp->s_cur = qp->s_last;
+		if (cmp24(psn, wqe->psn) <= 0) {
+			qp->s_state = IB_OPCODE_RC_SEND_LAST;
+			qp->s_psn = wqe->psn;
+		} else {
+			u32 n;
+
+			n = qp->s_cur;
+			for (;;) {
+				if (++n == qp->s_size)
+					n = 0;
+				if (n == qp->s_tail) {
+					if (cmp24(psn, qp->s_next_psn) >= 0) {
+						qp->s_cur = n;
+						wqe = get_swqe_ptr(qp, n);
+					}
+					break;
+				}
+				wqe = get_swqe_ptr(qp, n);
+				if (cmp24(psn, wqe->psn) < 0)
+					break;
+				qp->s_cur = n;
+			}
+			qp->s_psn = psn;
+
+			/*
+			 * Set the state to restart in the middle of a request.
+			 * Don't change the s_sge, s_cur_sge, or s_cur_size.
+			 * See do_rc_send().
+			 */
+			switch (wqe->wr.opcode) {
+			case IB_WR_SEND:
+			case IB_WR_SEND_WITH_IMM:
+				qp->s_state =
+					IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST;
+				break;
+
+			case IB_WR_RDMA_WRITE:
+			case IB_WR_RDMA_WRITE_WITH_IMM:
+				qp->s_state =
+					IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST;
+				break;
+
+			case IB_WR_RDMA_READ:
+				qp->s_state =
+					IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE;
+				break;
+
+			default:
+				/*
+				 * This case shouldn't happen since its only
+				 * one PSN per req.
+				 */
+				qp->s_state = IB_OPCODE_RC_SEND_LAST;
+			}
+		}
+
+		qp->s_rnr_timeout = rnr_table[(aeth >> 24) & 0x1F];
+		insert_rnr_queue(qp);
+		return 0;
+
+	case 3:		/* NAK */
+		/* The last valid PSN seen is the previous request's. */
+		if (qp->s_last != qp->s_tail)
+			qp->s_last_psn = wqe->psn - 1;
+		switch ((aeth >> 24) & 0x1F) {
+		case 0:	/* PSN sequence error */
+			dev->n_seq_naks++;
+			/*
+			 * Back up to the responder's expected PSN.
+			 * XXX Note that we might get a NAK in the
+			 * middle of an RDMA READ response which
+			 * terminates the RDMA READ.
+			 */
+			if (qp->s_last == qp->s_tail)
+				break;
+
+			if (cmp24(psn, wqe->psn) < 0) {
+				break;
+			}
+			/* Retry the request. */
+			ipath_restart_rc(qp, psn, &wc);
+			break;
+
+		case 1:	/* Invalid Request */
+			wc.status = IB_WC_REM_INV_REQ_ERR;
+			dev->n_other_naks++;
+			goto class_b;
+
+		case 2:	/* Remote Access Error */
+			wc.status = IB_WC_REM_ACCESS_ERR;
+			dev->n_other_naks++;
+			goto class_b;
+
+		case 3:	/* Remote Operation Error */
+			wc.status = IB_WC_REM_OP_ERR;
+			dev->n_other_naks++;
+		      class_b:
+			wc.wr_id = wqe->wr.wr_id;
+			wc.opcode = wc_opcode[wqe->wr.opcode];
+			wc.vendor_err = 0;
+			wc.byte_len = 0;
+			wc.qp_num = qp->ibqp.qp_num;
+			wc.src_qp = qp->remote_qpn;
+			wc.pkey_index = 0;
+			wc.slid = qp->remote_ah_attr.dlid;
+			wc.sl = qp->remote_ah_attr.sl;
+			wc.dlid_path_bits = 0;
+			wc.port_num = 0;
+			ipath_sqerror_qp(qp, &wc);
+			break;
+
+		default:
+			/* Ignore other reserved NAK error codes */
+			goto reserved;
+		}
+		qp->s_rnr_retry = qp->s_rnr_retry_cnt;
+		return 0;
+
+	default:		/* 2: reserved */
+	      reserved:
+		/* Ignore reserved NAK codes. */
+		return 0;
+	}
+}
+
+/*
+ * This is called from ipath_qp_rcv() to process an incomming RC packet
+ * for the given QP.
+ * Called at interrupt level.
+ */
+static void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
+			 int has_grh, void *data, u32 tlen, struct ipath_qp *qp)
+{
+	struct ipath_other_headers *ohdr;
+	int opcode;
+	u32 hdrsize;
+	u32 psn;
+	u32 pad;
+	unsigned long flags;
+	struct ib_wc wc;
+	u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);
+	int diff;
+	struct ib_reth *reth;
+
+	/* Check for GRH */
+	if (!has_grh) {
+		ohdr = &hdr->u.oth;
+		hdrsize = 8 + 12;	/* LRH + BTH */
+		psn = be32_to_cpu(ohdr->bth[2]);
+	} else {
+		ohdr = &hdr->u.l.oth;
+		hdrsize = 8 + 40 + 12;	/* LRH + GRH + BTH */
+		/*
+		 * The header with GRH is 60 bytes and the
+		 * core driver sets the eager header buffer
+		 * size to 56 bytes so the last 4 bytes of
+		 * the BTH header (PSN) is in the data buffer.
+		 */
+		psn = be32_to_cpu(((u32 *) data)[0]);
+		data += sizeof(u32);
+	}
+	/*
+	 * The opcode is in the low byte when its in network order
+	 * (top byte when in host order).
+	 */
+	opcode = *(u8 *) (&ohdr->bth[0]);
+
+	/*
+	 * Process responses (ACKs) before anything else.
+	 * Note that the packet sequence number will be for something
+	 * in the send work queue rather than the expected receive
+	 * packet sequence number.  In other words, this QP is the
+	 * requester.
+	 */
+	if (opcode >= IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST &&
+	    opcode <= IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE) {
+
+		spin_lock_irqsave(&qp->s_lock, flags);
+
+		/* Ignore invalid responses. */
+		if (cmp24(psn, qp->s_next_psn) >= 0) {
+			goto ack_done;
+		}
+
+		/* Ignore duplicate responses. */
+		diff = cmp24(psn, qp->s_last_psn);
+		if (unlikely(diff <= 0)) {
+			/* Update credits for "ghost" ACKs */
+			if (diff == 0 && opcode == IB_OPCODE_RC_ACKNOWLEDGE) {
+				if (!has_grh) {
+					pad = be32_to_cpu(ohdr->u.aeth);
+				} else {
+					pad = be32_to_cpu(((u32 *) data)[0]);
+					data += sizeof(u32);
+				}
+				if ((pad >> 29) == 0) {
+					ipath_get_credit(qp, pad);
+				}
+			}
+			goto ack_done;
+		}
+
+		switch (opcode) {
+		case IB_OPCODE_RC_ACKNOWLEDGE:
+		case IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE:
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST:
+			if (!has_grh) {
+				pad = be32_to_cpu(ohdr->u.aeth);
+			} else {
+				pad = be32_to_cpu(((u32 *) data)[0]);
+				data += sizeof(u32);
+			}
+			if (opcode == IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE) {
+				*(u64 *) qp->s_sge.sge.vaddr = *(u64 *) data;
+			}
+			if (!do_rc_ack(qp, pad, psn, opcode) ||
+			    opcode != IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST) {
+				goto ack_done;
+			}
+			hdrsize += 4;
+			/*
+			 * do_rc_ack() has already checked the PSN so skip
+			 * the sequence check.
+			 */
+			goto rdma_read;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:
+			/* no AETH, no ACK */
+			if (unlikely(cmp24(psn, qp->s_last_psn + 1) != 0)) {
+				dev->n_rdma_seq++;
+				ipath_restart_rc(qp, qp->s_last_psn + 1, &wc);
+				goto ack_done;
+			}
+		      rdma_read:
+			if (unlikely(qp->s_state !=
+				     IB_OPCODE_RC_RDMA_READ_REQUEST))
+				goto ack_done;
+			if (unlikely(tlen != (hdrsize + pmtu + 4)))
+				goto ack_done;
+			if (unlikely(pmtu >= qp->s_len))
+				goto ack_done;
+			/* We got a response so update the timeout. */
+			if (unlikely(qp->s_last == qp->s_tail ||
+				     get_swqe_ptr(qp, qp->s_last)->wr.opcode !=
+				     IB_WR_RDMA_READ))
+				goto ack_done;
+			spin_lock(&dev->pending_lock);
+			if (qp->s_rnr_timeout == 0 &&
+			    qp->timerwait.next != LIST_POISON1) {
+				list_move_tail(&qp->timerwait,
+					       &dev->pending[dev->
+							     pending_index]);
+			}
+			spin_unlock(&dev->pending_lock);
+			/*
+			 * Update the RDMA receive state but do the copy w/o
+			 * holding the locks and blocking interrupts.
+			 * XXX Yet another place that affects relaxed
+			 * RDMA order since we don't want s_sge modified.
+			 */
+			qp->s_len -= pmtu;
+			qp->s_last_psn = psn;
+			spin_unlock_irqrestore(&qp->s_lock, flags);
+			copy_sge(&qp->s_sge, data, pmtu);
+			return;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST:
+			/* ACKs READ req. */
+			if (unlikely(cmp24(psn, qp->s_last_psn + 1) != 0)) {
+				dev->n_rdma_seq++;
+				ipath_restart_rc(qp, qp->s_last_psn + 1, &wc);
+				goto ack_done;
+			}
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY:
+			if (unlikely(qp->s_state !=
+				     IB_OPCODE_RC_RDMA_READ_REQUEST)) {
+				goto ack_done;
+			}
+			/* Get the number of bytes the message was padded by. */
+			pad = (ohdr->bth[0] >> 12) & 3;
+			/*
+			 * Check that the data size is >= 1 && <= pmtu.
+			 * Remember to account for the AETH header (4)
+			 * and ICRC (4).
+			 */
+			if (unlikely(tlen <= (hdrsize + pad + 8))) {
+				/* XXX Need to generate an error CQ entry. */
+				goto ack_done;
+			}
+			tlen -= hdrsize + pad + 8;
+			if (unlikely(tlen != qp->s_len)) {
+				/* XXX Need to generate an error CQ entry. */
+				goto ack_done;
+			}
+			if (!has_grh) {
+				pad = be32_to_cpu(ohdr->u.aeth);
+			} else {
+				pad = be32_to_cpu(((u32 *) data)[0]);
+				data += sizeof(u32);
+			}
+			copy_sge(&qp->s_sge, data, tlen);
+			if (do_rc_ack(qp, pad, psn,
+				      IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST)) {
+				/*
+				 * Change the state so we contimue
+				 * processing new requests.
+				 */
+				qp->s_state = IB_OPCODE_RC_SEND_LAST;
+			}
+			goto ack_done;
+		}
+	      ack_done:
+		spin_unlock_irqrestore(&qp->s_lock, flags);
+		return;
+	}
+
+	spin_lock_irqsave(&qp->r_rq.lock, flags);
+
+	/* Compute 24 bits worth of difference. */
+	diff = cmp24(psn, qp->r_psn);
+	if (unlikely(diff)) {
+		if (diff > 0) {
+			/*
+			 * Packet sequence error.
+			 * A NAK will ACK earlier sends and RDMA writes.
+			 * Don't queue the NAK if a RDMA read, atomic, or
+			 * NAK is pending though.
+			 */
+			spin_lock(&qp->s_lock);
+			if ((qp->s_ack_state >=
+			     IB_OPCODE_RC_RDMA_READ_REQUEST &&
+			     qp->s_ack_state != IB_OPCODE_ACKNOWLEDGE) ||
+			    qp->s_nak_state != 0) {
+				spin_unlock(&qp->s_lock);
+				goto done;
+			}
+			qp->s_ack_state = IB_OPCODE_RC_SEND_ONLY;
+			qp->s_nak_state = IB_NAK_PSN_ERROR;
+			/* Use the expected PSN. */
+			qp->s_ack_psn = qp->r_psn;
+			goto resched;
+		}
+
+		/*
+		 * Handle a duplicate request.
+		 * Don't re-execute SEND, RDMA write or atomic op.
+		 * Don't NAK errors, just silently drop the duplicate request.
+		 * Note that r_sge, r_len, and r_rcv_len may be
+		 * in use so don't modify them.
+		 *
+		 * We are supposed to ACK the earliest duplicate PSN
+		 * but we can coalesce an outstanding duplicate ACK.
+		 * We have to send the earliest so that RDMA reads
+		 * can be restarted at the requester's expected PSN.
+		 */
+		spin_lock(&qp->s_lock);
+		if (qp->s_ack_state != IB_OPCODE_ACKNOWLEDGE &&
+		    cmp24(psn, qp->s_ack_psn) >= 0) {
+			if (qp->s_ack_state < IB_OPCODE_RDMA_READ_REQUEST)
+				qp->s_ack_psn = psn;
+			spin_unlock(&qp->s_lock);
+			goto done;
+		}
+		switch (opcode) {
+		case IB_OPCODE_RC_RDMA_READ_REQUEST:
+			/*
+			 * We have to be careful to not change s_rdma_sge
+			 * while do_rc_send() is using it and not holding
+			 * the s_lock.
+			 */
+			if (qp->s_ack_state != IB_OPCODE_RC_ACKNOWLEDGE &&
+			    qp->s_ack_state >= IB_OPCODE_RDMA_READ_REQUEST) {
+				spin_unlock(&qp->s_lock);
+				dev->n_rdma_dup_busy++;
+				goto done;
+			}
+			/* RETH comes after BTH */
+			if (!has_grh)
+				reth = &ohdr->u.rc.reth;
+			else {
+				reth = (struct ib_reth *)data;
+				data += sizeof(*reth);
+			}
+			qp->s_rdma_len = be32_to_cpu(reth->length);
+			if (qp->s_rdma_len != 0) {
+				u32 rkey = be32_to_cpu(reth->rkey);
+				u64 vaddr = be64_to_cpu(reth->vaddr);
+
+				/*
+				 * Address range must be a subset of the
+				 * original request and start on pmtu
+				 * boundaries.
+				 */
+				if (unlikely(!ipath_rkey_ok(dev,
+							    &qp->s_rdma_sge,
+							    qp->s_rdma_len,
+							    vaddr, rkey,
+							    IB_ACCESS_REMOTE_READ)))
+				{
+					goto done;
+				}
+			} else {
+				qp->s_rdma_sge.sg_list = NULL;
+				qp->s_rdma_sge.num_sge = 0;
+				qp->s_rdma_sge.sge.mr = NULL;
+				qp->s_rdma_sge.sge.vaddr = NULL;
+				qp->s_rdma_sge.sge.length = 0;
+				qp->s_rdma_sge.sge.sge_length = 0;
+			}
+			break;
+
+		case IB_OPCODE_RC_COMPARE_SWAP:
+		case IB_OPCODE_RC_FETCH_ADD:
+			/*
+			 * Check for the PSN of the last atomic operations
+			 * performed and resend the result if found.
+			 */
+			if ((psn & 0xFFFFFF) != qp->r_atomic_psn) {
+				spin_unlock(&qp->s_lock);
+				goto done;
+			}
+			qp->s_ack_atomic = qp->r_atomic_data;
+			break;
+		}
+		qp->s_ack_state = opcode;
+		qp->s_nak_state = 0;
+		qp->s_ack_psn = psn;
+		goto resched;
+	}
+
+	/* Check for opcode sequence errors. */
+	switch (qp->r_state) {
+	case IB_OPCODE_RC_SEND_FIRST:
+	case IB_OPCODE_RC_SEND_MIDDLE:
+		if (opcode == IB_OPCODE_RC_SEND_MIDDLE ||
+		    opcode == IB_OPCODE_RC_SEND_LAST ||
+		    opcode == IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE)
+			break;
+	      nack_inv:
+		/*
+		 * A NAK will ACK earlier sends and RDMA writes.
+		 * Don't queue the NAK if a RDMA read, atomic, or
+		 * NAK is pending though.
+		 */
+		spin_lock(&qp->s_lock);
+		if (qp->s_ack_state >= IB_OPCODE_RC_RDMA_READ_REQUEST &&
+		    qp->s_ack_state != IB_OPCODE_ACKNOWLEDGE) {
+			spin_unlock(&qp->s_lock);
+			goto done;
+		}
+		/* XXX Flush WQEs */
+		qp->state = IB_QPS_ERR;
+		qp->s_ack_state = IB_OPCODE_RC_SEND_ONLY;
+		qp->s_nak_state = IB_NAK_INVALID_REQUEST;
+		qp->s_ack_psn = qp->r_psn;
+		goto resched;
+
+	case IB_OPCODE_RC_RDMA_WRITE_FIRST:
+	case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:
+		if (opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE ||
+		    opcode == IB_OPCODE_RC_RDMA_WRITE_LAST ||
+		    opcode == IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE)
+			break;
+		goto nack_inv;
+
+	case IB_OPCODE_RC_RDMA_READ_REQUEST:
+	case IB_OPCODE_RC_COMPARE_SWAP:
+	case IB_OPCODE_RC_FETCH_ADD:
+		/*
+		 * Drop all new requests until a response has been sent.
+		 * A new request then ACKs the RDMA response we sent.
+		 * Relaxed ordering would allow new requests to be
+		 * processed but we would need to keep a queue
+		 * of rwqe's for all that are in progress.
+		 * Note that we can't RNR NAK this request since the RDMA
+		 * READ or atomic response is already queued to be sent
+		 * (unless we implement a response send queue).
+		 */
+		goto done;
+
+	default:
+		if (opcode == IB_OPCODE_RC_SEND_MIDDLE ||
+		    opcode == IB_OPCODE_RC_SEND_LAST ||
+		    opcode == IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE ||
+		    opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE ||
+		    opcode == IB_OPCODE_RC_RDMA_WRITE_LAST ||
+		    opcode == IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE)
+			goto nack_inv;
+		break;
+	}
+
+	wc.imm_data = 0;
+	wc.wc_flags = 0;
+
+	/* OK, process the packet. */
+	switch (opcode) {
+	case IB_OPCODE_RC_SEND_FIRST:
+		if (!get_rwqe(qp, 0)) {
+		      rnr_nak:
+			/*
+			 * A RNR NAK will ACK earlier sends and RDMA writes.
+			 * Don't queue the NAK if a RDMA read or atomic
+			 * is pending though.
+			 */
+			spin_lock(&qp->s_lock);
+			if (qp->s_ack_state >= IB_OPCODE_RC_RDMA_READ_REQUEST &&
+			    qp->s_ack_state != IB_OPCODE_ACKNOWLEDGE) {
+				spin_unlock(&qp->s_lock);
+				goto done;
+			}
+			qp->s_ack_state = IB_OPCODE_RC_SEND_ONLY;
+			qp->s_nak_state = IB_RNR_NAK | qp->s_min_rnr_timer;
+			qp->s_ack_psn = qp->r_psn;
+			goto resched;
+		}
+		qp->r_rcv_len = 0;
+		/* FALLTHROUGH */
+	case IB_OPCODE_RC_SEND_MIDDLE:
+	case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:
+	      send_middle:
+		/* Check for invalid length PMTU or posted rwqe len. */
+		if (unlikely(tlen != (hdrsize + pmtu + 4))) {
+			goto nack_inv;
+		}
+		qp->r_rcv_len += pmtu;
+		if (unlikely(qp->r_rcv_len > qp->r_len)) {
+			goto nack_inv;
+		}
+		copy_sge(&qp->r_sge, data, pmtu);
+		break;
+
+	case IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
+		/* consume RWQE */
+		if (!get_rwqe(qp, 1))
+			goto rnr_nak;
+		goto send_last_imm;
+
+	case IB_OPCODE_RC_SEND_ONLY:
+	case IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE:
+		if (!get_rwqe(qp, 0))
+			goto rnr_nak;
+		qp->r_rcv_len = 0;
+		if (opcode == IB_OPCODE_RC_SEND_ONLY)
+			goto send_last;
+		/* FALLTHROUGH */
+	case IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE:
+	      send_last_imm:
+		if (has_grh) {
+			wc.imm_data = *(u32 *) data;
+			data += sizeof(u32);
+		} else {
+			/* Immediate data comes after BTH */
+			wc.imm_data = ohdr->u.imm_data;
+		}
+		hdrsize += 4;
+		wc.wc_flags = IB_WC_WITH_IMM;
+		/* FALLTHROUGH */
+	case IB_OPCODE_RC_SEND_LAST:
+	case IB_OPCODE_RC_RDMA_WRITE_LAST:
+	      send_last:
+		/* Get the number of bytes the message was padded by. */
+		pad = (ohdr->bth[0] >> 12) & 3;
+		/* Check for invalid length. */
+		/* XXX LAST len should be >= 1 */
+		if (unlikely(tlen < (hdrsize + pad + 4))) {
+			goto nack_inv;
+		}
+		/* Don't count the CRC. */
+		tlen -= (hdrsize + pad + 4);
+		wc.byte_len = tlen + qp->r_rcv_len;
+		if (unlikely(wc.byte_len > qp->r_len)) {
+			goto nack_inv;
+		}
+		/* XXX Need to free SGEs */
+		copy_sge(&qp->r_sge, data, tlen);
+		atomic_inc(&qp->msn);
+		if (opcode == IB_OPCODE_RC_RDMA_WRITE_LAST ||
+		    opcode == IB_OPCODE_RC_RDMA_WRITE_ONLY)
+			break;
+		wc.wr_id = qp->r_wr_id;
+		wc.status = IB_WC_SUCCESS;
+		wc.opcode = IB_WC_RECV;
+		wc.vendor_err = 0;
+		wc.qp_num = qp->ibqp.qp_num;
+		wc.src_qp = qp->remote_qpn;
+		wc.pkey_index = 0;
+		wc.slid = qp->remote_ah_attr.dlid;
+		wc.sl = qp->remote_ah_attr.sl;
+		wc.dlid_path_bits = 0;
+		wc.port_num = 0;
+		/* Signal completion event if the solicited bit is set. */
+		ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,
+			       ohdr->bth[0] & __constant_cpu_to_be32(1 << 23));
+		break;
+
+	case IB_OPCODE_RC_RDMA_WRITE_FIRST:
+	case IB_OPCODE_RC_RDMA_WRITE_ONLY:
+	case IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE:
+		/* consume RWQE */
+		/* RETH comes after BTH */
+		if (!has_grh)
+			reth = &ohdr->u.rc.reth;
+		else {
+			reth = (struct ib_reth *)data;
+			data += sizeof(*reth);
+		}
+		hdrsize += sizeof(*reth);
+		qp->r_len = be32_to_cpu(reth->length);
+		qp->r_rcv_len = 0;
+		if (qp->r_len != 0) {
+			u32 rkey = be32_to_cpu(reth->rkey);
+			u64 vaddr = be64_to_cpu(reth->vaddr);
+
+			/* Check rkey & NAK */
+			if (unlikely(!ipath_rkey_ok(dev, &qp->r_sge, qp->r_len,
+						    vaddr, rkey,
+						    IB_ACCESS_REMOTE_WRITE))) {
+			      nack_acc:
+				/*
+				 * A NAK will ACK earlier sends and RDMA
+				 * writes.
+				 * Don't queue the NAK if a RDMA read,
+				 * atomic, or NAK is pending though.
+				 */
+				spin_lock(&qp->s_lock);
+				if (qp->s_ack_state >=
+				    IB_OPCODE_RC_RDMA_READ_REQUEST &&
+				    qp->s_ack_state != IB_OPCODE_ACKNOWLEDGE) {
+					spin_unlock(&qp->s_lock);
+					goto done;
+				}
+				/* XXX Flush WQEs */
+				qp->state = IB_QPS_ERR;
+				qp->s_ack_state = IB_OPCODE_RC_RDMA_WRITE_ONLY;
+				qp->s_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
+				qp->s_ack_psn = qp->r_psn;
+				goto resched;
+			}
+		} else {
+			qp->r_sge.sg_list = NULL;
+			qp->r_sge.sge.mr = NULL;
+			qp->r_sge.sge.vaddr = NULL;
+			qp->r_sge.sge.length = 0;
+			qp->r_sge.sge.sge_length = 0;
+		}
+		if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
+			goto nack_acc;
+		if (opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST)
+			goto send_middle;
+		else if (opcode == IB_OPCODE_RC_RDMA_WRITE_ONLY)
+			goto send_last;
+		if (!get_rwqe(qp, 1))
+			goto rnr_nak;
+		goto send_last_imm;
+
+	case IB_OPCODE_RC_RDMA_READ_REQUEST:
+		/* RETH comes after BTH */
+		if (!has_grh)
+			reth = &ohdr->u.rc.reth;
+		else {
+			reth = (struct ib_reth *)data;
+			data += sizeof(*reth);
+		}
+		spin_lock(&qp->s_lock);
+		if (qp->s_ack_state != IB_OPCODE_RC_ACKNOWLEDGE &&
+		    qp->s_ack_state >= IB_OPCODE_RDMA_READ_REQUEST) {
+			spin_unlock(&qp->s_lock);
+			goto done;
+		}
+		qp->s_rdma_len = be32_to_cpu(reth->length);
+		if (qp->s_rdma_len != 0) {
+			u32 rkey = be32_to_cpu(reth->rkey);
+			u64 vaddr = be64_to_cpu(reth->vaddr);
+
+			/* Check rkey & NAK */
+			if (unlikely(!ipath_rkey_ok(dev, &qp->s_rdma_sge,
+						    qp->s_rdma_len,
+						    vaddr, rkey,
+						    IB_ACCESS_REMOTE_READ))) {
+				spin_unlock(&qp->s_lock);
+				goto nack_acc;
+			}
+			/*
+			 * Update the next expected PSN.
+			 * We add 1 later below, so only add the remainder here.
+			 */
+			if (qp->s_rdma_len > pmtu)
+				qp->r_psn += (qp->s_rdma_len - 1) / pmtu;
+		} else {
+			qp->s_rdma_sge.sg_list = NULL;
+			qp->s_rdma_sge.num_sge = 0;
+			qp->s_rdma_sge.sge.mr = NULL;
+			qp->s_rdma_sge.sge.vaddr = NULL;
+			qp->s_rdma_sge.sge.length = 0;
+			qp->s_rdma_sge.sge.sge_length = 0;
+		}
+		if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
+			goto nack_acc;
+		/*
+		 * We need to increment the MSN here instead of when we
+		 * finish sending the result since a duplicate request would
+		 * increment it more than once.
+		 */
+		atomic_inc(&qp->msn);
+		qp->s_ack_state = opcode;
+		qp->s_nak_state = 0;
+		qp->s_ack_psn = psn;
+		qp->r_psn++;
+		qp->r_state = opcode;
+		goto rdmadone;
+
+	case IB_OPCODE_RC_COMPARE_SWAP:
+	case IB_OPCODE_RC_FETCH_ADD:{
+			struct ib_atomic_eth *ateth;
+			u64 vaddr;
+			u64 sdata;
+			u32 rkey;
+
+			if (!has_grh)
+				ateth = &ohdr->u.atomic_eth;
+			else {
+				ateth = (struct ib_atomic_eth *)data;
+				data += sizeof(*ateth);
+			}
+			vaddr = be64_to_cpu(ateth->vaddr);
+			if (unlikely(vaddr & 0x7))
+				goto nack_inv;
+			rkey = be32_to_cpu(ateth->rkey);
+			/* Check rkey & NAK */
+			if (unlikely(!ipath_rkey_ok(dev, &qp->r_sge,
+						    sizeof(u64), vaddr, rkey,
+						    IB_ACCESS_REMOTE_ATOMIC))) {
+				goto nack_acc;
+			}
+			if (unlikely(!(qp->qp_access_flags &
+					IB_ACCESS_REMOTE_ATOMIC)))
+				goto nack_acc;
+			/* Perform atomic OP and save result. */
+			sdata = be64_to_cpu(ateth->swap_data);
+			spin_lock(&dev->pending_lock);
+			qp->r_atomic_data = *(u64 *) qp->r_sge.sge.vaddr;
+			if (opcode == IB_OPCODE_RC_FETCH_ADD) {
+				*(u64 *) qp->r_sge.sge.vaddr =
+				    qp->r_atomic_data + sdata;
+			} else if (qp->r_atomic_data ==
+				   be64_to_cpu(ateth->compare_data)) {
+				*(u64 *) qp->r_sge.sge.vaddr = sdata;
+			}
+			spin_unlock(&dev->pending_lock);
+			atomic_inc(&qp->msn);
+			qp->r_atomic_psn = psn & 0xFFFFFF;
+			psn |= 1 << 31;
+			break;
+		}
+
+	default:
+		/* Drop packet for unknown opcodes. */
+		spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+		return;
+	}
+	qp->r_psn++;
+	qp->r_state = opcode;
+	/* Send an ACK if requested or required. */
+	if (psn & (1 << 31)) {
+		/*
+		 * Coalesce ACKs unless there is a RDMA READ or
+		 * ATOMIC pending.
+		 */
+		spin_lock(&qp->s_lock);
+		if (qp->s_ack_state == IB_OPCODE_RC_ACKNOWLEDGE ||
+		    qp->s_ack_state < IB_OPCODE_RDMA_READ_REQUEST) {
+			qp->s_ack_state = opcode;
+			qp->s_nak_state = 0;
+			qp->s_ack_psn = psn;
+			qp->s_ack_atomic = qp->r_atomic_data;
+			goto resched;
+		}
+		spin_unlock(&qp->s_lock);
+	}
+done:
+	spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+	return;
+
+resched:
+	/* Try to send ACK right away but not if do_rc_send() is active. */
+	if (qp->s_hdrwords == 0 &&
+	    (qp->s_ack_state < IB_OPCODE_RDMA_READ_REQUEST ||
+	     qp->s_ack_state >= IB_OPCODE_COMPARE_SWAP))
+		send_rc_ack(qp);
+
+rdmadone:
+	spin_unlock(&qp->s_lock);
+	spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+
+	/* Call do_rc_send() in another thread. */
+	tasklet_schedule(&qp->s_task);
+}
+
+/*
+ * This is called from ipath_ib_rcv() to process an incomming packet
+ * for the given QP.
+ * Called at interrupt level.
+ */
+static inline void ipath_qp_rcv(struct ipath_ibdev *dev,
+				struct ipath_ib_header *hdr, int has_grh,
+				void *data, u32 tlen, struct ipath_qp *qp)
+{
+	/* Check for valid receive state. */
+	if (!(state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) {
+		dev->n_pkt_drops++;
+		return;
+	}
+
+	switch (qp->ibqp.qp_type) {
+	case IB_QPT_SMI:
+	case IB_QPT_GSI:
+	case IB_QPT_UD:
+		ipath_ud_rcv(dev, hdr, has_grh, data, tlen, qp);
+		break;
+
+	case IB_QPT_RC:
+		ipath_rc_rcv(dev, hdr, has_grh, data, tlen, qp);
+		break;
+
+	case IB_QPT_UC:
+		ipath_uc_rcv(dev, hdr, has_grh, data, tlen, qp);
+		break;
+
+	default:
+		break;
+	}
+}
+
+/*
+ * This is called from ipath_kreceive() to process an incomming packet at
+ * interrupt level. Tlen is the length of the header + data + CRC in bytes.
+ */
+static void ipath_ib_rcv(const ipath_type t, void *rhdr, void *data, u32 tlen)
+{
+	struct ipath_ibdev *dev = ipath_devices[t];
+	struct ipath_ib_header *hdr = rhdr;
+	struct ipath_other_headers *ohdr;
+	struct ipath_qp *qp;
+	u32 qp_num;
+	int lnh;
+	u8 opcode;
+
+	if (dev == NULL)
+		return;
+
+	if (tlen < 24) {	/* LRH+BTH+CRC */
+		dev->n_pkt_drops++;
+		return;
+	}
+
+	/* Check for GRH */
+	lnh = be16_to_cpu(hdr->lrh[0]) & 3;
+	if (lnh == IPS_LRH_BTH)
+		ohdr = &hdr->u.oth;
+	else if (lnh == IPS_LRH_GRH)
+		ohdr = &hdr->u.l.oth;
+	else {
+		dev->n_pkt_drops++;
+		return;
+	}
+
+	opcode = *(u8 *) (&ohdr->bth[0]);
+	dev->opstats[opcode].n_bytes += tlen;
+	dev->opstats[opcode].n_packets++;
+
+	/* Get the destination QP number. */
+	qp_num = be32_to_cpu(ohdr->bth[1]) & 0xFFFFFF;
+	if (qp_num == 0xFFFFFF) {
+		struct ipath_mcast *mcast;
+		struct ipath_mcast_qp *p;
+
+		mcast = ipath_mcast_find(&hdr->u.l.grh.dgid);
+		if (mcast == NULL) {
+			dev->n_pkt_drops++;
+			return;
+		}
+		dev->n_multicast_rcv++;
+		list_for_each_entry_rcu(p, &mcast->qp_list, list)
+			ipath_qp_rcv(dev, hdr, lnh == IPS_LRH_GRH, data, tlen,
+				     p->qp);
+		/*
+		 * Notify ipath_multicast_detach() if it is waiting for us
+		 * to finish.
+		 */
+		if (atomic_dec_return(&mcast->refcount) <= 1)
+			wake_up(&mcast->wait);
+	} else if ((qp = ipath_lookup_qpn(&dev->qp_table, qp_num)) != NULL) {
+		ipath_qp_rcv(dev, hdr, lnh == IPS_LRH_GRH, data, tlen, qp);
+		/*
+		 * Notify ipath_destroy_qp() if it is waiting for us to finish.
+		 */
+		if (atomic_dec_and_test(&qp->refcount))
+			wake_up(&qp->wait);
+	} else
+		dev->n_pkt_drops++;
+}
+
+/*
+ * This is called from ipath_do_rcv_timer() at interrupt level
+ * to check for QPs which need retransmits and to collect performance numbers.
+ */
+static void ipath_ib_timer(const ipath_type t)
+{
+	struct ipath_ibdev *dev = ipath_devices[t];
+	struct ipath_qp *resend = NULL;
+	struct ipath_qp *rnr = NULL;
+	struct list_head *last;
+	struct ipath_qp *qp;
+	unsigned long flags;
+
+	if (dev == NULL)
+		return;
+
+	spin_lock_irqsave(&dev->pending_lock, flags);
+	/* Start filling the next pending queue. */
+	if (++dev->pending_index >= ARRAY_SIZE(dev->pending))
+		dev->pending_index = 0;
+	/* Save any requests still in the new queue, they have timed out. */
+	last = &dev->pending[dev->pending_index];
+	while (!list_empty(last)) {
+		qp = list_entry(last->next, struct ipath_qp, timerwait);
+		if (last->next == LIST_POISON1 ||
+		    last->next != &qp->timerwait ||
+		    qp->timerwait.prev != last) {
+			INIT_LIST_HEAD(last);
+		} else {
+			list_del(&qp->timerwait);
+			qp->timerwait.prev = (struct list_head *) resend;
+			resend = qp;
+			atomic_inc(&qp->refcount);
+		}
+	}
+	last = &dev->rnrwait;
+	if (!list_empty(last)) {
+		qp = list_entry(last->next, struct ipath_qp, timerwait);
+		if (--qp->s_rnr_timeout == 0) {
+			do {
+				if (last->next == LIST_POISON1 ||
+				    last->next != &qp->timerwait ||
+				    qp->timerwait.prev != last) {
+					INIT_LIST_HEAD(last);
+					break;
+				}
+				list_del(&qp->timerwait);
+				qp->timerwait.prev = (struct list_head *) rnr;
+				rnr = qp;
+				if (list_empty(last))
+					break;
+				qp = list_entry(last->next, struct ipath_qp,
+						timerwait);
+			} while (qp->s_rnr_timeout == 0);
+		}
+	}
+	/* We should only be in the started state if pma_sample_start != 0 */
+	if (dev->pma_sample_status == IB_PMA_SAMPLE_STATUS_STARTED &&
+	    --dev->pma_sample_start == 0) {
+		dev->pma_sample_status = IB_PMA_SAMPLE_STATUS_RUNNING;
+		ipath_layer_snapshot_counters(dev->ib_unit, &dev->ipath_sword,
+					      &dev->ipath_rword,
+					      &dev->ipath_spkts,
+					      &dev->ipath_rpkts);
+	}
+	if (dev->pma_sample_status == IB_PMA_SAMPLE_STATUS_RUNNING) {
+		if (dev->pma_sample_interval == 0) {
+			u64 ta, tb, tc, td;
+
+			dev->pma_sample_status = IB_PMA_SAMPLE_STATUS_DONE;
+			ipath_layer_snapshot_counters(dev->ib_unit,
+						      &ta, &tb, &tc, &td);
+
+			dev->ipath_sword = ta - dev->ipath_sword;
+			dev->ipath_rword = tb - dev->ipath_rword;
+			dev->ipath_spkts = tc - dev->ipath_spkts;
+			dev->ipath_rpkts = td - dev->ipath_rpkts;
+		} else {
+			dev->pma_sample_interval--;
+		}
+	}
+	spin_unlock_irqrestore(&dev->pending_lock, flags);
+
+	/* XXX What if timer fires again while this is running? */
+	for (qp = resend; qp != NULL;
+	     qp = (struct ipath_qp *) qp->timerwait.prev) {
+		struct ib_wc wc;
+
+		spin_lock_irqsave(&qp->s_lock, flags);
+		if (qp->s_last != qp->s_tail && qp->state == IB_QPS_RTS) {
+			dev->n_timeouts++;
+			ipath_restart_rc(qp, qp->s_last_psn + 1, &wc);
+		}
+		spin_unlock_irqrestore(&qp->s_lock, flags);
+
+		/* Notify ipath_destroy_qp() if it is waiting. */
+		if (atomic_dec_and_test(&qp->refcount))
+			wake_up(&qp->wait);
+	}
+	for (qp = rnr; qp != NULL;
+	     qp = (struct ipath_qp *) qp->timerwait.prev) {
+		tasklet_schedule(&qp->s_task);
+	}
+}
+
+/*
+ * This is called from ipath_intr() at interrupt level when a PIO buffer
+ * is available after ipath_verbs_send() returned an error that no
+ * buffers were available.
+ * Return 0 if we consumed all the PIO buffers and we still have QPs
+ * waiting for buffers (for now, just do a tasklet_schedule and return one).
+ */
+static int ipath_ib_piobufavail(const ipath_type t)
+{
+	struct ipath_ibdev *dev = ipath_devices[t];
+	struct ipath_qp *qp;
+	unsigned long flags;
+
+	if (dev == NULL)
+		return 1;
+
+	spin_lock_irqsave(&dev->pending_lock, flags);
+	while (!list_empty(&dev->piowait)) {
+		qp = list_entry(dev->piowait.next, struct ipath_qp, piowait);
+		list_del(&qp->piowait);
+		tasklet_schedule(&qp->s_task);
+	}
+	spin_unlock_irqrestore(&dev->pending_lock, flags);
+
+	return 1;
+}
+
+static struct ib_qp *ipath_create_qp(struct ib_pd *ibpd,
+				     struct ib_qp_init_attr *init_attr,
+				     struct ib_udata *udata)
+{
+	struct ipath_qp *qp;
+	int err;
+	struct ipath_swqe *swq = NULL;
+	struct ipath_ibdev *dev;
+	size_t sz;
+
+	if (init_attr->cap.max_send_sge > 255 ||
+	    init_attr->cap.max_recv_sge > 255)
+		return ERR_PTR(-ENOMEM);
+
+	switch (init_attr->qp_type) {
+	case IB_QPT_UC:
+	case IB_QPT_RC:
+		sz = sizeof(struct ipath_sge) * init_attr->cap.max_send_sge +
+		    sizeof(struct ipath_swqe);
+		swq = vmalloc((init_attr->cap.max_send_wr + 1) * sz);
+		if (swq == NULL)
+			return ERR_PTR(-ENOMEM);
+		/* FALLTHROUGH */
+	case IB_QPT_UD:
+	case IB_QPT_SMI:
+	case IB_QPT_GSI:
+		qp = kmalloc(sizeof(*qp), GFP_KERNEL);
+		if (!qp)
+			return ERR_PTR(-ENOMEM);
+		qp->r_rq.size = init_attr->cap.max_recv_wr + 1;
+		sz = sizeof(struct ipath_sge) * init_attr->cap.max_recv_sge +
+		    sizeof(struct ipath_rwqe);
+		qp->r_rq.wq = vmalloc(qp->r_rq.size * sz);
+		if (!qp->r_rq.wq) {
+			kfree(qp);
+			return ERR_PTR(-ENOMEM);
+		}
+
+		/*
+		 * ib_create_qp() will initialize qp->ibqp
+		 * except for qp->ibqp.qp_num.
+		 */
+		spin_lock_init(&qp->s_lock);
+		spin_lock_init(&qp->r_rq.lock);
+		atomic_set(&qp->refcount, 0);
+		init_waitqueue_head(&qp->wait);
+		tasklet_init(&qp->s_task,
+			     init_attr->qp_type == IB_QPT_RC ? do_rc_send :
+			     do_uc_send, (unsigned long)qp);
+		qp->piowait.next = LIST_POISON1;
+		qp->piowait.prev = LIST_POISON2;
+		qp->timerwait.next = LIST_POISON1;
+		qp->timerwait.prev = LIST_POISON2;
+		qp->state = IB_QPS_RESET;
+		qp->s_wq = swq;
+		qp->s_size = init_attr->cap.max_send_wr + 1;
+		qp->s_max_sge = init_attr->cap.max_send_sge;
+		qp->r_rq.max_sge = init_attr->cap.max_recv_sge;
+		qp->s_flags = init_attr->sq_sig_type == IB_SIGNAL_REQ_WR ?
+			1 << IPATH_S_SIGNAL_REQ_WR : 0;
+		dev = to_idev(ibpd->device);
+		err = ipath_alloc_qpn(&dev->qp_table, qp, init_attr->qp_type);
+		if (err) {
+			vfree(swq);
+			vfree(qp->r_rq.wq);
+			kfree(qp);
+			return ERR_PTR(err);
+		}
+		ipath_reset_qp(qp);
+
+		/* Tell the core driver that the kernel SMA is present. */
+		if (qp->ibqp.qp_type == IB_QPT_SMI)
+			ipath_verbs_set_flags(dev->ib_unit,
+					      IPATH_VERBS_KERNEL_SMA);
+		break;
+
+	default:
+		/* Don't support raw QPs */
+		return ERR_PTR(-ENOSYS);
+	}
+
+	init_attr->cap.max_inline_data = 0;
+
+	return &qp->ibqp;
+}
+
+/*
+ * Note that this can be called while the QP is actively sending or receiving!
+ */
+static int ipath_destroy_qp(struct ib_qp *ibqp)
+{
+	struct ipath_qp *qp = to_iqp(ibqp);
+	struct ipath_ibdev *dev = to_idev(ibqp->device);
+	unsigned long flags;
+
+	/* Tell the core driver that the kernel SMA is gone. */
+	if (qp->ibqp.qp_type == IB_QPT_SMI)
+		ipath_verbs_set_flags(dev->ib_unit, 0);
+
+	spin_lock_irqsave(&qp->r_rq.lock, flags);
+	spin_lock(&qp->s_lock);
+	qp->state = IB_QPS_ERR;
+	spin_unlock(&qp->s_lock);
+	spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+
+	/* Stop the sending tasklet. */
+	tasklet_kill(&qp->s_task);
+
+	/* Make sure the QP isn't on the timeout list. */
+	spin_lock_irqsave(&dev->pending_lock, flags);
+	if (qp->timerwait.next != LIST_POISON1)
+		list_del(&qp->timerwait);
+	if (qp->piowait.next != LIST_POISON1)
+		list_del(&qp->piowait);
+	spin_unlock_irqrestore(&dev->pending_lock, flags);
+
+	/*
+	 * Make sure that the QP is not in the QPN table so receive interrupts
+	 * will discard packets for this QP.
+	 * XXX Also remove QP from multicast table.
+	 */
+	if (atomic_read(&qp->refcount) != 0)
+		ipath_free_qp(&dev->qp_table, qp);
+
+	vfree(qp->s_wq);
+	vfree(qp->r_rq.wq);
+	kfree(qp);
+	return 0;
+}
+
+static struct ib_srq *ipath_create_srq(struct ib_pd *ibpd,
+				       struct ib_srq_init_attr *srq_init_attr,
+				       struct ib_udata *udata)
+{
+	struct ipath_srq *srq;
+	u32 sz;
+
+	if (srq_init_attr->attr.max_sge < 1)
+		return ERR_PTR(-EINVAL);
+
+	srq = kmalloc(sizeof(*srq), GFP_KERNEL);
+	if (!srq)
+		return ERR_PTR(-ENOMEM);
+
+	/* Need to use vmalloc() if we want to support large #s of entries. */
+	srq->rq.size = srq_init_attr->attr.max_wr + 1;
+	sz = sizeof(struct ipath_sge) * srq_init_attr->attr.max_sge +
+	    sizeof(struct ipath_rwqe);
+	srq->rq.wq = vmalloc(srq->rq.size * sz);
+	if (!srq->rq.wq) {
+		kfree(srq);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	/*
+	 * ib_create_srq() will initialize srq->ibsrq.
+	 */
+	spin_lock_init(&srq->rq.lock);
+	srq->rq.head = 0;
+	srq->rq.tail = 0;
+	srq->rq.max_sge = srq_init_attr->attr.max_sge;
+	srq->limit = srq_init_attr->attr.srq_limit;
+
+	return &srq->ibsrq;
+}
+
+int ipath_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
+		     enum ib_srq_attr_mask attr_mask)
+{
+	struct ipath_srq *srq = to_isrq(ibsrq);
+	unsigned long flags;
+
+	if (attr_mask & IB_SRQ_LIMIT) {
+		spin_lock_irqsave(&srq->rq.lock, flags);
+		srq->limit = attr->srq_limit;
+		spin_unlock_irqrestore(&srq->rq.lock, flags);
+	}
+	if (attr_mask & IB_SRQ_MAX_WR) {
+		u32 size = attr->max_wr + 1;
+		struct ipath_rwqe *wq, *p;
+		u32 n;
+		u32 sz;
+
+		if (attr->max_sge < srq->rq.max_sge)
+			return -EINVAL;
+
+		sz = sizeof(struct ipath_rwqe) +
+			attr->max_sge * sizeof(struct ipath_sge);
+		wq = vmalloc(size * sz);
+		if (!wq)
+			return -ENOMEM;
+
+		spin_lock_irqsave(&srq->rq.lock, flags);
+		if (srq->rq.head < srq->rq.tail)
+			n = srq->rq.size + srq->rq.head - srq->rq.tail;
+		else
+			n = srq->rq.head - srq->rq.tail;
+		if (size <= n || size <= srq->limit) {
+			spin_unlock_irqrestore(&srq->rq.lock, flags);
+			vfree(wq);
+			return -EINVAL;
+		}
+		n = 0;
+		p = wq;
+		while (srq->rq.tail != srq->rq.head) {
+			struct ipath_rwqe *wqe;
+			int i;
+
+			wqe = get_rwqe_ptr(&srq->rq, srq->rq.tail);
+			p->wr_id = wqe->wr_id;
+			p->length = wqe->length;
+			p->num_sge = wqe->num_sge;
+			for (i = 0; i < wqe->num_sge; i++)
+				p->sg_list[i] = wqe->sg_list[i];
+			n++;
+			p = (struct ipath_rwqe *)((char *) p + sz);
+			if (++srq->rq.tail >= srq->rq.size)
+				srq->rq.tail = 0;
+		}
+		vfree(srq->rq.wq);
+		srq->rq.wq = wq;
+		srq->rq.size = size;
+		srq->rq.head = n;
+		srq->rq.tail = 0;
+		srq->rq.max_sge = attr->max_sge;
+		spin_unlock_irqrestore(&srq->rq.lock, flags);
+	}
+	return 0;
+}
+
+static int ipath_destroy_srq(struct ib_srq *ibsrq)
+{
+	struct ipath_srq *srq = to_isrq(ibsrq);
+
+	vfree(srq->rq.wq);
+	kfree(srq);
+
+	return 0;
+}
+
+/*
+ * This may be called from interrupt context.
+ */
+static int ipath_poll_cq(struct ib_cq *ibcq, int num_entries,
+			 struct ib_wc *entry)
+{
+	struct ipath_cq *cq = to_icq(ibcq);
+	unsigned long flags;
+	int npolled;
+
+	spin_lock_irqsave(&cq->lock, flags);
+
+	for (npolled = 0; npolled < num_entries; ++npolled, ++entry) {
+		if (cq->tail == cq->head)
+			break;
+		*entry = cq->queue[cq->tail];
+		if (++cq->tail == cq->ibcq.cqe)
+			cq->tail = 0;
+	}
+
+	spin_unlock_irqrestore(&cq->lock, flags);
+
+	return npolled;
+}
+
+static struct ib_cq *ipath_create_cq(struct ib_device *ibdev, int entries,
+				     struct ib_ucontext *context,
+				     struct ib_udata *udata)
+{
+	struct ipath_cq *cq;
+
+	/* Need to use vmalloc() if we want to support large #s of entries. */
+	cq = vmalloc(sizeof(*cq) + entries * sizeof(*cq->queue));
+	if (!cq)
+		return ERR_PTR(-ENOMEM);
+	/*
+	 * ib_create_cq() will initialize cq->ibcq except for cq->ibcq.cqe.
+	 * The number of entries should be >= the number requested or
+	 * return an error.
+	 */
+	cq->ibcq.cqe = entries + 1;
+	cq->notify = IB_CQ_NONE;
+	cq->triggered = 0;
+	spin_lock_init(&cq->lock);
+	tasklet_init(&cq->comptask, send_complete, (unsigned long)cq);
+	cq->head = 0;
+	cq->tail = 0;
+
+	return &cq->ibcq;
+}
+
+static int ipath_destroy_cq(struct ib_cq *ibcq)
+{
+	struct ipath_cq *cq = to_icq(ibcq);
+
+	tasklet_kill(&cq->comptask);
+	vfree(cq);
+
+	return 0;
+}
+
+/*
+ * This may be called from interrupt context.
+ */
+static int ipath_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify notify)
+{
+	struct ipath_cq *cq = to_icq(ibcq);
+	unsigned long flags;
+
+	spin_lock_irqsave(&cq->lock, flags);
+	/*
+	 * Don't change IB_CQ_NEXT_COMP to IB_CQ_SOLICITED but allow
+	 * any other transitions.
+	 */
+	if (cq->notify != IB_CQ_NEXT_COMP)
+		cq->notify = notify;
+	spin_unlock_irqrestore(&cq->lock, flags);
+	return 0;
+}
+
+static int ipath_query_device(struct ib_device *ibdev,
+			      struct ib_device_attr *props)
+{
+	struct ipath_ibdev *dev = to_idev(ibdev);
+	uint32_t vendor, boardrev, majrev, minrev;
+
+	memset(props, 0, sizeof(*props));
+
+	props->device_cap_flags = IB_DEVICE_BAD_PKEY_CNTR |
+		IB_DEVICE_BAD_QKEY_CNTR | IB_DEVICE_SHUTDOWN_PORT |
+		IB_DEVICE_SYS_IMAGE_GUID;
+	ipath_layer_query_device(dev->ib_unit, &vendor, &boardrev,
+				 &majrev, &minrev);
+	props->vendor_id = vendor;
+	props->vendor_part_id = boardrev;
+	props->hw_ver = boardrev << 16 | majrev << 8 | minrev;
+
+	props->sys_image_guid = dev->sys_image_guid;
+	props->node_guid = ipath_layer_get_guid(dev->ib_unit);
+
+	props->max_mr_size = ~0ull;
+	props->max_qp = 0xffff;
+	props->max_qp_wr = 0xffff;
+	props->max_sge = 255;
+	props->max_cq = 0xffff;
+	props->max_cqe = 0xffff;
+	props->max_mr = 0xffff;
+	props->max_pd = 0xffff;
+	props->max_qp_rd_atom = 1;
+	props->max_qp_init_rd_atom = 1;
+	/* props->max_res_rd_atom */
+	props->max_srq = 0xffff;
+	props->max_srq_wr = 0xffff;
+	props->max_srq_sge = 255;
+	/* props->local_ca_ack_delay */
+	props->atomic_cap = IB_ATOMIC_HCA;
+	props->max_pkeys = ipath_layer_get_npkeys(dev->ib_unit);
+	props->max_mcast_grp = 0xffff;
+	props->max_mcast_qp_attach = 0xffff;
+	props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
+					   props->max_mcast_grp;
+
+	return 0;
+}
+
+static int ipath_query_port(struct ib_device *ibdev,
+			    u8 port, struct ib_port_attr *props)
+{
+	struct ipath_ibdev *dev = to_idev(ibdev);
+	uint32_t flags = ipath_layer_get_flags(dev->ib_unit);
+	enum ib_mtu mtu;
+	uint32_t l;
+	uint16_t lid = ipath_layer_get_lid(dev->ib_unit);
+
+	memset(props, 0, sizeof(*props));
+	props->lid = lid ? lid : IB_LID_PERMISSIVE;
+	props->lmc = dev->mkeyprot_resv_lmc & 7;
+	props->sm_lid = dev->sm_lid;
+	props->sm_sl = dev->sm_sl;
+	if (flags & IPATH_LINKDOWN)
+		props->state = IB_PORT_DOWN;
+	else if (flags & IPATH_LINKARMED)
+		props->state = IB_PORT_ARMED;
+	else if (flags & IPATH_LINKACTIVE)
+		props->state = IB_PORT_ACTIVE;
+	else if (flags & IPATH_LINK_SLEEPING)
+		props->state = IB_PORT_ACTIVE_DEFER;
+	else
+		props->state = IB_PORT_NOP;
+	/* See phys_state_show() */
+	props->phys_state = 5;	/* LinkUp */
+	props->port_cap_flags = dev->port_cap_flags;
+	props->gid_tbl_len = 1;
+	props->max_msg_sz = 4096;
+	props->pkey_tbl_len = ipath_layer_get_npkeys(dev->ib_unit);
+	props->bad_pkey_cntr = ipath_layer_get_cr_errpkey(dev->ib_unit);
+	props->qkey_viol_cntr = dev->qkey_violations;
+	props->active_width = IB_WIDTH_4X;
+	/* See rate_show() */
+	props->active_speed = 1;	/* Regular 10Mbs speed. */
+	props->max_vl_num = 1;		/* VLCap = VL0 */
+	props->init_type_reply = 0;
+
+	props->max_mtu = IB_MTU_4096;
+	l = ipath_layer_get_ibmtu(dev->ib_unit);
+	switch (l) {
+	case 4096:
+		mtu = IB_MTU_4096;
+		break;
+	case 2048:
+		mtu = IB_MTU_2048;
+		break;
+	case 1024:
+		mtu = IB_MTU_1024;
+		break;
+	case 512:
+		mtu = IB_MTU_512;
+		break;
+	case 256:
+		mtu = IB_MTU_256;
+		break;
+	default:
+		mtu = IB_MTU_2048;
+	}
+	props->active_mtu = mtu;
+	props->subnet_timeout = dev->subnet_timeout;
+
+	return 0;
+}
+
+static int ipath_modify_device(struct ib_device *device,
+			       int device_modify_mask,
+			       struct ib_device_modify *device_modify)
+{
+	if (device_modify_mask & IB_DEVICE_MODIFY_SYS_IMAGE_GUID)
+		to_idev(device)->sys_image_guid = device_modify->sys_image_guid;
+
+	return 0;
+}
+
+static int ipath_modify_port(struct ib_device *ibdev,
+			     u8 port, int port_modify_mask,
+			     struct ib_port_modify *props)
+{
+	struct ipath_ibdev *dev = to_idev(ibdev);
+
+	atomic_set_mask(props->set_port_cap_mask, &dev->port_cap_flags);
+	atomic_clear_mask(props->clr_port_cap_mask, &dev->port_cap_flags);
+	if (port_modify_mask & IB_PORT_SHUTDOWN)
+		ipath_kset_linkstate(dev->ib_unit << 16 | IPATH_IB_LINKDOWN);
+	if (port_modify_mask & IB_PORT_RESET_QKEY_CNTR)
+		dev->qkey_violations = 0;
+	return 0;
+}
+
+static int ipath_query_pkey(struct ib_device *ibdev,
+			    u8 port, u16 index, u16 *pkey)
+{
+	struct ipath_ibdev *dev = to_idev(ibdev);
+
+	if (index >= ipath_layer_get_npkeys(dev->ib_unit))
+		return -EINVAL;
+	*pkey = ipath_layer_get_pkey(dev->ib_unit, index);
+	return 0;
+}
+
+static int ipath_query_gid(struct ib_device *ibdev, u8 port,
+			   int index, union ib_gid *gid)
+{
+	struct ipath_ibdev *dev = to_idev(ibdev);
+
+	if (index >= 1)
+		return -EINVAL;
+	gid->global.subnet_prefix = dev->gid_prefix;
+	gid->global.interface_id = ipath_layer_get_guid(dev->ib_unit);
+
+	return 0;
+}
+
+static struct ib_pd *ipath_alloc_pd(struct ib_device *ibdev,
+				    struct ib_ucontext *context,
+				    struct ib_udata *udata)
+{
+	struct ipath_pd *pd;
+
+	pd = kmalloc(sizeof *pd, GFP_KERNEL);
+	if (!pd)
+		return ERR_PTR(-ENOMEM);
+
+	/* ib_alloc_pd() will initialize pd->ibpd. */
+	pd->user = udata != NULL;
+
+	return &pd->ibpd;
+}
+
+static int ipath_dealloc_pd(struct ib_pd *ibpd)
+{
+	struct ipath_pd *pd = to_ipd(ibpd);
+
+	kfree(pd);
+
+	return 0;
+}
+
+/*
+ * This may be called from interrupt context.
+ */
+static struct ib_ah *ipath_create_ah(struct ib_pd *pd,
+				     struct ib_ah_attr *ah_attr)
+{
+	struct ipath_ah *ah;
+
+	ah = kmalloc(sizeof *ah, GFP_ATOMIC);
+	if (!ah)
+		return ERR_PTR(-ENOMEM);
+
+	/* ib_create_ah() will initialize ah->ibah. */
+	ah->attr = *ah_attr;
+
+	return &ah->ibah;
+}
+
+/*
+ * This may be called from interrupt context.
+ */
+static int ipath_destroy_ah(struct ib_ah *ibah)
+{
+	struct ipath_ah *ah = to_iah(ibah);
+
+	kfree(ah);
+
+	return 0;
+}
+
+static struct ib_mr *ipath_get_dma_mr(struct ib_pd *pd, int acc)
+{
+	struct ipath_mr *mr;
+
+	mr = kmalloc(sizeof *mr, GFP_KERNEL);
+	if (!mr)
+		return ERR_PTR(-ENOMEM);
+
+	/* ib_get_dma_mr() will initialize mr->ibmr except for lkey and rkey. */
+	memset(mr, 0, sizeof *mr);
+	mr->mr.access_flags = acc;
+	return &mr->ibmr;
+}
+
+static struct ib_mr *ipath_reg_phys_mr(struct ib_pd *pd,
+				       struct ib_phys_buf *buffer_list,
+				       int num_phys_buf,
+				       int acc, u64 *iova_start)
+{
+	struct ipath_mr *mr;
+	int n, m, i;
+
+	/* Allocate struct plus pointers to first level page tables. */
+	m = (num_phys_buf + IPATH_SEGSZ - 1) / IPATH_SEGSZ;
+	mr = kmalloc(sizeof *mr + m * sizeof mr->mr.map[0], GFP_KERNEL);
+	if (!mr)
+		return ERR_PTR(-ENOMEM);
+
+	/* Allocate first level page tables. */
+	for (i = 0; i < m; i++) {
+		mr->mr.map[i] = kmalloc(sizeof *mr->mr.map[0], GFP_KERNEL);
+		if (!mr->mr.map[i]) {
+			while (i)
+				kfree(mr->mr.map[--i]);
+			kfree(mr);
+			return ERR_PTR(-ENOMEM);
+		}
+	}
+	mr->mr.mapsz = m;
+
+	/*
+	 * ib_reg_phys_mr() will initialize mr->ibmr except for
+	 * lkey and rkey.
+	 */
+	if (!ipath_alloc_lkey(&to_idev(pd->device)->lk_table, &mr->mr)) {
+		while (i)
+			kfree(mr->mr.map[--i]);
+		kfree(mr);
+		return ERR_PTR(-ENOMEM);
+	}
+	mr->ibmr.rkey = mr->ibmr.lkey = mr->mr.lkey;
+	mr->mr.user_base = *iova_start;
+	mr->mr.iova = *iova_start;
+	mr->mr.length = 0;
+	mr->mr.offset = 0;
+	mr->mr.access_flags = acc;
+	mr->mr.max_segs = num_phys_buf;
+	m = 0;
+	n = 0;
+	for (i = 0; i < num_phys_buf; i++) {
+		mr->mr.map[m]->segs[n].vaddr =
+			phys_to_virt(buffer_list[i].addr);
+		mr->mr.map[m]->segs[n].length = buffer_list[i].size;
+		mr->mr.length += buffer_list[i].size;
+		if (++n == IPATH_SEGSZ) {
+			m++;
+			n = 0;
+		}
+	}
+	return &mr->ibmr;
+}
+
+static struct ib_mr *ipath_reg_user_mr(struct ib_pd *pd,
+				       struct ib_umem *region,
+				       int mr_access_flags,
+				       struct ib_udata *udata)
+{
+	struct ipath_mr *mr;
+	struct ib_umem_chunk *chunk;
+	int n, m, i;
+
+	n = 0;
+	list_for_each_entry(chunk, &region->chunk_list, list)
+	    n += chunk->nents;
+
+	/* Allocate struct plus pointers to first level page tables. */
+	m = (n + IPATH_SEGSZ - 1) / IPATH_SEGSZ;
+	mr = kmalloc(sizeof *mr + m * sizeof mr->mr.map[0], GFP_KERNEL);
+	if (!mr)
+		return ERR_PTR(-ENOMEM);
+
+	/* Allocate first level page tables. */
+	for (i = 0; i < m; i++) {
+		mr->mr.map[i] = kmalloc(sizeof *mr->mr.map[0], GFP_KERNEL);
+		if (!mr->mr.map[i]) {
+			while (i)
+				kfree(mr->mr.map[--i]);
+			kfree(mr);
+			return ERR_PTR(-ENOMEM);
+		}
+	}
+	mr->mr.mapsz = m;
+
+	/*
+	 * ib_uverbs_reg_mr() will initialize mr->ibmr except for
+	 * lkey and rkey.
+	 */
+	if (!ipath_alloc_lkey(&to_idev(pd->device)->lk_table, &mr->mr)) {
+		while (i)
+			kfree(mr->mr.map[--i]);
+		kfree(mr);
+		return ERR_PTR(-ENOMEM);
+	}
+	mr->ibmr.rkey = mr->ibmr.lkey = mr->mr.lkey;
+	mr->mr.user_base = region->user_base;
+	mr->mr.iova = region->virt_base;
+	mr->mr.length = region->length;
+	mr->mr.offset = region->offset;
+	mr->mr.access_flags = mr_access_flags;
+	mr->mr.max_segs = n;
+	m = 0;
+	n = 0;
+	list_for_each_entry(chunk, &region->chunk_list, list) {
+		for (i = 0; i < chunk->nmap; i++) {
+			mr->mr.map[m]->segs[n].vaddr =
+			    page_address(chunk->page_list[i].page);
+			mr->mr.map[m]->segs[n].length = region->page_size;
+			if (++n == IPATH_SEGSZ) {
+				m++;
+				n = 0;
+			}
+		}
+	}
+	return &mr->ibmr;
+}
+
+/*
+ * Note that this is called to free MRs created by
+ * ipath_get_dma_mr() or ipath_reg_user_mr().
+ */
+static int ipath_dereg_mr(struct ib_mr *ibmr)
+{
+	struct ipath_mr *mr = to_imr(ibmr);
+	int i;
+
+	ipath_free_lkey(&to_idev(ibmr->device)->lk_table, ibmr->lkey);
+	i = mr->mr.mapsz;
+	while (i)
+		kfree(mr->mr.map[--i]);
+	kfree(mr);
+	return 0;
+}
+
+static struct ib_fmr *ipath_alloc_fmr(struct ib_pd *pd,
+				      int mr_access_flags,
+				      struct ib_fmr_attr *fmr_attr)
+{
+	struct ipath_fmr *fmr;
+	int m, i;
+
+	/* Allocate struct plus pointers to first level page tables. */
+	m = (fmr_attr->max_pages + IPATH_SEGSZ - 1) / IPATH_SEGSZ;
+	fmr = kmalloc(sizeof *fmr + m * sizeof fmr->mr.map[0], GFP_KERNEL);
+	if (!fmr)
+		return ERR_PTR(-ENOMEM);
+
+	/* Allocate first level page tables. */
+	for (i = 0; i < m; i++) {
+		fmr->mr.map[i] = kmalloc(sizeof *fmr->mr.map[0], GFP_KERNEL);
+		if (!fmr->mr.map[i]) {
+			while (i)
+				kfree(fmr->mr.map[--i]);
+			kfree(fmr);
+			return ERR_PTR(-ENOMEM);
+		}
+	}
+	fmr->mr.mapsz = m;
+
+	/* ib_alloc_fmr() will initialize fmr->ibfmr except for lkey & rkey. */
+	if (!ipath_alloc_lkey(&to_idev(pd->device)->lk_table, &fmr->mr)) {
+		while (i)
+			kfree(fmr->mr.map[--i]);
+		kfree(fmr);
+		return ERR_PTR(-ENOMEM);
+	}
+	fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mr.lkey;
+	/* Resources are allocated but no valid mapping (RKEY can't be used). */
+	fmr->mr.user_base = 0;
+	fmr->mr.iova = 0;
+	fmr->mr.length = 0;
+	fmr->mr.offset = 0;
+	fmr->mr.access_flags = mr_access_flags;
+	fmr->mr.max_segs = fmr_attr->max_pages;
+	fmr->page_size = fmr_attr->page_size;
+	return &fmr->ibfmr;
+}
+
+/*
+ * This may be called from interrupt context.
+ * XXX Can we ever be called to map a portion of the RKEY space?
+ */
+static int ipath_map_phys_fmr(struct ib_fmr *ibfmr,
+			      u64 * page_list, int list_len, u64 iova)
+{
+	struct ipath_fmr *fmr = to_ifmr(ibfmr);
+	struct ipath_lkey_table *rkt;
+	unsigned long flags;
+	int m, n, i;
+	u32 ps;
+
+	if (list_len > fmr->mr.max_segs)
+		return -EINVAL;
+	rkt = &to_idev(ibfmr->device)->lk_table;
+	spin_lock_irqsave(&rkt->lock, flags);
+	fmr->mr.user_base = iova;
+	fmr->mr.iova = iova;
+	ps = 1 << fmr->page_size;
+	fmr->mr.length = list_len * ps;
+	m = 0;
+	n = 0;
+	ps = 1 << fmr->page_size;
+	for (i = 0; i < list_len; i++) {
+		fmr->mr.map[m]->segs[n].vaddr = phys_to_virt(page_list[i]);
+		fmr->mr.map[m]->segs[n].length = ps;
+		if (++n == IPATH_SEGSZ) {
+			m++;
+			n = 0;
+		}
+	}
+	spin_unlock_irqrestore(&rkt->lock, flags);
+	return 0;
+}
+
+static int ipath_unmap_fmr(struct list_head *fmr_list)
+{
+	struct ipath_fmr *fmr;
+
+	list_for_each_entry(fmr, fmr_list, ibfmr.list) {
+		fmr->mr.user_base = 0;
+		fmr->mr.iova = 0;
+		fmr->mr.length = 0;
+	}
+	return 0;
+}
+
+static int ipath_dealloc_fmr(struct ib_fmr *ibfmr)
+{
+	struct ipath_fmr *fmr = to_ifmr(ibfmr);
+	int i;
+
+	ipath_free_lkey(&to_idev(ibfmr->device)->lk_table, ibfmr->lkey);
+	i = fmr->mr.mapsz;
+	while (i)
+		kfree(fmr->mr.map[--i]);
+	kfree(fmr);
+	return 0;
+}
+
+static ssize_t show_rev(struct class_device *cdev, char *buf)
+{
+	struct ipath_ibdev *dev =
+	    container_of(cdev, struct ipath_ibdev, ibdev.class_dev);
+	int vendor, boardrev, majrev, minrev;
+
+	ipath_layer_query_device(dev->ib_unit, &vendor, &boardrev,
+				 &majrev, &minrev);
+	return sprintf(buf, "%d.%d\n", majrev, minrev);
+}
+
+static ssize_t show_hca(struct class_device *cdev, char *buf)
+{
+	struct ipath_ibdev *dev =
+	    container_of(cdev, struct ipath_ibdev, ibdev.class_dev);
+	int vendor, boardrev, majrev, minrev;
+
+	ipath_layer_query_device(dev->ib_unit, &vendor, &boardrev,
+				 &majrev, &minrev);
+	ipath_get_boardname(dev->ib_unit, buf, 128);
+	strcat(buf, "\n");
+	return strlen(buf);
+}
+
+static ssize_t show_board(struct class_device *cdev, char *buf)
+{
+	struct ipath_ibdev *dev =
+	    container_of(cdev, struct ipath_ibdev, ibdev.class_dev);
+	int vendor, boardrev, majrev, minrev;
+
+	ipath_layer_query_device(dev->ib_unit, &vendor, &boardrev,
+				 &majrev, &minrev);
+	ipath_get_boardname(dev->ib_unit, buf, 128);
+	strcat(buf, "\n");
+	return strlen(buf);
+}
+
+static ssize_t show_stats(struct class_device *cdev, char *buf)
+{
+	struct ipath_ibdev *dev =
+	    container_of(cdev, struct ipath_ibdev, ibdev.class_dev);
+	char *p;
+	int i;
+
+	sprintf(buf,
+		"RC resends  %d\n"
+		"RC QACKs    %d\n"
+		"RC ACKs     %d\n"
+		"RC SEQ NAKs %d\n"
+		"RC RDMA seq %d\n"
+		"RC RNR NAKs %d\n"
+		"RC OTH NAKs %d\n"
+		"RC timeouts %d\n"
+		"RC RDMA dup %d\n"
+		"piobuf wait %d\n"
+		"no piobuf   %d\n"
+		"PKT drops   %d\n"
+		"WQE errs    %d\n",
+		dev->n_rc_resends, dev->n_rc_qacks, dev->n_rc_acks,
+		dev->n_seq_naks, dev->n_rdma_seq, dev->n_rnr_naks,
+		dev->n_other_naks, dev->n_timeouts, dev->n_rdma_dup_busy,
+		dev->n_piowait, dev->n_no_piobuf, dev->n_pkt_drops,
+		dev->n_wqe_errs);
+	p = buf;
+	for (i = 0; i < ARRAY_SIZE(dev->opstats); i++) {
+		if (!dev->opstats[i].n_packets && !dev->opstats[i].n_bytes)
+			continue;
+		p += strlen(p);
+		sprintf(p, "%02x %llu/%llu\n",
+			i, dev->opstats[i].n_packets, dev->opstats[i].n_bytes);
+	}
+	return strlen(buf);
+}
+
+static CLASS_DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
+static CLASS_DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
+static CLASS_DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL);
+static CLASS_DEVICE_ATTR(stats, S_IRUGO, show_stats, NULL);
+
+static struct class_device_attribute *ipath_class_attributes[] = {
+	&class_device_attr_hw_rev,
+	&class_device_attr_hca_type,
+	&class_device_attr_board_id,
+	&class_device_attr_stats
+};
+
+/*
+ * Allocate a ucontext.
+ */
+
+static struct ib_ucontext *ipath_alloc_ucontext(struct ib_device *ibdev,
+						struct ib_udata *udata)
+{
+	struct ipath_ucontext *context;
+
+	context = kmalloc(sizeof *context, GFP_KERNEL);
+	if (!context)
+		return ERR_PTR(-ENOMEM);
+
+	return &context->ibucontext;
+}
+
+static int ipath_dealloc_ucontext(struct ib_ucontext *context)
+{
+	kfree(to_iucontext(context));
+	return 0;
+}
+
+/*
+ * Register our device with the infiniband core.
+ */
+static int ipath_register_ib_device(const ipath_type t)
+{
+	struct ipath_ibdev *idev;
+	struct ib_device *dev;
+	int i;
+	int ret;
+
+	idev = (struct ipath_ibdev *)ib_alloc_device(sizeof *idev);
+	if (idev == NULL)
+		return -ENOMEM;
+
+	dev = &idev->ibdev;
+
+	/* Only need to initialize non-zero fields. */
+	spin_lock_init(&idev->qp_table.lock);
+	spin_lock_init(&idev->lk_table.lock);
+	idev->sm_lid = IB_LID_PERMISSIVE;
+	idev->gid_prefix = __constant_cpu_to_be64(0xfe80000000000000UL);
+	idev->qp_table.last = 1;	/* QPN 0 and 1 are special. */
+	idev->qp_table.max = ib_ipath_qp_table_size;
+	idev->qp_table.nmaps = 1;
+	idev->qp_table.table = kmalloc(idev->qp_table.max *
+				       sizeof(*idev->qp_table.table),
+				       GFP_KERNEL);
+	if (idev->qp_table.table == NULL) {
+		ret = -ENOMEM;
+		goto err_qp;
+	}
+	memset(idev->qp_table.table, 0,
+	       idev->qp_table.max * sizeof(*idev->qp_table.table));
+	for (i = 0; i < ARRAY_SIZE(idev->qp_table.map); i++) {
+		atomic_set(&idev->qp_table.map[i].n_free, BITS_PER_PAGE);
+		idev->qp_table.map[i].page = NULL;
+	}
+	/*
+	 * The top ib_ipath_lkey_table_size bits are used to index the table.
+	 * The lower 8 bits can be owned by the user (copied from the LKEY).
+	 * The remaining bits act as a generation number or tag.
+	 */
+	idev->lk_table.max = 1 << ib_ipath_lkey_table_size;
+	idev->lk_table.table = kmalloc(idev->lk_table.max *
+				       sizeof(*idev->lk_table.table),
+				       GFP_KERNEL);
+	if (idev->lk_table.table == NULL) {
+		ret = -ENOMEM;
+		goto err_lk;
+	}
+	memset(idev->lk_table.table, 0,
+	       idev->lk_table.max * sizeof(*idev->lk_table.table));
+	spin_lock_init(&idev->pending_lock);
+	INIT_LIST_HEAD(&idev->pending[0]);
+	INIT_LIST_HEAD(&idev->pending[1]);
+	INIT_LIST_HEAD(&idev->pending[2]);
+	INIT_LIST_HEAD(&idev->piowait);
+	INIT_LIST_HEAD(&idev->rnrwait);
+	idev->pending_index = 0;
+	idev->port_cap_flags =
+		IB_PORT_SYS_IMAGE_GUID_SUP | IB_PORT_CLIENT_REG_SUP;
+	idev->pma_counter_select[0] = IB_PMA_PORT_XMIT_DATA;
+	idev->pma_counter_select[1] = IB_PMA_PORT_RCV_DATA;
+	idev->pma_counter_select[2] = IB_PMA_PORT_XMIT_PKTS;
+	idev->pma_counter_select[3] = IB_PMA_PORT_RCV_PKTS;
+	idev->pma_counter_select[5] = IB_PMA_PORT_XMIT_WAIT;
+
+	/*
+	 * The system image GUI is supposed to be the same for all
+	 * IB HCAs in a single system.
+	 * Note that this code assumes device zero is found first.
+	 */
+	idev->sys_image_guid =
+		t ? ipath_devices[t]->sys_image_guid : ipath_layer_get_guid(t);
+	idev->ib_unit = t;
+
+	strlcpy(dev->name, "ipath%d", IB_DEVICE_NAME_MAX);
+	dev->node_guid = ipath_layer_get_guid(t);
+	dev->uverbs_abi_ver = IPATH_UVERBS_ABI_VERSION;
+	dev->uverbs_cmd_mask =
+		(1ull << IB_USER_VERBS_CMD_GET_CONTEXT)		|
+		(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)	|
+		(1ull << IB_USER_VERBS_CMD_QUERY_PORT)		|
+		(1ull << IB_USER_VERBS_CMD_ALLOC_PD)		|
+		(1ull << IB_USER_VERBS_CMD_DEALLOC_PD)		|
+		(1ull << IB_USER_VERBS_CMD_CREATE_AH)		|
+		(1ull << IB_USER_VERBS_CMD_DESTROY_AH)		|
+		(1ull << IB_USER_VERBS_CMD_REG_MR)		|
+		(1ull << IB_USER_VERBS_CMD_DEREG_MR)		|
+		(1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
+		(1ull << IB_USER_VERBS_CMD_CREATE_CQ)		|
+		(1ull << IB_USER_VERBS_CMD_DESTROY_CQ)		|
+		(1ull << IB_USER_VERBS_CMD_POLL_CQ)		|
+		(1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ)	|
+		(1ull << IB_USER_VERBS_CMD_CREATE_QP)		|
+		(1ull << IB_USER_VERBS_CMD_MODIFY_QP)		|
+		(1ull << IB_USER_VERBS_CMD_DESTROY_QP)		|
+		(1ull << IB_USER_VERBS_CMD_POST_SEND)		|
+		(1ull << IB_USER_VERBS_CMD_POST_RECV)		|
+		(1ull << IB_USER_VERBS_CMD_ATTACH_MCAST)	|
+		(1ull << IB_USER_VERBS_CMD_DETACH_MCAST)	|
+		(1ull << IB_USER_VERBS_CMD_CREATE_SRQ)		|
+		(1ull << IB_USER_VERBS_CMD_MODIFY_SRQ)		|
+		(1ull << IB_USER_VERBS_CMD_DESTROY_SRQ)		|
+		(1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV);
+	dev->node_type = IB_NODE_CA;
+	dev->phys_port_cnt = 1;
+	dev->dma_device = ipath_layer_get_pcidev(t);
+	dev->class_dev.dev = dev->dma_device;
+	dev->query_device = ipath_query_device;
+	dev->modify_device = ipath_modify_device;
+	dev->query_port = ipath_query_port;
+	dev->modify_port = ipath_modify_port;
+	dev->query_pkey = ipath_query_pkey;
+	dev->query_gid = ipath_query_gid;
+	dev->alloc_ucontext = ipath_alloc_ucontext;
+	dev->dealloc_ucontext = ipath_dealloc_ucontext;
+	dev->alloc_pd = ipath_alloc_pd;
+	dev->dealloc_pd = ipath_dealloc_pd;
+	dev->create_ah = ipath_create_ah;
+	dev->destroy_ah = ipath_destroy_ah;
+	dev->create_srq = ipath_create_srq;
+	dev->modify_srq = ipath_modify_srq;
+	dev->destroy_srq = ipath_destroy_srq;
+	dev->create_qp = ipath_create_qp;
+	dev->modify_qp = ipath_modify_qp;
+	dev->destroy_qp = ipath_destroy_qp;
+	dev->post_send = ipath_post_send;
+	dev->post_recv = ipath_post_receive;
+	dev->post_srq_recv = ipath_post_srq_receive;
+	dev->create_cq = ipath_create_cq;
+	dev->destroy_cq = ipath_destroy_cq;
+	dev->poll_cq = ipath_poll_cq;
+	dev->req_notify_cq = ipath_req_notify_cq;
+	dev->get_dma_mr = ipath_get_dma_mr;
+	dev->reg_phys_mr = ipath_reg_phys_mr;
+	dev->reg_user_mr = ipath_reg_user_mr;
+	dev->dereg_mr = ipath_dereg_mr;
+	dev->alloc_fmr = ipath_alloc_fmr;
+	dev->map_phys_fmr = ipath_map_phys_fmr;
+	dev->unmap_fmr = ipath_unmap_fmr;
+	dev->dealloc_fmr = ipath_dealloc_fmr;
+	dev->attach_mcast = ipath_multicast_attach;
+	dev->detach_mcast = ipath_multicast_detach;
+	dev->process_mad = ipath_process_mad;
+
+	ret = ib_register_device(dev);
+	if (ret)
+		goto err_reg;
+
+	for (i = 0; i < ARRAY_SIZE(ipath_class_attributes); ++i) {
+		ret = class_device_create_file(&dev->class_dev,
+					       ipath_class_attributes[i]);
+		if (ret)
+			goto err_class;
+	}
+
+	ipath_layer_enable_timer(t);
+
+	ipath_devices[t] = idev;
+	return 0;
+
+err_class:
+	ib_unregister_device(dev);
+err_reg:
+	kfree(idev->lk_table.table);
+err_lk:
+	kfree(idev->qp_table.table);
+err_qp:
+	ib_dealloc_device(dev);
+	return ret;
+}
+
+static void ipath_unregister_ib_device(struct ipath_ibdev *dev)
+{
+	struct ib_device *ibdev = &dev->ibdev;
+
+	ipath_layer_disable_timer(dev->ib_unit);
+
+	ib_unregister_device(ibdev);
+
+	if (!list_empty(&dev->pending[0]) || !list_empty(&dev->pending[1]) ||
+	    !list_empty(&dev->pending[2]))
+		_VERBS_ERROR("ipath%d pending list not empty!\n", dev->ib_unit);
+	if (!list_empty(&dev->piowait))
+		_VERBS_ERROR("ipath%d piowait list not empty!\n", dev->ib_unit);
+	if (!list_empty(&dev->rnrwait))
+		_VERBS_ERROR("ipath%d rnrwait list not empty!\n", dev->ib_unit);
+	if (mcast_tree.rb_node != NULL)
+		_VERBS_ERROR("ipath%d multicast table memory leak!\n",
+			     dev->ib_unit);
+	/*
+	 * Note that ipath_unregister_ib_device() can be called before all
+	 * the QPs are destroyed!
+	 */
+	ipath_free_all_qps(&dev->qp_table);
+	kfree(dev->qp_table.table);
+	kfree(dev->lk_table.table);
+	ib_dealloc_device(ibdev);
+}
+
+int __init ipath_verbs_init(void)
+{
+	int i;
+
+	number_of_devices = ipath_layer_get_num_of_dev();
+	i = number_of_devices * sizeof(struct ipath_ibdev *);
+	ipath_devices = kmalloc(i, GFP_ATOMIC);
+	if (ipath_devices == NULL)
+		return -ENOMEM;
+
+	for (i = 0; i < number_of_devices; i++) {
+		int ret = ipath_verbs_register(i, ipath_ib_piobufavail,
+					       ipath_ib_rcv, ipath_ib_timer);
+
+		if (ret == 0)
+			ipath_devices[i] = NULL;
+		else if ((ret = ipath_register_ib_device(i)) != 0) {
+			_VERBS_ERROR("ib_ipath%d cannot register ib device "
+				     "(%d)!\n", i, ret);
+			ipath_verbs_unregister(i);
+			ipath_devices[i] = NULL;
+		}
+	}
+
+	return 0;
+}
+
+void __exit ipath_verbs_cleanup(void)
+{
+	int i;
+
+	for (i = 0; i < number_of_devices; i++)
+		if (ipath_devices[i]) {
+			ipath_unregister_ib_device(ipath_devices[i]);
+			ipath_verbs_unregister(i);
+		}
+
+	kfree(ipath_devices);
+}
+
+module_init(ipath_verbs_init);
+module_exit(ipath_verbs_cleanup);
Signature · 1 line
0.99.9n
#13 lore raw

RFC PATCH 12/13 ipath verbs MAD handling

MAD handling for ipath verbs driver

---

 drivers/infiniband/hw/ipath/ipath_mad.c | 1020 +++++++++++++++++++++++++++++++
 1 files changed, 1020 insertions(+), 0 deletions(-)
 create mode 100644 drivers/infiniband/hw/ipath/ipath_mad.c

21556dcebce3886215a02ff1b730a60beea53125

drivers/infiniband/hw/ipath/ipath_mad.c new file +1020 -0
new file mode 100644
index 0000000..361c7fb
@@ -0,0 +1,1020 @@
+/*
+ * Copyright (c) 2005. PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Patent licenses, if any, provided herein do not apply to
+ * combinations of this program with other software, or any other
+ * product whatsoever.
+ *
+ * $Id: ipath_mad.c 4491 2005-12-15 22:20:31Z rjwalsh $
+ */
+
+#include <linux/version.h>
+#include <rdma/ib_smi.h>
+
+#include "ips_common.h"
+#include "ipath_verbs.h"
+#include "ipath_layer.h"
+
+
+#define IB_SMP_INVALID_FIELD	__constant_htons(0x001C)
+
+static int reply(struct ib_smp *smp, int line)
+{
+
+	/*
+	 * The verbs framework will handle the directed/LID route
+	 * packet changes.
+	 */
+	smp->method = IB_MGMT_METHOD_GET_RESP;
+	if (smp->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
+		smp->status |= IB_SMP_DIRECTION;
+	return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+}
+
+static inline int recv_subn_get_nodedescription(struct ib_smp *smp)
+{
+
+	strncpy(smp->data, "Infinipath", sizeof(smp->data));
+
+	return reply(smp, __LINE__);
+}
+
+struct nodeinfo {
+	u8 base_version;
+	u8 class_version;
+	u8 node_type;
+	u8 num_ports;
+	__be64 sys_guid;
+	__be64 node_guid;
+	__be64 port_guid;
+	__be16 partition_cap;
+	__be16 device_id;
+	__be32 revision;
+	u8 local_port_num;
+	u8 vendor_id[3];
+} __attribute__ ((packed));
+
+/*
+ * XXX The num_ports value will need a layer function to get the value
+ * if we ever have more than one IB port on a chip.
+ * We will also need to get the GUID for the port.
+ */
+static inline int recv_subn_get_nodeinfo(struct ib_smp *smp,
+					 struct ib_device *ibdev, u8 port)
+{
+	struct nodeinfo *nip = (struct nodeinfo *)&smp->data;
+	ipath_type t = to_idev(ibdev)->ib_unit;
+	uint32_t vendor, boardid, majrev, minrev;
+
+	nip->base_version = 1;
+	nip->class_version = 1;
+	nip->node_type = 1;	/* channel adapter */
+	nip->num_ports = 1;
+	/* This is already in network order */
+	nip->sys_guid = to_idev(ibdev)->sys_image_guid;
+	nip->node_guid = ipath_layer_get_guid(t);
+	nip->port_guid = nip->sys_guid;
+	nip->partition_cap = cpu_to_be16(ipath_layer_get_npkeys(t));
+	nip->device_id = cpu_to_be16(ipath_layer_get_deviceid(t));
+	ipath_layer_query_device(t, &vendor, &boardid, &majrev, &minrev);
+	nip->revision = cpu_to_be32((majrev << 16) | minrev);
+	nip->local_port_num = port;
+	nip->vendor_id[0] = 0;
+	nip->vendor_id[1] = vendor >> 8;
+	nip->vendor_id[2] = vendor;
+
+	return reply(smp, __LINE__);
+}
+
+static int recv_subn_get_guidinfo(struct ib_smp *smp, struct ib_device *ibdev)
+{
+	uint32_t t = to_idev(ibdev)->ib_unit;
+	u32 startgx = 8 * be32_to_cpu(smp->attr_mod);
+	u64 *p = (u64 *) smp->data;
+
+	/* 32 blocks of 8 64-bit GUIDs per block */
+
+	memset(smp->data, 0, sizeof(smp->data));
+
+	/*
+	 * We only support one GUID for now.
+	 * If this changes, the portinfo.guid_cap field needs to be updated too.
+	 */
+	if (startgx == 0) {
+		/* The first is a copy of the read-only HW GUID. */
+		*p = ipath_layer_get_guid(t);
+	}
+
+	return reply(smp, __LINE__);
+}
+
+struct port_info {
+	__be64 mkey;
+	__be64 gid_prefix;
+	__be16 lid;
+	__be16 sm_lid;
+	__be32 cap_mask;
+	__be16 diag_code;
+	__be16 mkey_lease_period;
+	u8 local_port_num;
+	u8 link_width_enabled;
+	u8 link_width_supported;
+	u8 link_width_active;
+	u8 linkspeed_portstate;			/* 4 bits, 4 bits */
+	u8 portphysstate_linkdown;		/* 4 bits, 4 bits */
+	u8 mkeyprot_resv_lmc;			/* 2 bits, 3 bits, 3 bits */
+	u8 linkspeedactive_enabled;		/* 4 bits, 4 bits */
+	u8 neighbormtu_mastersmsl;		/* 4 bits, 4 bits */
+	u8 vlcap_inittype;			/* 4 bits, 4 bits */
+	u8 vl_high_limit;
+	u8 vl_arb_high_cap;
+	u8 vl_arb_low_cap;
+	u8 inittypereply_mtucap;		/* 4 bits, 4 bits */
+	u8 vlstallcnt_hoqlife;			/* 3 bits, 5 bits */
+	u8 operationalvl_pei_peo_fpi_fpo;	/* 4 bits, 1, 1, 1, 1 */
+	__be16 mkey_violations;
+	__be16 pkey_violations;
+	__be16 qkey_violations;
+	u8 guid_cap;
+	u8 clientrereg_resv_subnetto;		/* 1 bit, 2 bits, 5 bits */
+	u8 resv_resptimevalue;			/* 3 bits, 5 bits */
+	u8 localphyerrors_overrunerrors;	/* 4 bits, 4 bits */
+	__be16 max_credit_hint;
+	u8 resv;
+	u8 link_roundtrip_latency[3];
+} __attribute__ ((packed));
+
+static int recv_subn_get_portinfo(struct ib_smp *smp, struct ib_device *ibdev,
+				  u8 port)
+{
+	u32 lportnum = be32_to_cpu(smp->attr_mod);
+	struct ipath_ibdev *dev;
+	struct port_info *pip = (struct port_info *)smp->data;
+	u32 tmp, tmp2;
+
+	if (lportnum == 0) {
+		lportnum = port;
+		smp->attr_mod = cpu_to_be32(lportnum);
+	}
+
+	if (lportnum < 1 || lportnum > ibdev->phys_port_cnt)
+		return IB_MAD_RESULT_FAILURE;
+
+	dev = to_idev(ibdev);
+
+	/* Clear all fields.  Only set the non-zero fields. */
+	memset(smp->data, 0, sizeof(smp->data));
+
+	/* Only return the mkey if the protection field allows it. */
+	if ((dev->mkeyprot_resv_lmc >> 6) == 0)
+		pip->mkey = dev->mkey;
+	else
+		pip->mkey = 0;
+	pip->gid_prefix = dev->gid_prefix;
+	tmp = ipath_layer_get_lid(dev->ib_unit);
+	pip->lid = tmp ? cpu_to_be16(tmp) : IB_LID_PERMISSIVE;
+	pip->sm_lid = cpu_to_be16(dev->sm_lid);
+	pip->cap_mask = cpu_to_be32(dev->port_cap_flags);
+	/* pip->diag_code; */
+	pip->mkey_lease_period = cpu_to_be16(dev->mkey_lease_period);
+	pip->local_port_num = port;
+	pip->link_width_enabled = 2;	/* 4x */
+	pip->link_width_supported = 3;	/* 1x or 4x */
+	pip->link_width_active = 2;	/* 4x */
+	pip->linkspeed_portstate = 0x10;	/* 2.5Gbps */
+	tmp = ipath_layer_get_lastibcstat(dev->ib_unit) & 0xff;
+	tmp2 = 5;		/* link up */
+	if (tmp == 0x11)
+		pip->linkspeed_portstate |= 2;	/* initialize */
+	else if (tmp == 0x21)
+		pip->linkspeed_portstate |= 3;	/* armed */
+	else if (tmp == 0x31)
+		pip->linkspeed_portstate |= 4;	/* active */
+	else {
+		pip->linkspeed_portstate |= 1;	/* down */
+		tmp2 = tmp & 0xf;
+	}
+	/* default state is polling */
+	pip->portphysstate_linkdown = (tmp2 << 4) | 2;
+	pip->mkeyprot_resv_lmc = dev->mkeyprot_resv_lmc;
+	pip->linkspeedactive_enabled = 0x11;	/* 2.5Gbps, 2.5Gbps */
+	switch (ipath_layer_get_ibmtu(dev->ib_unit)) {
+	case 4096:
+		tmp = IB_MTU_4096;
+		break;
+	case 2048:
+		tmp = IB_MTU_2048;
+		break;
+	case 1024:
+		tmp = IB_MTU_1024;
+		break;
+	case 512:
+		tmp = IB_MTU_512;
+		break;
+	case 256:
+		tmp = IB_MTU_256;
+		break;
+	default:		/* oops, something is wrong */
+		tmp = IB_MTU_2048;
+		break;
+	}
+	pip->neighbormtu_mastersmsl = (tmp << 4) | dev->sm_sl;
+	pip->vlcap_inittype = 0x10;	/* VLCap = VL0, InitType = 0 */
+	/* pip->vl_high_limit; // only one VL */
+	/* pip->vl_arb_high_cap; // only one VL */
+	/* pip->vl_arb_low_cap; // only one VL */
+	pip->inittypereply_mtucap = IB_MTU_4096;	/* InitTypeReply = 0 */
+	/* pip->vlstallcnt_hoqlife; // HCAs ignore VLStallCount and HOQLife */
+	pip->operationalvl_pei_peo_fpi_fpo = 0x18;	/* OVLs = 1, PEI = 1 */
+	pip->mkey_violations = cpu_to_be16(dev->mkey_violations);
+	/* P_KeyViolations are counted by hardware. */
+	tmp = ipath_layer_get_cr_errpkey(dev->ib_unit) & 0xFFFF;
+	pip->pkey_violations = cpu_to_be16(tmp);
+	pip->qkey_violations = cpu_to_be16(dev->qkey_violations);
+	/* Only the hardware GUID is supported for now */
+	pip->guid_cap = 1;
+	pip->clientrereg_resv_subnetto = dev->subnet_timeout;
+	/* 32.768 usec. response time (guessing) */
+	pip->resv_resptimevalue = 3;
+	/* LocalPhyErrors=max, OverRunErrors=max */
+	pip->localphyerrors_overrunerrors = 0xFF;
+	/* pip->max_credit_hint; */
+	/* pip->link_roundtrip_latency[3]; */
+
+	return reply(smp, __LINE__);
+}
+
+static int recv_subn_get_pkeytable(struct ib_smp *smp, struct ib_device *ibdev)
+{
+	u32 startpx = 32 * (be32_to_cpu(smp->attr_mod) & 0xffff);
+	u16 *p = (u16 *) smp->data;
+
+	/* 64 blocks of 32 16-bit P_Key entries */
+
+	memset(smp->data, 0, sizeof(smp->data));
+	if (startpx == 0)
+		ipath_layer_get_pkeys(to_idev(ibdev)->ib_unit, p);
+	else
+		smp->status |= IB_SMP_INVALID_FIELD;
+
+	return reply(smp, __LINE__);
+}
+
+static inline int recv_subn_set_guidinfo(struct ib_smp *smp,
+					 struct ib_device *ibdev)
+{
+	/* The only GUID we support is the first read-only entry. */
+	return recv_subn_get_guidinfo(smp, ibdev);
+}
+
+static inline int recv_subn_set_portinfo(struct ib_smp *smp,
+					 struct ib_device *ibdev, u8 port)
+{
+	struct port_info *pip = (struct port_info *)smp->data;
+	uint32_t lportnum = be32_to_cpu(smp->attr_mod);
+	struct ib_event event;
+	struct ipath_ibdev *dev;
+	uint32_t flags;
+	char clientrereg = 0;
+	u32 tmp;
+	u32 tmp2;
+	int ret;
+
+	if (lportnum == 0) {
+		lportnum = port;
+		smp->attr_mod = cpu_to_be32(lportnum);
+	}
+
+	if (lportnum < 1 || lportnum > ibdev->phys_port_cnt)
+		return IB_MAD_RESULT_FAILURE;
+
+	dev = to_idev(ibdev);
+	event.device = ibdev;
+	event.element.port_num = port;
+
+	if (dev->mkey != pip->mkey)
+		dev->mkey = pip->mkey;
+
+	if (pip->gid_prefix != dev->gid_prefix)
+		dev->gid_prefix = pip->gid_prefix;
+
+	tmp = be16_to_cpu(pip->lid);
+	if (tmp != ipath_layer_get_lid(dev->ib_unit)) {
+		ipath_set_sps_lid(dev->ib_unit, tmp);
+		event.event = IB_EVENT_LID_CHANGE;
+		ib_dispatch_event(&event);
+	}
+
+	tmp = be16_to_cpu(pip->sm_lid);
+	if (tmp != dev->sm_lid) {
+		dev->sm_lid = tmp;
+		event.event = IB_EVENT_SM_CHANGE;
+		ib_dispatch_event(&event);
+	}
+
+	dev->mkey_lease_period = be16_to_cpu(pip->mkey_lease_period);
+
+#if 0
+	tmp = pip->link_width_enabled;
+	if (tmp && (tmp != lpp->linkwidthenabled)) {
+		lpp->linkwidthenabled = tmp;
+		/* JAG - notify driver here */
+	}
+#endif
+
+	tmp = pip->linkspeed_portstate & 0xF;
+	flags = ipath_layer_get_flags(dev->ib_unit);
+	if (flags & IPATH_LINKDOWN)
+		tmp2 = IB_PORT_DOWN;
+	else if (flags & IPATH_LINKINIT)
+		tmp2 = IB_PORT_INIT;
+	else if (flags & IPATH_LINKARMED)
+		tmp2 = IB_PORT_ARMED;
+	else if (flags & IPATH_LINKACTIVE)
+		tmp2 = IB_PORT_ACTIVE;
+	else
+		tmp2 = IB_PORT_NOP;
+	if (tmp && tmp != tmp2) {
+		switch (tmp) {
+		case IB_PORT_DOWN:
+		case IB_PORT_INIT:
+			ipath_kset_linkstate(dev->ib_unit << 16 |
+					     IPATH_IB_LINKDOWN);
+			if (tmp2 == IB_PORT_ACTIVE) {
+				event.event = IB_EVENT_PORT_ERR;
+				ib_dispatch_event(&event);
+			}
+			break;
+
+		case IB_PORT_ARMED:
+			ipath_kset_linkstate(dev->ib_unit << 16 |
+					     IPATH_IB_LINKARM);
+			if (tmp2 == IB_PORT_ACTIVE) {
+				event.event = IB_EVENT_PORT_ERR;
+				ib_dispatch_event(&event);
+			}
+			break;
+
+		case IB_PORT_ACTIVE:
+			ipath_kset_linkstate(dev->ib_unit << 16 |
+					     IPATH_IB_LINKACTIVE);
+			event.event = IB_EVENT_PORT_ACTIVE;
+			ib_dispatch_event(&event);
+			break;
+
+		default:
+			/* XXX We have already partially updated our state! */
+			return IB_MAD_RESULT_FAILURE;
+		}
+	}
+#if 0
+	tmp = BF_GET(g.madp, iba_Subn_PortInfo, FIELD_PortPhysicalState);
+	if (tmp && (tmp != lpp->portphysicalstate)) {
+		lpp->portphysicalstate = tmp;
+		/* JAG - notify driver here */
+	}
+
+	tmp = BF_GET(g.madp, iba_Subn_PortInfo, FIELD_LinkDownDefaultState);
+	if (tmp && (tmp != lpp->linkdowndefaultstate)) {
+		lpp->linkdowndefaultstate = tmp;
+		/* JAG - notify driver here */
+	}
+#endif
+
+	dev->mkeyprot_resv_lmc = pip->mkeyprot_resv_lmc;
+
+#if 0
+	tmp = BF_GET(g.madp, iba_Subn_PortInfo, FIELD_LinkSpeedEnabled);
+	if (tmp && (tmp != lpp->linkspeedenabled)) {
+		lpp->linkspeedenabled = tmp;
+		/* JAG - notify driver here */
+	}
+#endif
+
+	tmp = (pip->neighbormtu_mastersmsl >> 4) & 0xF;
+	if (tmp) {
+		switch (tmp) {
+		case IB_MTU_256:
+			tmp2 = 256;
+			break;
+		case IB_MTU_512:
+			tmp2 = 512;
+			break;
+		case IB_MTU_1024:
+			tmp2 = 1024;
+			break;
+		case IB_MTU_2048:
+			tmp2 = 2048;
+			break;
+		case IB_MTU_4096:
+			tmp2 = 4096;
+			break;
+		default:
+			/* XXX We have already partially updated our state! */
+			return IB_MAD_RESULT_FAILURE;
+		}
+
+		ipath_kset_mtu(dev->ib_unit << 16 | tmp2);
+	}
+
+	dev->sm_sl = pip->neighbormtu_mastersmsl & 0xF;
+
+#if 0
+	tmp = BF_GET(g.madp, iba_Subn_PortInfo, FIELD_VLHighLimit);
+	if (tmp != lpp->vlhighlimit) {
+		lpp->vlhighlimit = tmp;
+		/* JAG - notify driver here */
+	}
+
+	lpp->inittypereply =
+	    BF_GET(g.madp, iba_Subn_PortInfo, FIELD_InitTypeReply);
+
+	tmp = BF_GET(g.madp, iba_Subn_PortInfo, FIELD_OperationalVLs);
+	if (tmp && (tmp != lpp->operationalvls)) {
+		lpp->operationalvls = tmp;
+		/* JAG - notify driver here */
+	}
+#endif
+
+	if (pip->mkey_violations != 0)
+		dev->mkey_violations = 0;
+#if 0
+	/* XXX Hardware counter can't be reset. */
+	if (pip->pkey_violations != 0)
+		dev->pkey_violations = 0;
+#endif
+
+	if (pip->qkey_violations != 0)
+		dev->qkey_violations = 0;
+
+#if 0
+	tmp = BF_GET(g.madp, iba_Subn_PortInfo, FIELD_LocalPhyErrors);
+	if (tmp != lpp->localphyerrors) {
+		lpp->localphyerrors = tmp;
+		/* JAG - notify driver here */
+	}
+
+	tmp = BF_GET(g.madp, iba_Subn_PortInfo, FIELD_OverrunErrors);
+	if (tmp != lpp->overrunerrors) {
+		lpp->overrunerrors = tmp;
+		/* JAG - notify driver here */
+	}
+#endif
+
+	dev->subnet_timeout = pip->clientrereg_resv_subnetto & 0x1F;
+
+	if (pip->clientrereg_resv_subnetto & 0x80) {
+		clientrereg = 1;
+		event.event = IB_EVENT_LID_CHANGE;
+		ib_dispatch_event(&event);
+	}
+
+	ret = recv_subn_get_portinfo(smp, ibdev, port);
+
+	if (clientrereg)
+		pip->clientrereg_resv_subnetto |= 0x80;
+
+	return ret;
+}
+
+static inline int recv_subn_set_pkeytable(struct ib_smp *smp,
+					  struct ib_device *ibdev)
+{
+	u32 startpx = 32 * (be32_to_cpu(smp->attr_mod) & 0xffff);
+	u16 *p = (u16 *) smp->data;
+
+	if (startpx != 0 ||
+	    ipath_layer_set_pkeys(to_idev(ibdev)->ib_unit, p) != 0)
+		smp->status |= IB_SMP_INVALID_FIELD;
+
+	return recv_subn_get_pkeytable(smp, ibdev);
+}
+
+#define IB_PMA_CLASS_PORT_INFO		__constant_htons(0x0001)
+#define IB_PMA_PORT_SAMPLES_CONTROL	__constant_htons(0x0010)
+#define IB_PMA_PORT_SAMPLES_RESULT	__constant_htons(0x0011)
+#define IB_PMA_PORT_COUNTERS		__constant_htons(0x0012)
+#define IB_PMA_PORT_COUNTERS_EXT	__constant_htons(0x001D)
+#define IB_PMA_PORT_SAMPLES_RESULT_EXT	__constant_htons(0x001E)
+
+struct ib_perf {
+	u8 base_version;
+	u8 mgmt_class;
+	u8 class_version;
+	u8 method;
+	__be16 status;
+	__be16 unused;
+	__be64 tid;
+	__be16 attr_id;
+	__be16 resv;
+	__be32 attr_mod;
+	u8 reserved[40];
+	u8 data[192];
+} __attribute__ ((packed));
+
+struct ib_pma_classportinfo {
+	u8 base_version;
+	u8 class_version;
+	__be16 cap_mask;
+	u8 reserved[3];
+	u8 resp_time_value;	/* only lower 5 bits */
+	union ib_gid redirect_gid;
+	__be32 redirect_tc_sl_fl;	/* 8, 4, 20 bits respectively */
+	__be16 redirect_lid;
+	__be16 redirect_pkey;
+	__be32 redirect_qp;	/* only lower 24 bits */
+	__be32 redirect_qkey;
+	union ib_gid trap_gid;
+	__be32 trap_tc_sl_fl;	/* 8, 4, 20 bits respectively */
+	__be16 trap_lid;
+	__be16 trap_pkey;
+	__be32 trap_hl_qp;	/* 8, 24 bits respectively */
+	__be32 trap_qkey;
+} __attribute__ ((packed));
+
+struct ib_pma_portsamplescontrol {
+	u8 opcode;
+	u8 port_select;
+	u8 tick;
+	u8 counter_width;	/* only lower 3 bits */
+	__be32 counter_mask0_9;	/* 2, 10 * 3, bits */
+	__be16 counter_mask10_14;	/* 1, 5 * 3, bits */
+	u8 sample_mechanisms;
+	u8 sample_status;	/* only lower 2 bits */
+	__be64 option_mask;
+	__be64 vendor_mask;
+	__be32 sample_start;
+	__be32 sample_interval;
+	__be16 tag;
+	__be16 counter_select[15];
+} __attribute__ ((packed));
+
+struct ib_pma_portsamplesresult {
+	__be16 tag;
+	__be16 sample_status;	/* only lower 2 bits */
+	__be32 counter[15];
+} __attribute__ ((packed));
+
+struct ib_pma_portsamplesresult_ext {
+	__be16 tag;
+	__be16 sample_status;	/* only lower 2 bits */
+	__be32 extended_width;	/* only upper 2 bits */
+	__be64 counter[15];
+} __attribute__ ((packed));
+
+struct ib_pma_portcounters {
+	u8 reserved;
+	u8 port_select;
+	__be16 counter_select;
+	__be16 symbol_error_counter;
+	u8 link_error_recovery_counter;
+	u8 link_downed_counter;
+	__be16 port_rcv_errors;
+	__be16 port_rcv_remphys_errors;
+	__be16 port_rcv_switch_relay_errors;
+	__be16 port_xmit_discards;
+	u8 port_xmit_constraint_errors;
+	u8 port_rcv_constraint_errors;
+	u8 reserved1;
+	u8 lli_ebor_errors;	/* 4, 4, bits */
+	__be16 reserved2;
+	__be16 vl15_dropped;
+	__be32 port_xmit_data;
+	__be32 port_rcv_data;
+	__be32 port_xmit_packets;
+	__be32 port_rcv_packets;
+} __attribute__ ((packed));
+
+struct ib_pma_portcounters_ext {
+	u8 reserved;
+	u8 port_select;
+	__be16 counter_select;
+	__be32 reserved1;
+	__be64 port_xmit_data;
+	__be64 port_rcv_data;
+	__be64 port_xmit_packets;
+	__be64 port_rcv_packets;
+	__be64 port_unicast_xmit_packets;
+	__be64 port_unicast_rcv_packets;
+	__be64 port_multicast_xmit_packets;
+	__be64 port_multicast_rcv_packets;
+} __attribute__ ((packed));
+
+static int recv_pma_get_classportinfo(struct ib_perf *pmp)
+{
+	/*
+	   struct ib_pma_classportinfo *p =
+	   (struct ib_pma_classportinfo *)pmp->data;
+	 */
+
+	memset(pmp->data, 0, sizeof(pmp->data));
+
+	return reply((struct ib_smp *)pmp, __LINE__);
+}
+
+static int recv_pma_get_portsamplescontrol(struct ib_perf *pmp,
+					   struct ib_device *ibdev, u8 port)
+{
+	struct ib_pma_portsamplescontrol *p =
+	    (struct ib_pma_portsamplescontrol *)pmp->data;
+	struct ipath_ibdev *dev = to_idev(ibdev);
+	unsigned long flags;
+
+	memset(pmp->data, 0, sizeof(pmp->data));
+
+	p->port_select = port;
+	p->tick = 0xFA;		/* 1 ms. */
+	p->counter_width = 4;	/* 32 bit counters */
+	p->counter_mask0_9 = __constant_htonl(0x09248000); /* counters 0-4 */
+	spin_lock_irqsave(&dev->pending_lock, flags);
+	p->sample_status = dev->pma_sample_status;
+	p->sample_start = cpu_to_be32(dev->pma_sample_start);
+	p->sample_interval = cpu_to_be32(dev->pma_sample_interval);
+	p->tag = cpu_to_be16(dev->pma_tag);
+	p->counter_select[0] = dev->pma_counter_select[0];
+	p->counter_select[1] = dev->pma_counter_select[1];
+	p->counter_select[2] = dev->pma_counter_select[2];
+	p->counter_select[3] = dev->pma_counter_select[3];
+	p->counter_select[4] = dev->pma_counter_select[4];
+	spin_unlock_irqrestore(&dev->pending_lock, flags);
+
+	return reply((struct ib_smp *)pmp, __LINE__);
+}
+
+static int recv_pma_set_portsamplescontrol(struct ib_perf *pmp,
+					   struct ib_device *ibdev, u8 port)
+{
+	struct ib_pma_portsamplescontrol *p =
+	    (struct ib_pma_portsamplescontrol *)pmp->data;
+	struct ipath_ibdev *dev = to_idev(ibdev);
+	unsigned long flags;
+	u32 start = be32_to_cpu(p->sample_start);
+
+	if (pmp->attr_mod == 0 && p->port_select == port && start != 0) {
+		spin_lock_irqsave(&dev->pending_lock, flags);
+		if (dev->pma_sample_status == IB_PMA_SAMPLE_STATUS_DONE) {
+			dev->pma_sample_status = IB_PMA_SAMPLE_STATUS_STARTED;
+			dev->pma_sample_start = start;
+			dev->pma_sample_interval =
+			    be32_to_cpu(p->sample_interval);
+			dev->pma_tag = be16_to_cpu(p->tag);
+			if (p->counter_select[0])
+				dev->pma_counter_select[0] =
+				    p->counter_select[0];
+			if (p->counter_select[1])
+				dev->pma_counter_select[1] =
+				    p->counter_select[1];
+			if (p->counter_select[2])
+				dev->pma_counter_select[2] =
+				    p->counter_select[2];
+			if (p->counter_select[3])
+				dev->pma_counter_select[3] =
+				    p->counter_select[3];
+			if (p->counter_select[4])
+				dev->pma_counter_select[4] =
+				    p->counter_select[4];
+		}
+		spin_unlock_irqrestore(&dev->pending_lock, flags);
+	}
+	return recv_pma_get_portsamplescontrol(pmp, ibdev, port);
+}
+
+static u64 get_counter(struct ipath_ibdev *dev, __be16 sel)
+{
+	switch (sel) {
+	case IB_PMA_PORT_XMIT_DATA:
+		return dev->ipath_sword;
+	case IB_PMA_PORT_RCV_DATA:
+		return dev->ipath_rword;
+	case IB_PMA_PORT_XMIT_PKTS:
+		return dev->ipath_spkts;
+	case IB_PMA_PORT_RCV_PKTS:
+		return dev->ipath_rpkts;
+	case IB_PMA_PORT_XMIT_WAIT:
+	default:
+		return 0;
+	}
+}
+
+static int recv_pma_get_portsamplesresult(struct ib_perf *pmp,
+					  struct ib_device *ibdev)
+{
+	struct ib_pma_portsamplesresult *p =
+	    (struct ib_pma_portsamplesresult *)pmp->data;
+	struct ipath_ibdev *dev = to_idev(ibdev);
+	int i;
+
+	memset(pmp->data, 0, sizeof(pmp->data));
+	p->tag = cpu_to_be16(dev->pma_tag);
+	p->sample_status = cpu_to_be16(dev->pma_sample_status);
+	for (i = 0; i < ARRAY_SIZE(dev->pma_counter_select); i++)
+		p->counter[i] =
+		    cpu_to_be32(get_counter(dev, dev->pma_counter_select[i]));
+
+	return reply((struct ib_smp *)pmp, __LINE__);
+}
+
+static int recv_pma_get_portsamplesresult_ext(struct ib_perf *pmp,
+					      struct ib_device *ibdev)
+{
+	struct ib_pma_portsamplesresult_ext *p =
+	    (struct ib_pma_portsamplesresult_ext *)pmp->data;
+	struct ipath_ibdev *dev = to_idev(ibdev);
+	int i;
+
+	memset(pmp->data, 0, sizeof(pmp->data));
+	p->tag = cpu_to_be16(dev->pma_tag);
+	p->sample_status = cpu_to_be16(dev->pma_sample_status);
+	p->extended_width = __constant_cpu_to_be16(0x80000000); /* 64 bits */
+	for (i = 0; i < ARRAY_SIZE(dev->pma_counter_select); i++)
+		p->counter[i] =
+		    cpu_to_be64(get_counter(dev, dev->pma_counter_select[i]));
+
+	return reply((struct ib_smp *)pmp, __LINE__);
+}
+
+static int recv_pma_get_portcounters(struct ib_perf *pmp,
+				     struct ib_device *ibdev, u8 port)
+{
+	struct ib_pma_portcounters *p = (struct ib_pma_portcounters *)pmp->data;
+	struct ipath_layer_counters cntrs;
+
+	ipath_layer_get_counters(to_idev(ibdev)->ib_unit, &cntrs);
+
+	memset(pmp->data, 0, sizeof(pmp->data));
+	p->port_select = port;
+	if (cntrs.symbol_error_counter > 0xFFFFUL)
+		p->symbol_error_counter = 0xFFFF;
+	else
+		p->symbol_error_counter =
+			cpu_to_be16((u16)cntrs.symbol_error_counter);
+	if (cntrs.link_error_recovery_counter > 0xFFUL)
+		p->link_error_recovery_counter = 0xFF;
+	else
+		p->link_error_recovery_counter =
+			(u8)cntrs.link_error_recovery_counter;
+	if (cntrs.link_downed_counter > 0xFFUL)
+		p->link_downed_counter = 0xFF;
+	else
+		p->link_downed_counter = (u8)cntrs.link_downed_counter;
+	if (cntrs.port_rcv_errors > 0xFFFFUL)
+		p->port_rcv_errors = 0xFFFF;
+	else
+		p->port_rcv_errors = cpu_to_be16((u16)cntrs.port_rcv_errors);
+	if (cntrs.port_rcv_remphys_errors > 0xFFFFUL)
+		p->port_rcv_remphys_errors = 0xFFFF;
+	else
+		p->port_rcv_remphys_errors =
+			cpu_to_be16((u16)cntrs.port_rcv_remphys_errors);
+	if (cntrs.port_xmit_discards > 0xFFFFUL)
+		p->port_xmit_discards = 0xFFFF;
+	else
+		p->port_xmit_discards =
+			cpu_to_be16((u16)cntrs.port_xmit_discards);
+	if (cntrs.port_xmit_data > 0xFFFFFFFFUL)
+		p->port_xmit_data = 0xFFFFFFFF;
+	else
+		p->port_xmit_data = cpu_to_be32((u32)cntrs.port_xmit_data);
+	if (cntrs.port_rcv_data > 0xFFFFFFFFUL)
+		p->port_rcv_data = 0xFFFFFFFF;
+	else
+		p->port_rcv_data = cpu_to_be32((u32)cntrs.port_rcv_data);
+	if (cntrs.port_xmit_packets > 0xFFFFFFFFUL)
+		p->port_xmit_packets = 0xFFFFFFFF;
+	else
+		p->port_xmit_packets =
+			cpu_to_be32((u32)cntrs.port_xmit_packets);
+	if (cntrs.port_rcv_packets > 0xFFFFFFFFUL)
+		p->port_rcv_packets = 0xFFFFFFFF;
+	else
+		p->port_rcv_packets = cpu_to_be32((u32)cntrs.port_rcv_packets);
+
+	return reply((struct ib_smp *)pmp, __LINE__);
+}
+
+static int recv_pma_get_portcounters_ext(struct ib_perf *pmp,
+					 struct ib_device *ibdev, u8 port)
+{
+	struct ib_pma_portcounters_ext *p =
+		(struct ib_pma_portcounters_ext *)pmp->data;
+	struct ipath_ibdev *dev = to_idev(ibdev);
+	u64 swords, rwords, spkts, rpkts;
+
+	ipath_layer_snapshot_counters(to_idev(ibdev)->ib_unit,
+				      &swords, &rwords, &spkts, &rpkts);
+
+	memset(pmp->data, 0, sizeof(pmp->data));
+	p->port_select = port;
+	p->port_xmit_data = cpu_to_be64(swords);
+	p->port_rcv_data = cpu_to_be64(rwords);
+	p->port_xmit_packets = cpu_to_be64(spkts);
+	p->port_rcv_packets = cpu_to_be64(rpkts);
+	p->port_unicast_xmit_packets =
+		cpu_to_be64(spkts - dev->n_multicast_xmit);
+	p->port_unicast_rcv_packets =
+		cpu_to_be64(rpkts - dev->n_multicast_rcv);
+	p->port_multicast_xmit_packets = cpu_to_be64(dev->n_multicast_xmit);
+	p->port_multicast_rcv_packets = cpu_to_be64(dev->n_multicast_rcv);
+
+	return reply((struct ib_smp *)pmp, __LINE__);
+}
+
+static int recv_pma_set_portcounters(struct ib_perf *pmp,
+				     struct ib_device *ibdev, u8 port)
+{
+	/* XXX HW counters can't be cleared. */
+	return recv_pma_get_portcounters(pmp, ibdev, port);
+}
+
+static int recv_pma_set_portcounters_ext(struct ib_perf *pmp,
+					 struct ib_device *ibdev, u8 port)
+{
+	/* XXX HW counters can't be cleared. */
+	return recv_pma_get_portcounters_ext(pmp, ibdev, port);
+}
+
+static inline int process_subn(struct ib_device *ibdev, int mad_flags,
+			       u8 port_num, struct ib_mad *in_mad,
+			       struct ib_mad *out_mad)
+{
+	struct ib_smp *smp = (struct ib_smp *)out_mad;
+	struct ipath_ibdev *dev = to_idev(ibdev);
+
+	/* Is the mkey in the process of expiring? */
+	if (dev->mkey_lease_timeout && jiffies >= dev->mkey_lease_timeout) {
+		dev->mkey_lease_timeout = 0;
+		dev->mkeyprot_resv_lmc &= 0x3F;
+	}
+
+	/*
+	 * M_Key checking depends on
+	 * Portinfo:M_Key_protect_bits
+	 */
+	if ((mad_flags & IB_MAD_IGNORE_MKEY) == 0 && dev->mkey != 0 &&
+	    dev->mkey != smp->mkey && (smp->method != IB_MGMT_METHOD_GET ||
+	     (dev->mkeyprot_resv_lmc >> 7) != 0)) {
+		if (dev->mkey_violations != 0xFFFF)
+			++dev->mkey_violations;
+		if (dev->mkey_lease_timeout || dev->mkey_lease_period == 0)
+			return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
+		dev->mkey_lease_timeout = jiffies + dev->mkey_lease_period * HZ;
+		/* Future: Generate a trap notice. */
+		return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED;
+	}
+
+	*out_mad = *in_mad;
+	switch (smp->method) {
+	case IB_MGMT_METHOD_GET:
+		switch (smp->attr_id) {
+		case IB_SMP_ATTR_NODE_DESC:
+			return recv_subn_get_nodedescription(smp);
+
+		case IB_SMP_ATTR_NODE_INFO:
+			return recv_subn_get_nodeinfo(smp, ibdev, port_num);
+
+		case IB_SMP_ATTR_GUID_INFO:
+			return recv_subn_get_guidinfo(smp, ibdev);
+
+		case IB_SMP_ATTR_PORT_INFO:
+			return recv_subn_get_portinfo(smp, ibdev, port_num);
+
+		case IB_SMP_ATTR_PKEY_TABLE:
+			return recv_subn_get_pkeytable(smp, ibdev);
+
+		default:
+			break;
+		}
+		break;
+
+	case IB_MGMT_METHOD_SET:
+		switch (smp->attr_id) {
+		case IB_SMP_ATTR_GUID_INFO:
+			return recv_subn_set_guidinfo(smp, ibdev);
+
+		case IB_SMP_ATTR_PORT_INFO:
+			return recv_subn_set_portinfo(smp, ibdev, port_num);
+
+		case IB_SMP_ATTR_PKEY_TABLE:
+			return recv_subn_set_pkeytable(smp, ibdev);
+
+		default:
+			break;
+		}
+		break;
+
+	default:
+		break;
+	}
+	return IB_MAD_RESULT_FAILURE;
+}
+
+static inline int process_perf(struct ib_device *ibdev, u8 port_num,
+			       struct ib_mad *in_mad, struct ib_mad *out_mad)
+{
+	struct ib_perf *pmp = (struct ib_perf *)out_mad;
+
+	*out_mad = *in_mad;
+	switch (pmp->method) {
+	case IB_MGMT_METHOD_GET:
+		switch (pmp->attr_id) {
+		case IB_PMA_CLASS_PORT_INFO:
+			return recv_pma_get_classportinfo(pmp);
+
+		case IB_PMA_PORT_SAMPLES_CONTROL:
+			return recv_pma_get_portsamplescontrol(pmp, ibdev,
+							       port_num);
+
+		case IB_PMA_PORT_SAMPLES_RESULT:
+			return recv_pma_get_portsamplesresult(pmp, ibdev);
+
+		case IB_PMA_PORT_SAMPLES_RESULT_EXT:
+			return recv_pma_get_portsamplesresult_ext(pmp, ibdev);
+
+		case IB_PMA_PORT_COUNTERS:
+			return recv_pma_get_portcounters(pmp, ibdev, port_num);
+
+		case IB_PMA_PORT_COUNTERS_EXT:
+			return recv_pma_get_portcounters_ext(pmp, ibdev,
+							     port_num);
+
+		default:
+			break;
+		}
+		break;
+
+	case IB_MGMT_METHOD_SET:
+		switch (pmp->attr_id) {
+		case IB_PMA_PORT_SAMPLES_CONTROL:
+			return recv_pma_set_portsamplescontrol(pmp, ibdev,
+							       port_num);
+
+		case IB_PMA_PORT_COUNTERS:
+			return recv_pma_set_portcounters(pmp, ibdev, port_num);
+
+		case IB_PMA_PORT_COUNTERS_EXT:
+			return recv_pma_set_portcounters_ext(pmp, ibdev,
+							     port_num);
+
+		default:
+			break;
+		}
+		break;
+
+	default:
+		break;
+	}
+	return IB_MAD_RESULT_FAILURE;
+}
+
+/*
+ * Note that the verbs framework has already done the MAD sanity checks,
+ * and hop count/pointer updating for IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE MADs.
+ *
+ * Return IB_MAD_RESULT_SUCCESS if this is a MAD that we are not interested
+ * in processing.
+ */
+int ipath_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
+		      struct ib_wc *in_wc, struct ib_grh *in_grh,
+		      struct ib_mad *in_mad, struct ib_mad *out_mad)
+{
+	switch (in_mad->mad_hdr.mgmt_class) {
+	case IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE:
+	case IB_MGMT_CLASS_SUBN_LID_ROUTED:
+		return process_subn(ibdev, mad_flags, port_num,
+				    in_mad, out_mad);
+
+	case IB_MGMT_CLASS_PERF_MGMT:
+		return process_perf(ibdev, port_num, in_mad, out_mad);
+
+	default:
+		return IB_MAD_RESULT_SUCCESS;
+	}
+}
Signature · 1 line
0.99.9n
#14 lore raw

RFC PATCH 13/13 ipath Kconfig and Makefile

Kconfig and Makefile for ipath driver. (Leaving out changes to base drivers/infiniband/{Kconfig,Makefile} to hook these new files into kernel build)

---

 drivers/infiniband/hw/ipath/Kconfig  |   18 ++++++++++++++++++
 drivers/infiniband/hw/ipath/Makefile |   15 +++++++++++++++
 2 files changed, 33 insertions(+), 0 deletions(-)
 create mode 100644 drivers/infiniband/hw/ipath/Kconfig
 create mode 100644 drivers/infiniband/hw/ipath/Makefile

8748441795d589631fc58cbf477f485ff6716348

drivers/infiniband/hw/ipath/Kconfig new file +18 -0
new file mode 100644
index 0000000..092faa6
@@ -0,0 +1,18 @@
+config IPATH_CORE
+	tristate "PathScale InfiniPath Driver"
+	depends on PCI_MSI && X86_64
+	---help---
+	This is a low-level driver for PathScale InfiniPath host
+	channel adapters (HCAs) based on the HT-400 chip, including the
+	InfiniPath HT-460, the small form factor InfiniPath HT-460,
+	the InfiniPath HT-470 and the Linux Networx LS/X.
+
+config INFINIBAND_IPATH
+	tristate "PathScale InfiniPath Verbs Driver"
+	depends on IPATH_CORE && INFINIBAND
+	---help---
+	This is a driver that provides InfiniBand verbs support for
+	PathScale InfiniPath host channel adapters (HCAs).  This
+	allows these devices to be used with both kernel upper level
+	protocols such as IP-over-InfiniBand as well as with userspace
+	applications (in conjunction with InfiniBand userspace access).
drivers/infiniband/hw/ipath/Makefile new file +15 -0
new file mode 100644
index 0000000..dbe2557
@@ -0,0 +1,15 @@
+EXTRA_CFLAGS += -Idrivers/infiniband/include
+
+EXTRA_CFLAGS += -Wall -O3 -g3
+
+_ipath_idstr:="$$""Id: kernel.org InfiniPath Release 1.1 $$"" $$""Date: $(shell date +%F-%R)"" $$"
+EXTRA_CFLAGS += -D_IPATH_IDSTR='$(_ipath_idstr)' -DIPATH_KERN_TYPE=0
+
+obj-$(CONFIG_IPATH_CORE) += ipath_core.o
+obj-$(CONFIG_INFINIBAND_IPATH) += ib_ipath.o
+
+ipath_core-objs := ipath_copy.o ipath_driver.o \
+	ipath_dwordcpy.o ipath_ht400.o ipath_i2c.o ipath_layer.o \
+	ipath_lib.o ipath_mlock.o
+
+ib_ipath-objs := ipath_mad.o ipath_verbs.o
Signature · 1 line
0.99.9n
#15 lore raw

Re: [PATCH 13/13] [RFC] ipath Kconfig and Makefile

drivers/infiniband/hw/ipath/Kconfig line 3

Quoted 8 lines · Roland Dreier

...

drivers/infiniband/hw/ipath/Kconfig new file +3 -0
@@ -0,0 +1,18 @@
+config IPATH_CORE
+	tristate "PathScale InfiniPath Driver"
+	depends on PCI_MSI && X86_64
...

The driver shouldn't use assembler code and therefore no longer depend on X86_64.

drivers/infiniband/hw/ipath/Makefile line 3

Quoted 6 lines
drivers/infiniband/hw/ipath/Makefile new file +3 -0
@@ -0,0 +1,15 @@
+EXTRA_CFLAGS += -Idrivers/infiniband/include
+
+EXTRA_CFLAGS += -Wall -O3 -g3

-Wall is always set when compiling the kernel.

-O3 doesn't make much sense since the fight for producing the fastest code is between -O2 and -Os.

You don't want to always compile your driver with -g3.

drivers/infiniband/hw/ipath/Makefile line 6

Quoted 3 lines
drivers/infiniband/hw/ipath/Makefile new file +2 -0
+_ipath_idstr:="$$""Id: kernel.org InfiniPath Release 1.1 $$"" $$""Date: $(shell date +%F-%R)"" $$"
+EXTRA_CFLAGS += -D_IPATH_IDSTR='$(_ipath_idstr)' -DIPATH_KERN_TYPE=0
...

Please move the _IPATH_IDSTR revision tag to a header file and remove IPATH_KERN_TYPE.

cu
Adrian

Signature · 4 lines
       "Is there not promise of rain?" Ling Tan asked suddenly out
        of the darkness. There had been need of rain for many days.
       "Only a promise," Lao Er said.
                                       Pearl S. Buck - Dragon Seed
#16 lore raw

Re: [openib-general] Re: [PATCH 13/13] [RFC] ipath Kconfig and Makefile

Quoted 2 lines

The driver shouldn't use assembler code and therefore no longer depend on X86_64.

Agreed about the assembler, but one way or the other, x86_64 is the only arch we support.

Quoted 1 line

-Wall is always set when compiling the kernel.

Fine.

Quoted 2 lines

-O3 doesn't make much sense since the fight for producing the fastest code is between -O2 and -Os.

Makes many nanoseconds of difference to us for our latency numbers. At the low latency numbers we measuring (1.29us), this is a very important difference to our customers.

Quoted 1 line

You don't want to always compile your driver with -g3.

Good point. I'll ask around here why we're doing this.

Quoted 6 lines

drivers/infiniband/hw/ipath/Makefile line 6

Quoted 3 lines
drivers/infiniband/hw/ipath/Makefile new file +2 -0
+_ipath_idstr:="$$""Id: kernel.org InfiniPath Release 1.1 $$"" $$""Date: $(shell date +%F-%R)"" $$"
+EXTRA_CFLAGS += -D_IPATH_IDSTR='$(_ipath_idstr)' -DIPATH_KERN_TYPE=0
...

Please move the _IPATH_IDSTR revision tag to a header file and remove IPATH_KERN_TYPE.

I'll see what I can do.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#17 lore raw

Re: [openib-general] Re: [PATCH 13/13] [RFC] ipath Kconfig and Makefile

Quoted 6 lines · Robert Walsh
Quoted 2 lines

The driver shouldn't use assembler code and therefore no longer depend on X86_64.

Agreed about the assembler, but one way or the other, x86_64 is the only arch we support.
...

There's a difference between "technically supported by the driver" and "officially supported for our costumers":

It's fine if you tell the costumers buying your hardware "anything else than 64bit x86_64 kernels is completely unsupported", but for getting your driver included into the kernel it should be 32bit clean [1] and should also work for people using 32bit kernels on an Opteron.

Quoted 7 lines
Quoted 2 lines

-O3 doesn't make much sense since the fight for producing the fastest code is between -O2 and -Os.

Makes many nanoseconds of difference to us for our latency numbers. At the low latency numbers we measuring (1.29us), this is a very important difference to our customers.
...

There's no doubt that this is important for your customers.

What surprises me is that -O3 turned out to be the fastest flag for you.

Can you send numbers comparing -Os/-O2/-O3 (without -g3, preferable with gcc 4.0) including a description what and how you are measuring?

Quoted 2 lines

Regards,
Robert.

cu
Adrian

[1] not long ago, it used to be the other way round that drivers weren't

    64bit clean...
Signature · 4 lines
       "Is there not promise of rain?" Ling Tan asked suddenly out
        of the darkness. There had been need of rain for many days.
       "Only a promise," Lao Er said.
                                       Pearl S. Buck - Dragon Seed
#18 lore raw

Re: [openib-general] Re: [PATCH 13/13] [RFC] ipath Kconfig and Makefile

Quoted 7 lines

There's a difference between "technically supported by the driver" and "officially supported for our costumers":

It's fine if you tell the costumers buying your hardware "anything else than 64bit x86_64 kernels is completely unsupported", but for getting your driver included into the kernel it should be 32bit clean [1] and should also work for people using 32bit kernels on an Opteron.

Fair enough - I'll see what I can do.

Quoted 4 lines

What surprises me is that -O3 turned out to be the fastest flag for you.

Can you send numbers comparing -Os/-O2/-O3 (without -g3, preferable with gcc 4.0) including a description what and how you are measuring?

I'll try get around to this after my vacation and after we've had time to absorb and address all the other feedback we received.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#19 lore raw

Re: [openib-general] Re: [PATCH 13/13] [RFC] ipath Kconfig and Makefile

In reply to Robert Walsh
Quoted 2 lines · Robert Walsh

Agreed about the assembler, but one way or the other, x86_64 is the only arch we support.

If you need a quad only copy then put it into asm/string.h (asm/io.h if its operating on I/O space I guess) or somewhere similar as a generic function that does just that. That allows people to come along and provide the same functions for other platforms if they need it, and also makes it possible for others to use this feature if their hardware has the same feature.

Nobody expects you as a vendor to support it on sparc64, x86-32 or whatever, nor to write sparc64 asm functions just to make it possible for someone to do so cleanly later on.

#20 lore raw

Re: [PATCH 13/13] [RFC] ipath Kconfig and Makefile

In reply to Roland Dreier

drivers/infiniband/hw/ipath/Makefile line 1

Quoted 2 lines · Roland Dreier
drivers/infiniband/hw/ipath/Makefile new file +1 -0
@@ -0,0 +1,15 @@
+EXTRA_CFLAGS += -Idrivers/infiniband/include

If this is needed then some header files should be moved to include/rdma

drivers/infiniband/hw/ipath/Makefile line 15

Quoted 6 lines
drivers/infiniband/hw/ipath/Makefile new file +6 -0
+
+ipath_core-objs := ipath_copy.o ipath_driver.o \
+	ipath_dwordcpy.o ipath_ht400.o ipath_i2c.o ipath_layer.o \
+	ipath_lib.o ipath_mlock.o
+
+ib_ipath-objs := ipath_mad.o ipath_verbs.o

Please use:
ipath_core-y := ...
ib_ipath-y := ...

Use of -y let you do better Kconfig selection in the makefile, and is preferred compared to -objs

Sam

#21 lore raw

Re: [openib-general] Re: [PATCH 13/13] [RFC] ipath Kconfig and Makefile

Quoted 3 lines

drivers/infiniband/hw/ipath/Makefile line 1

Quoted 2 lines
drivers/infiniband/hw/ipath/Makefile new file +1 -0
@@ -0,0 +1,15 @@
+EXTRA_CFLAGS += -Idrivers/infiniband/include

If this is needed then some header files should be moved to include/rdma

Actually, this is done by other IB drivers, too, so I assumed it was OK. Roland, have you any comments on this?

Quoted 11 lines

drivers/infiniband/hw/ipath/Makefile line 15

Quoted 6 lines
drivers/infiniband/hw/ipath/Makefile new file +6 -0
+
+ipath_core-objs := ipath_copy.o ipath_driver.o \
+	ipath_dwordcpy.o ipath_ht400.o ipath_i2c.o ipath_layer.o \
+	ipath_lib.o ipath_mlock.o
+
+ib_ipath-objs := ipath_mad.o ipath_verbs.o

Please use:
ipath_core-y := ...
ib_ipath-y := ...

Quoted 2 lines

Use of -y let you do better Kconfig selection in the makefile, and is preferred compared to -objs

No problem.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043
#22 lore raw

Re: [PATCH 13/13] [RFC] ipath Kconfig and Makefile

In reply to Sam Ravnborg
Quoted 3 lines
Quoted 1 line

+EXTRA_CFLAGS += -Idrivers/infiniband/include

If this is needed then some header files should be moved to include/rdma

Sorry, this is really my fault -- it's a remnant to make building our subversion tree easier. It's not needed when the driver is part of the kernel proper, and I'll make sure to remove it when finally merging.

Thanks,
Roland

#23 lore raw

Re: [PATCH 10/13] [RFC] ipath verbs, part 1

In reply to Roland Dreier
Quoted 1 line · Roland Dreier

First half of ipath verbs driver

Some RCU-related questions interspersed. Basic question is "where is the lock-free read-side traversal?"

Thanx, Paul

Quoted 380 lines

---

 drivers/infiniband/hw/ipath/ipath_verbs.c | 3244 +++++++++++++++++++++++++++++
 1 files changed, 3244 insertions(+), 0 deletions(-)
 create mode 100644 drivers/infiniband/hw/ipath/ipath_verbs.c

72075ecec75f8c42e444a7d7d8ffcf340a845b96

drivers/infiniband/hw/ipath/ipath_verbs.c new file +367 -0
new file mode 100644
index 0000000..808326e
@@ -0,0 +1,3244 @@
+/*
+ * Copyright (c) 2005. PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Patent licenses, if any, provided herein do not apply to
+ * combinations of this program with other software, or any other
+ * product whatsoever.
+ *
+ * $Id: ipath_verbs.c 4491 2005-12-15 22:20:31Z rjwalsh $
+ */
+
+#include <linux/config.h>
+#include <linux/version.h>
+#include <linux/pci.h>
+#include <linux/err.h>
+#include <rdma/ib_pack.h>
+#include <rdma/ib_smi.h>
+#include <rdma/ib_mad.h>
+#include <rdma/ib_user_verbs.h>
+
+#include <asm/uaccess.h>
+#include <asm-generic/bug.h>
+
+#include "ipath_common.h"
+#include "ips_common.h"
+#include "ipath_layer.h"
+#include "ipath_verbs.h"
+
+/*
+ * Compare the lower 24 bits of the two values.
+ * Returns an integer <, ==, or > than zero.
+ */
+static inline int cmp24(u32 a, u32 b)
+{
+	return (((int) a) - ((int) b)) << 8;
+}
+
+#define MODNAME "ib_ipath"
+#define DRIVER_LOAD_MSG "PathScale " MODNAME " loaded: "
+#define PFX MODNAME ": "
+
+
+/* Not static, because we don't want the compiler removing it */
+const char ipath_verbs_version[] = "ipath_verbs " _IPATH_IDSTR;
+
+unsigned int ib_ipath_qp_table_size = 251;
+module_param(ib_ipath_qp_table_size, uint, 0444);
+MODULE_PARM_DESC(ib_ipath_qp_table_size, "QP table size");
+
+unsigned int ib_ipath_lkey_table_size = 12;
+module_param(ib_ipath_lkey_table_size, uint, 0444);
+MODULE_PARM_DESC(ib_ipath_lkey_table_size,
+		 "LKEY table size in bits (2^n, 1 <= n <= 23)");
+
+unsigned int ib_ipath_debug;	/* debug mask */
+module_param(ib_ipath_debug, uint, 0644);
+MODULE_PARM_DESC(ib_ipath_debug, "Verbs debug mask");
+
+
+static void ipath_ud_loopback(struct ipath_qp *sqp, struct ipath_sge_state *ss,
+			      u32 len, struct ib_send_wr *wr, struct ib_wc *wc);
+static void ipath_ruc_loopback(struct ipath_qp *sqp, struct ib_wc *wc);
+static int ipath_destroy_qp(struct ib_qp *ibqp);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("PathScale <[email protected]>");
+MODULE_DESCRIPTION("Pathscale InfiniPath driver");
+
+enum {
+	IPATH_FAULT_RC_DROP_SEND_F = 1,
+	IPATH_FAULT_RC_DROP_SEND_M,
+	IPATH_FAULT_RC_DROP_SEND_L,
+	IPATH_FAULT_RC_DROP_SEND_O,
+	IPATH_FAULT_RC_DROP_RDMA_WRITE_F,
+	IPATH_FAULT_RC_DROP_RDMA_WRITE_M,
+	IPATH_FAULT_RC_DROP_RDMA_WRITE_L,
+	IPATH_FAULT_RC_DROP_RDMA_WRITE_O,
+	IPATH_FAULT_RC_DROP_RDMA_READ_RESP_F,
+	IPATH_FAULT_RC_DROP_RDMA_READ_RESP_M,
+	IPATH_FAULT_RC_DROP_RDMA_READ_RESP_L,
+	IPATH_FAULT_RC_DROP_RDMA_READ_RESP_O,
+	IPATH_FAULT_RC_DROP_ACK,
+};
+
+enum {
+	IPATH_TRANS_INVALID = 0,
+	IPATH_TRANS_ANY2RST,
+	IPATH_TRANS_RST2INIT,
+	IPATH_TRANS_INIT2INIT,
+	IPATH_TRANS_INIT2RTR,
+	IPATH_TRANS_RTR2RTS,
+	IPATH_TRANS_RTS2RTS,
+	IPATH_TRANS_SQERR2RTS,
+	IPATH_TRANS_ANY2ERR,
+	IPATH_TRANS_RTS2SQD,	/* XXX Wait for expected ACKs & signal event */
+	IPATH_TRANS_SQD2SQD,	/* error if not drained & parameter change */
+	IPATH_TRANS_SQD2RTS,	/* error if not drained */
+};
+
+enum {
+	IPATH_POST_SEND_OK = 0x0001,
+	IPATH_POST_RECV_OK = 0x0002,
+	IPATH_PROCESS_RECV_OK = 0x0004,
+	IPATH_PROCESS_SEND_OK = 0x0008,
+};
+
+static int state_ops[IB_QPS_ERR + 1] = {
+	[IB_QPS_RESET] = 0,
+	[IB_QPS_INIT] = IPATH_POST_RECV_OK,
+	[IB_QPS_RTR] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK,
+	[IB_QPS_RTS] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK |
+	    IPATH_POST_SEND_OK | IPATH_PROCESS_SEND_OK,
+	[IB_QPS_SQD] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK |
+	    IPATH_POST_SEND_OK,
+	[IB_QPS_SQE] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK,
+	[IB_QPS_ERR] = 0,
+};
+
+/*
+ * Convert the AETH credit code into the number of credits.
+ */
+static u32 credit_table[31] = {
+	0,			/* 0 */
+	1,			/* 1 */
+	2,			/* 2 */
+	3,			/* 3 */
+	4,			/* 4 */
+	6,			/* 5 */
+	8,			/* 6 */
+	12,			/* 7 */
+	16,			/* 8 */
+	24,			/* 9 */
+	32,			/* A */
+	48,			/* B */
+	64,			/* C */
+	96,			/* D */
+	128,			/* E */
+	192,			/* F */
+	256,			/* 10 */
+	384,			/* 11 */
+	512,			/* 12 */
+	768,			/* 13 */
+	1024,			/* 14 */
+	1536,			/* 15 */
+	2048,			/* 16 */
+	3072,			/* 17 */
+	4096,			/* 18 */
+	6144,			/* 19 */
+	8192,			/* 1A */
+	12288,			/* 1B */
+	16384,			/* 1C */
+	24576,			/* 1D */
+	32768			/* 1E */
+};
+
+/*
+ * Convert the AETH RNR timeout code into the number of milliseconds.
+ */
+static u32 rnr_table[32] = {
+	656,			/* 0 */
+	1,			/* 1 */
+	1,			/* 2 */
+	1,			/* 3 */
+	1,			/* 4 */
+	1,			/* 5 */
+	1,			/* 6 */
+	1,			/* 7 */
+	1,			/* 8 */
+	1,			/* 9 */
+	1,			/* A */
+	1,			/* B */
+	1,			/* C */
+	1,			/* D */
+	2,			/* E */
+	2,			/* F */
+	3,			/* 10 */
+	4,			/* 11 */
+	6,			/* 12 */
+	8,			/* 13 */
+	11,			/* 14 */
+	16,			/* 15 */
+	21,			/* 16 */
+	31,			/* 17 */
+	41,			/* 18 */
+	62,			/* 19 */
+	82,			/* 1A */
+	123,			/* 1B */
+	164,			/* 1C */
+	246,			/* 1D */
+	328,			/* 1E */
+	492			/* 1F */
+};
+
+/*
+ * Translate ib_wr_opcode into ib_wc_opcode.
+ */
+static enum ib_wc_opcode wc_opcode[] = {
+	[IB_WR_RDMA_WRITE] = IB_WC_RDMA_WRITE,
+	[IB_WR_RDMA_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE,
+	[IB_WR_SEND] = IB_WC_SEND,
+	[IB_WR_SEND_WITH_IMM] = IB_WC_SEND,
+	[IB_WR_RDMA_READ] = IB_WC_RDMA_READ,
+	[IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP,
+	[IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD
+};
+
+/*
+ * Array of device pointers.
+ */
+static uint32_t number_of_devices;
+static struct ipath_ibdev **ipath_devices;
+
+/*
+ * Global table of GID to attached QPs.
+ * The table is global to all ipath devices since a send from one QP/device
+ * needs to be locally routed to any locally attached QPs on the same
+ * or different device.
+ */
+static struct rb_root mcast_tree;
+static spinlock_t mcast_lock = SPIN_LOCK_UNLOCKED;
+
+/*
+ * Allocate a structure to link a QP to the multicast GID structure.
+ */
+static struct ipath_mcast_qp *ipath_mcast_qp_alloc(struct ipath_qp *qp)
+{
+	struct ipath_mcast_qp *mqp;
+
+	mqp = kmalloc(sizeof(*mqp), GFP_KERNEL);
+	if (!mqp)
+		return NULL;
+
+	mqp->qp = qp;
+	atomic_inc(&qp->refcount);
+
+	return mqp;
+}
+
+static void ipath_mcast_qp_free(struct ipath_mcast_qp *mqp)
+{
+	struct ipath_qp *qp = mqp->qp;
+
+	/* Notify ipath_destroy_qp() if it is waiting. */
+	if (atomic_dec_and_test(&qp->refcount))
+		wake_up(&qp->wait);
+
+	kfree(mqp);
+}
+
+/*
+ * Allocate a structure for the multicast GID.
+ * A list of QPs will be attached to this structure.
+ */
+static struct ipath_mcast *ipath_mcast_alloc(union ib_gid *mgid)
+{
+	struct ipath_mcast *mcast;
+
+	mcast = kmalloc(sizeof(*mcast), GFP_KERNEL);
+	if (!mcast)
+		return NULL;
+
+	mcast->mgid = *mgid;
+	INIT_LIST_HEAD(&mcast->qp_list);
+	init_waitqueue_head(&mcast->wait);
+	atomic_set(&mcast->refcount, 0);
+
+	return mcast;
+}
+
+static void ipath_mcast_free(struct ipath_mcast *mcast)
+{
+	struct ipath_mcast_qp *p, *tmp;
+
+	list_for_each_entry_safe(p, tmp, &mcast->qp_list, list)
+		ipath_mcast_qp_free(p);
+
+	kfree(mcast);
+}
+
+/*
+ * Search the global table for the given multicast GID.
+ * Return it or NULL if not found.
+ * The caller is responsible for decrementing the reference count if found.
+ */
+static struct ipath_mcast *ipath_mcast_find(union ib_gid *mgid)
+{
+	struct rb_node *n;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mcast_lock, flags);
+	n = mcast_tree.rb_node;
+	while (n) {
+		struct ipath_mcast *mcast;
+		int ret;
+
+		mcast = rb_entry(n, struct ipath_mcast, rb_node);
+
+		ret = memcmp(mgid->raw, mcast->mgid.raw, sizeof(union ib_gid));
+		if (ret < 0)
+			n = n->rb_left;
+		else if (ret > 0)
+			n = n->rb_right;
+		else {
+			atomic_inc(&mcast->refcount);
+			spin_unlock_irqrestore(&mcast_lock, flags);
+			return mcast;
+		}
+	}
+	spin_unlock_irqrestore(&mcast_lock, flags);
+
+	return NULL;
+}
+
+/*
+ * Insert the multicast GID into the table and
+ * attach the QP structure.
+ * Return zero if both were added.
+ * Return EEXIST if the GID was already in the table but the QP was added.
+ * Return ESRCH if the QP was already attached and neither structure was added.
+ */
+static int ipath_mcast_add(struct ipath_mcast *mcast,
+			   struct ipath_mcast_qp *mqp)
+{
+	struct rb_node **n = &mcast_tree.rb_node;
+	struct rb_node *pn = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mcast_lock, flags);
+
+	while (*n) {
+		struct ipath_mcast *tmcast;
+		struct ipath_mcast_qp *p;
+		int ret;
+
+		pn = *n;
+		tmcast = rb_entry(pn, struct ipath_mcast, rb_node);
+
+		ret = memcmp(mcast->mgid.raw, tmcast->mgid.raw,
+			     sizeof(union ib_gid));
+		if (ret < 0) {

drivers/infiniband/hw/ipath/ipath_verbs.c line 377

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_verbs.c new file +10 -0
+			n = &pn->rb_left;
+			continue;
+		}
+		if (ret > 0) {
+			n = &pn->rb_right;
+			continue;
+		}
+
+		/* Search the QP list to see if this is already there. */
+		list_for_each_entry_rcu(p, &tmcast->qp_list, list) {

Given that we hold the global mcast_lock, how is RCU helping here?

Is there a lock-free read-side traversal path somewhere that I am missing?

drivers/infiniband/hw/ipath/ipath_verbs.c line 383

Quoted 6 lines
drivers/infiniband/hw/ipath/ipath_verbs.c new file +6 -0
+			if (p->qp == mqp->qp) {
+				spin_unlock_irqrestore(&mcast_lock, flags);
+				return ESRCH;
+			}
+		}
+		list_add_tail_rcu(&mqp->list, &tmcast->qp_list);

Ditto...

drivers/infiniband/hw/ipath/ipath_verbs.c line 388

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_verbs.c new file +5 -0
+		spin_unlock_irqrestore(&mcast_lock, flags);
+		return EEXIST;
+	}
+
+	list_add_tail_rcu(&mqp->list, &mcast->qp_list);

Ditto...

Quoted 76 lines
drivers/infiniband/hw/ipath/ipath_verbs.c new file +76 -0
+		spin_unlock_irqrestore(&mcast_lock, flags);
+
+	atomic_inc(&mcast->refcount);
+	rb_link_node(&mcast->rb_node, pn, n);
+	rb_insert_color(&mcast->rb_node, &mcast_tree);
+
+	spin_unlock_irqrestore(&mcast_lock, flags);
+
+	return 0;
+}
+
+static int ipath_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid,
+				  u16 lid)
+{
+	struct ipath_qp *qp = to_iqp(ibqp);
+	struct ipath_mcast *mcast;
+	struct ipath_mcast_qp *mqp;
+
+	/*
+	 * Allocate data structures since its better to do this outside of
+	 * spin locks and it will most likely be needed.
+	 */
+	mcast = ipath_mcast_alloc(gid);
+	if (mcast == NULL)
+		return -ENOMEM;
+	mqp = ipath_mcast_qp_alloc(qp);
+	if (mqp == NULL) {
+		ipath_mcast_free(mcast);
+		return -ENOMEM;
+	}
+	switch (ipath_mcast_add(mcast, mqp)) {
+	case ESRCH:
+		/* Neither was used: can't attach the same QP twice. */
+		ipath_mcast_qp_free(mqp);
+		ipath_mcast_free(mcast);
+		return -EINVAL;
+	case EEXIST:		/* The mcast wasn't used */
+		ipath_mcast_free(mcast);
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int ipath_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid,
+				  u16 lid)
+{
+	struct ipath_qp *qp = to_iqp(ibqp);
+	struct ipath_mcast *mcast = NULL;
+	struct ipath_mcast_qp *p, *tmp;
+	struct rb_node *n;
+	unsigned long flags;
+	int last = 0;
+
+	spin_lock_irqsave(&mcast_lock, flags);
+
+	/* Find the GID in the mcast table. */
+	n = mcast_tree.rb_node;
+	while (1) {
+		int ret;
+
+		if (n == NULL) {
+			spin_unlock_irqrestore(&mcast_lock, flags);
+			return 0;
+		}
+
+		mcast = rb_entry(n, struct ipath_mcast, rb_node);
+		ret = memcmp(gid->raw, mcast->mgid.raw, sizeof(union ib_gid));
+		if (ret < 0)
+			n = n->rb_left;
+		else if (ret > 0)
+			n = n->rb_right;
+		else
+			break;
+	}

drivers/infiniband/hw/ipath/ipath_verbs.c line 473

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_verbs.c new file +10 -0
+
+	/* Search the QP list. */
+	list_for_each_entry_safe(p, tmp, &mcast->qp_list, list) {
+		if (p->qp != qp)
+			continue;
+		/*
+		 * We found it, so remove it, but don't poison the forward link
+		 * until we are sure there are no list walkers.
+		 */
+		list_del_rcu(&p->list);

Ditto...

Quoted 2,782 lines
drivers/infiniband/hw/ipath/ipath_verbs.c new file +2772 -0
+		spin_unlock_irqrestore(&mcast_lock, flags);
+
+		/* If this was the last attached QP, remove the GID too. */
+		if (list_empty(&mcast->qp_list)) {
+			rb_erase(&mcast->rb_node, &mcast_tree);
+			last = 1;
+		}
+		break;
+	}
+
+	spin_unlock_irqrestore(&mcast_lock, flags);
+
+	if (p) {
+		/*
+		 * Wait for any list walkers to finish before freeing the
+		 * list element.
+		 */
+		wait_event(mcast->wait, atomic_read(&mcast->refcount) <= 1);
+		ipath_mcast_qp_free(p);
+	}
+	if (last) {
+		atomic_dec(&mcast->refcount);
+		wait_event(mcast->wait, !atomic_read(&mcast->refcount));
+		ipath_mcast_free(mcast);
+	}
+
+	return 0;
+}
+
+/*
+ * Copy data to SGE memory.
+ */
+static void copy_sge(struct ipath_sge_state *ss, void *data, u32 length)
+{
+	struct ipath_sge *sge = &ss->sge;
+
+	while (length) {
+		u32 len = sge->length;
+
+		BUG_ON(len == 0);
+		if (len > length)
+			len = length;
+		memcpy(sge->vaddr, data, len);
+		sge->vaddr += len;
+		sge->length -= len;
+		sge->sge_length -= len;
+		if (sge->sge_length == 0) {
+			if (--ss->num_sge)
+				*sge = *ss->sg_list++;
+		} else if (sge->length == 0 && sge->mr != NULL) {
+			if (++sge->n >= IPATH_SEGSZ) {
+				if (++sge->m >= sge->mr->mapsz)
+					break;
+				sge->n = 0;
+			}
+			sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
+			sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
+		}
+		data += len;
+		length -= len;
+	}
+}
+
+/*
+ * Skip over length bytes of SGE memory.
+ */
+static void skip_sge(struct ipath_sge_state *ss, u32 length)
+{
+	struct ipath_sge *sge = &ss->sge;
+
+	while (length > sge->sge_length) {
+		length -= sge->sge_length;
+		ss->sge = *ss->sg_list++;
+	}
+	while (length) {
+		u32 len = sge->length;
+
+		BUG_ON(len == 0);
+		if (len > length)
+			len = length;
+		sge->vaddr += len;
+		sge->length -= len;
+		sge->sge_length -= len;
+		if (sge->sge_length == 0) {
+			if (--ss->num_sge)
+				*sge = *ss->sg_list++;
+		} else if (sge->length == 0 && sge->mr != NULL) {
+			if (++sge->n >= IPATH_SEGSZ) {
+				if (++sge->m >= sge->mr->mapsz)
+					break;
+				sge->n = 0;
+			}
+			sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
+			sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
+		}
+		length -= len;
+	}
+}
+
+static inline u32 alloc_qpn(struct ipath_qp_table *qpt)
+{
+	u32 i, offset, max_scan, qpn;
+	struct qpn_map *map;
+
+	qpn = qpt->last + 1;
+	if (qpn >= QPN_MAX)
+		qpn = 2;
+	offset = qpn & BITS_PER_PAGE_MASK;
+	map = &qpt->map[qpn / BITS_PER_PAGE];
+	max_scan = qpt->nmaps - !offset;
+	for (i = 0;;) {
+		if (unlikely(!map->page)) {
+			unsigned long page = get_zeroed_page(GFP_KERNEL);
+			unsigned long flags;
+
+			/*
+			 * Free the page if someone raced with us
+			 * installing it:
+			 */
+			spin_lock_irqsave(&qpt->lock, flags);
+			if (map->page)
+				free_page(page);
+			else
+				map->page = (void *)page;
+			spin_unlock_irqrestore(&qpt->lock, flags);
+			if (unlikely(!map->page))
+				break;
+		}
+		if (likely(atomic_read(&map->n_free))) {
+			do {
+				if (!test_and_set_bit(offset, map->page)) {
+					atomic_dec(&map->n_free);
+					qpt->last = qpn;
+					return qpn;
+				}
+				offset = find_next_offset(map, offset);
+				qpn = mk_qpn(qpt, map, offset);
+				/*
+				 * This test differs from alloc_pidmap().
+				 * If find_next_offset() does find a zero bit,
+				 * we don't need to check for QPN wrapping
+				 * around past our starting QPN.  We
+				 * just need to be sure we don't loop forever.
+				 */
+			} while (offset < BITS_PER_PAGE && qpn < QPN_MAX);
+		}
+		/*
+		 * In order to keep the number of pages allocated to a minimum,
+		 * we scan the all existing pages before increasing the size
+		 * of the bitmap table.
+		 */
+		if (++i > max_scan) {
+			if (qpt->nmaps == QPNMAP_ENTRIES)
+				break;
+			map = &qpt->map[qpt->nmaps++];
+			offset = 0;
+		} else if (map < &qpt->map[qpt->nmaps]) {
+			++map;
+			offset = 0;
+		} else {
+			map = &qpt->map[0];
+			offset = 2;
+		}
+		qpn = mk_qpn(qpt, map, offset);
+	}
+	return 0;
+}
+
+static inline void free_qpn(struct ipath_qp_table *qpt, u32 qpn)
+{
+	struct qpn_map *map;
+
+	map = qpt->map + qpn / BITS_PER_PAGE;
+	if (map->page)
+		clear_bit(qpn & BITS_PER_PAGE_MASK, map->page);
+	atomic_inc(&map->n_free);
+}
+
+/*
+ * Allocate the next available QPN and put the QP into the hash table.
+ * The hash table holds a reference to the QP.
+ */
+static int ipath_alloc_qpn(struct ipath_qp_table *qpt, struct ipath_qp *qp,
+			   enum ib_qp_type type)
+{
+	unsigned long flags;
+	u32 qpn;
+
+	if (type == IB_QPT_SMI)
+		qpn = 0;
+	else if (type == IB_QPT_GSI)
+		qpn = 1;
+	else {
+		/* Allocate the next available QPN */
+		qpn = alloc_qpn(qpt);
+		if (qpn == 0) {
+			return -ENOMEM;
+		}
+	}
+	qp->ibqp.qp_num = qpn;
+
+	/* Add the QP to the hash table. */
+	spin_lock_irqsave(&qpt->lock, flags);
+
+	qpn %= qpt->max;
+	qp->next = qpt->table[qpn];
+	qpt->table[qpn] = qp;
+	atomic_inc(&qp->refcount);
+
+	spin_unlock_irqrestore(&qpt->lock, flags);
+	return 0;
+}
+
+/*
+ * Remove the QP from the table so it can't be found asynchronously by
+ * the receive interrupt routine.
+ */
+static void ipath_free_qp(struct ipath_qp_table *qpt, struct ipath_qp *qp)
+{
+	struct ipath_qp *q, **qpp;
+	unsigned long flags;
+	int fnd = 0;
+
+	spin_lock_irqsave(&qpt->lock, flags);
+
+	/* Remove QP from the hash table. */
+	qpp = &qpt->table[qp->ibqp.qp_num % qpt->max];
+	for (; (q = *qpp) != NULL; qpp = &q->next) {
+		if (q == qp) {
+			*qpp = qp->next;
+			qp->next = NULL;
+			atomic_dec(&qp->refcount);
+			fnd = 1;
+			break;
+		}
+	}
+
+	spin_unlock_irqrestore(&qpt->lock, flags);
+
+	if (!fnd)
+		return;
+
+	/* If QPN is not reserved, mark QPN free in the bitmap. */
+	if (qp->ibqp.qp_num > 1)
+		free_qpn(qpt, qp->ibqp.qp_num);
+
+	wait_event(qp->wait, !atomic_read(&qp->refcount));
+}
+
+/*
+ * Remove all QPs from the table.
+ */
+static void ipath_free_all_qps(struct ipath_qp_table *qpt)
+{
+	unsigned long flags;
+	struct ipath_qp *qp, *nqp;
+	u32 n;
+
+	for (n = 0; n < qpt->max; n++) {
+		spin_lock_irqsave(&qpt->lock, flags);
+		qp = qpt->table[n];
+		qpt->table[n] = NULL;
+		spin_unlock_irqrestore(&qpt->lock, flags);
+
+		while (qp) {
+			nqp = qp->next;
+			if (qp->ibqp.qp_num > 1)
+				free_qpn(qpt, qp->ibqp.qp_num);
+			if (!atomic_dec_and_test(&qp->refcount) ||
+			    !ipath_destroy_qp(&qp->ibqp))
+				_VERBS_INFO("QP memory leak!\n");
+			qp = nqp;
+		}
+	}
+
+	for (n = 0; n < ARRAY_SIZE(qpt->map); n++) {
+		if (qpt->map[n].page)
+			free_page((unsigned long)qpt->map[n].page);
+	}
+}
+
+/*
+ * Return the QP with the given QPN.
+ * The caller is responsible for decrementing the QP reference count when done.
+ */
+static struct ipath_qp *ipath_lookup_qpn(struct ipath_qp_table *qpt, u32 qpn)
+{
+	unsigned long flags;
+	struct ipath_qp *qp;
+
+	spin_lock_irqsave(&qpt->lock, flags);
+
+	for (qp = qpt->table[qpn % qpt->max]; qp; qp = qp->next) {
+		if (qp->ibqp.qp_num == qpn) {
+			atomic_inc(&qp->refcount);
+			break;
+		}
+	}
+
+	spin_unlock_irqrestore(&qpt->lock, flags);
+	return qp;
+}
+
+static int ipath_alloc_lkey(struct ipath_lkey_table *rkt,
+			    struct ipath_mregion *mr)
+{
+	unsigned long flags;
+	u32 r;
+	u32 n;
+
+	spin_lock_irqsave(&rkt->lock, flags);
+
+	/* Find the next available LKEY */
+	r = n = rkt->next;
+	for (;;) {
+		if (rkt->table[r] == NULL)
+			break;
+		r = (r + 1) & (rkt->max - 1);
+		if (r == n) {
+			spin_unlock_irqrestore(&rkt->lock, flags);
+			_VERBS_INFO("LKEY table full\n");
+			return 0;
+		}
+	}
+	rkt->next = (r + 1) & (rkt->max - 1);
+	/*
+	 * Make sure lkey is never zero which is reserved to indicate an
+	 * unrestricted LKEY.
+	 */
+	rkt->gen++;
+	mr->lkey = (r << (32 - ib_ipath_lkey_table_size)) |
+	    ((((1 << (24 - ib_ipath_lkey_table_size)) - 1) & rkt->gen) << 8);
+	if (mr->lkey == 0) {
+		mr->lkey |= 1 << 8;
+		rkt->gen++;
+	}
+	rkt->table[r] = mr;
+	spin_unlock_irqrestore(&rkt->lock, flags);
+
+	return 1;
+}
+
+static void ipath_free_lkey(struct ipath_lkey_table *rkt, u32 lkey)
+{
+	unsigned long flags;
+	u32 r;
+
+	if (lkey == 0)
+		return;
+	r = lkey >> (32 - ib_ipath_lkey_table_size);
+	spin_lock_irqsave(&rkt->lock, flags);
+	rkt->table[r] = NULL;
+	spin_unlock_irqrestore(&rkt->lock, flags);
+}
+
+/*
+ * Check the IB SGE for validity and initialize our internal version of it.
+ * Return 1 if OK, else zero.
+ */
+static int ipath_lkey_ok(struct ipath_lkey_table *rkt, struct ipath_sge *isge,
+			 struct ib_sge *sge, int acc)
+{
+	struct ipath_mregion *mr;
+	size_t off;
+
+	/*
+	 * We use LKEY == zero to mean a physical kmalloc() address.
+	 * This is a bit of a hack since we rely on dma_map_single()
+	 * being reversible by calling bus_to_virt().
+	 */
+	if (sge->lkey == 0) {
+		isge->mr = NULL;
+		isge->vaddr = bus_to_virt(sge->addr);
+		isge->length = sge->length;
+		isge->sge_length = sge->length;
+		return 1;
+	}
+	spin_lock(&rkt->lock);
+	mr = rkt->table[(sge->lkey >> (32 - ib_ipath_lkey_table_size))];
+	spin_unlock(&rkt->lock);
+	if (unlikely(mr == NULL || mr->lkey != sge->lkey))
+		return 0;
+
+	off = sge->addr - mr->user_base;
+	if (unlikely(sge->addr < mr->user_base ||
+		     off + sge->length > mr->length ||
+		     (mr->access_flags & acc) != acc))
+		return 0;
+
+	off += mr->offset;
+	isge->mr = mr;
+	isge->m = 0;
+	isge->n = 0;
+	while (off >= mr->map[isge->m]->segs[isge->n].length) {
+		off -= mr->map[isge->m]->segs[isge->n].length;
+		if (++isge->n >= IPATH_SEGSZ) {
+			isge->m++;
+			isge->n = 0;
+		}
+	}
+	isge->vaddr = mr->map[isge->m]->segs[isge->n].vaddr + off;
+	isge->length = mr->map[isge->m]->segs[isge->n].length - off;
+	isge->sge_length = sge->length;
+	return 1;
+}
+
+/*
+ * Initialize the qp->s_sge after a restart.
+ * The QP s_lock should be held.
+ */
+static void ipath_init_restart(struct ipath_qp *qp, struct ipath_swqe *wqe)
+{
+	struct ipath_ibdev *dev;
+	u32 len;
+
+	len = ((qp->s_psn - wqe->psn) & 0xFFFFFF) *
+	    ib_mtu_enum_to_int(qp->path_mtu);
+	qp->s_sge.sge = wqe->sg_list[0];
+	qp->s_sge.sg_list = wqe->sg_list + 1;
+	qp->s_sge.num_sge = wqe->wr.num_sge;
+	skip_sge(&qp->s_sge, len);
+	qp->s_len = wqe->length - len;
+	dev = to_idev(qp->ibqp.device);
+	spin_lock(&dev->pending_lock);
+	if (qp->timerwait.next == LIST_POISON1)
+		list_add_tail(&qp->timerwait,
+			      &dev->pending[dev->pending_index]);
+	spin_unlock(&dev->pending_lock);
+}
+
+/*
+ * Check the IB virtual address, length, and RKEY.
+ * Return 1 if OK, else zero.
+ * The QP r_rq.lock should be held.
+ */
+static int ipath_rkey_ok(struct ipath_ibdev *dev, struct ipath_sge_state *ss,
+			 u32 len, u64 vaddr, u32 rkey, int acc)
+{
+	struct ipath_lkey_table *rkt = &dev->lk_table;
+	struct ipath_sge *sge = &ss->sge;
+	struct ipath_mregion *mr;
+	size_t off;
+
+	spin_lock(&rkt->lock);
+	mr = rkt->table[(rkey >> (32 - ib_ipath_lkey_table_size))];
+	spin_unlock(&rkt->lock);
+	if (unlikely(mr == NULL || mr->lkey != rkey))
+		return 0;
+
+	off = vaddr - mr->iova;
+	if (unlikely(vaddr < mr->iova || off + len > mr->length ||
+		     (mr->access_flags & acc) == 0))
+		return 0;
+
+	off += mr->offset;
+	sge->mr = mr;
+	sge->m = 0;
+	sge->n = 0;
+	while (off >= mr->map[sge->m]->segs[sge->n].length) {
+		off -= mr->map[sge->m]->segs[sge->n].length;
+		if (++sge->n >= IPATH_SEGSZ) {
+			sge->m++;
+			sge->n = 0;
+		}
+	}
+	sge->vaddr = mr->map[sge->m]->segs[sge->n].vaddr + off;
+	sge->length = mr->map[sge->m]->segs[sge->n].length - off;
+	sge->sge_length = len;
+	ss->sg_list = NULL;
+	ss->num_sge = 1;
+	return 1;
+}
+
+/*
+ * Add a new entry to the completion queue.
+ * This may be called with one of the qp->s_lock or qp->r_rq.lock held.
+ */
+static void ipath_cq_enter(struct ipath_cq *cq, struct ib_wc *entry, int sig)
+{
+	unsigned long flags;
+	u32 next;
+
+	spin_lock_irqsave(&cq->lock, flags);
+
+	cq->queue[cq->head] = *entry;
+	next = cq->head + 1;
+	if (next == cq->ibcq.cqe)
+		next = 0;
+	if (next != cq->tail)
+		cq->head = next;
+	else {
+		/* XXX - need to mark current wr as having an error... */
+	}
+
+	if (cq->notify == IB_CQ_NEXT_COMP ||
+	    (cq->notify == IB_CQ_SOLICITED && sig)) {
+		cq->notify = IB_CQ_NONE;
+		cq->triggered++;
+		/*
+		 * This will cause send_complete() to be called in
+		 * another thread.
+		 */
+		tasklet_schedule(&cq->comptask);
+	}
+
+	spin_unlock_irqrestore(&cq->lock, flags);
+
+	if (entry->status != IB_WC_SUCCESS)
+		to_idev(cq->ibcq.device)->n_wqe_errs++;
+}
+
+static void send_complete(unsigned long data)
+{
+	struct ipath_cq *cq = (struct ipath_cq *)data;
+
+	/*
+	 * The completion handler will most likely rearm the notification
+	 * and poll for all pending entries.  If a new completion entry
+	 * is added while we are in this routine, tasklet_schedule()
+	 * won't call us again until we return so we check triggered to
+	 * see if we need to call the handler again.
+	 */
+	for (;;) {
+		u8 triggered = cq->triggered;
+
+		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
+
+		if (cq->triggered == triggered)
+			return;
+	}
+}
+
+/*
+ * This is the QP state transition table.
+ * See ipath_modify_qp() for details.
+ */
+static const struct {
+	int trans;
+	u32 req_param[IB_QPT_RAW_IPV6];
+	u32 opt_param[IB_QPT_RAW_IPV6];
+} qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
+	[IB_QPS_RESET] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_INIT] = {
+			.trans = IPATH_TRANS_RST2INIT,
+			.req_param = {
+				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_ACCESS_FLAGS),
+				[IB_QPT_RC] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_ACCESS_FLAGS),
+			},
+		},
+	},
+	[IB_QPS_INIT] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_INIT] = {
+			.trans = IPATH_TRANS_INIT2INIT,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_ACCESS_FLAGS),
+				[IB_QPT_RC] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_ACCESS_FLAGS),
+			}
+		},
+		[IB_QPS_RTR] = {
+			.trans = IPATH_TRANS_INIT2RTR,
+			.req_param = {
+				[IB_QPT_UC] = (IB_QP_AV |
+					 IB_QP_PATH_MTU |
+					 IB_QP_DEST_QPN |
+					 IB_QP_RQ_PSN),
+				[IB_QPT_RC] = (IB_QP_AV |
+					 IB_QP_PATH_MTU |
+					 IB_QP_DEST_QPN |
+					 IB_QP_RQ_PSN |
+					 IB_QP_MAX_DEST_RD_ATOMIC |
+					 IB_QP_MIN_RNR_TIMER),
+			},
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX),
+				[IB_QPT_RC] = (IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX),
+			}
+		}
+	},
+	[IB_QPS_RTR] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_RTS] = {
+			.trans = IPATH_TRANS_RTR2RTS,
+			.req_param = {
+				[IB_QPT_SMI] = IB_QP_SQ_PSN,
+				[IB_QPT_GSI] = IB_QP_SQ_PSN,
+				[IB_QPT_UD] = IB_QP_SQ_PSN,
+				[IB_QPT_UC] = IB_QP_SQ_PSN,
+				[IB_QPT_RC] = (IB_QP_TIMEOUT |
+					 IB_QP_RETRY_CNT |
+					 IB_QP_RNR_RETRY |
+					 IB_QP_SQ_PSN |
+					 IB_QP_MAX_QP_RD_ATOMIC),
+			},
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX |
+					 IB_QP_PATH_MIG_STATE),
+				[IB_QPT_RC] = (IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX |
+					 IB_QP_MIN_RNR_TIMER |
+					 IB_QP_PATH_MIG_STATE),
+			}
+		}
+	},
+	[IB_QPS_RTS] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_RTS] = {
+			.trans = IPATH_TRANS_RTS2RTS,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_ACCESS_FLAGS |
+					 IB_QP_ALT_PATH |
+					 IB_QP_PATH_MIG_STATE),
+				[IB_QPT_RC] = (IB_QP_ACCESS_FLAGS |
+					 IB_QP_ALT_PATH |
+					 IB_QP_PATH_MIG_STATE |
+					 IB_QP_MIN_RNR_TIMER),
+			}
+		},
+		[IB_QPS_SQD] = {
+			.trans = IPATH_TRANS_RTS2SQD,
+		},
+	},
+	[IB_QPS_SQD] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_RTS] = {
+			.trans = IPATH_TRANS_SQD2RTS,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PATH_MIG_STATE),
+				[IB_QPT_RC] = (IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_MIN_RNR_TIMER |
+					 IB_QP_PATH_MIG_STATE),
+			}
+		},
+		[IB_QPS_SQD] = {
+			.trans = IPATH_TRANS_SQD2SQD,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_PKEY_INDEX | IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_AV |
+					 IB_QP_TIMEOUT |
+					 IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX |
+					 IB_QP_PATH_MIG_STATE),
+				[IB_QPT_RC] = (IB_QP_AV |
+					 IB_QP_TIMEOUT |
+					 IB_QP_RETRY_CNT |
+					 IB_QP_RNR_RETRY |
+					 IB_QP_MAX_QP_RD_ATOMIC |
+					 IB_QP_MAX_DEST_RD_ATOMIC |
+					 IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX |
+					 IB_QP_MIN_RNR_TIMER |
+					 IB_QP_PATH_MIG_STATE),
+			}
+		}
+	},
+	[IB_QPS_SQE] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_RTS] = {
+			.trans = IPATH_TRANS_SQERR2RTS,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UC] = IB_QP_CUR_STATE,
+				[IB_QPT_RC] = (IB_QP_CUR_STATE |
+					 IB_QP_MIN_RNR_TIMER),
+			}
+		}
+	},
+	[IB_QPS_ERR] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR }
+	}
+};
+
+/*
+ * Initialize the QP state to the reset state.
+ */
+static void ipath_reset_qp(struct ipath_qp *qp)
+{
+	qp->remote_qpn = 0;
+	qp->qkey = 0;
+	qp->qp_access_flags = 0;
+	qp->s_hdrwords = 0;
+	qp->s_psn = 0;
+	qp->r_psn = 0;
+	atomic_set(&qp->msn, 0);
+	if (qp->ibqp.qp_type == IB_QPT_RC) {
+		qp->s_state = IB_OPCODE_RC_SEND_LAST;
+		qp->r_state = IB_OPCODE_RC_SEND_LAST;
+	} else {
+		qp->s_state = IB_OPCODE_UC_SEND_LAST;
+		qp->r_state = IB_OPCODE_UC_SEND_LAST;
+	}
+	qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+	qp->s_nak_state = 0;
+	qp->s_rnr_timeout = 0;
+	qp->s_head = 0;
+	qp->s_tail = 0;
+	qp->s_cur = 0;
+	qp->s_last = 0;
+	qp->s_ssn = 1;
+	qp->s_lsn = 0;
+	qp->r_rq.head = 0;
+	qp->r_rq.tail = 0;
+	qp->r_reuse_sge = 0;
+}
+
+/*
+ * Flush send work queue.
+ * The QP s_lock should be held.
+ */
+static void ipath_sqerror_qp(struct ipath_qp *qp, struct ib_wc *wc)
+{
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
+
+	_VERBS_INFO("Send queue error on QP%d/%d: err: %d\n",
+		    qp->ibqp.qp_num, qp->remote_qpn, wc->status);
+
+	spin_lock(&dev->pending_lock);
+	/* XXX What if its already removed by the timeout code? */
+	if (qp->timerwait.next != LIST_POISON1)
+		list_del(&qp->timerwait);
+	if (qp->piowait.next != LIST_POISON1)
+		list_del(&qp->piowait);
+	spin_unlock(&dev->pending_lock);
+
+	ipath_cq_enter(to_icq(qp->ibqp.send_cq), wc, 1);
+	if (++qp->s_last >= qp->s_size)
+		qp->s_last = 0;
+
+	wc->status = IB_WC_WR_FLUSH_ERR;
+
+	while (qp->s_last != qp->s_head) {
+		wc->wr_id = wqe->wr.wr_id;
+		wc->opcode = wc_opcode[wqe->wr.opcode];
+		ipath_cq_enter(to_icq(qp->ibqp.send_cq), wc, 1);
+		if (++qp->s_last >= qp->s_size)
+			qp->s_last = 0;
+		wqe = get_swqe_ptr(qp, qp->s_last);
+	}
+	qp->s_cur = qp->s_tail = qp->s_head;
+	qp->state = IB_QPS_SQE;
+}
+
+/*
+ * Flush both send and receive work queues.
+ * QP r_rq.lock and s_lock should be held.
+ */
+static void ipath_error_qp(struct ipath_qp *qp)
+{
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ib_wc wc;
+
+	_VERBS_INFO("QP%d/%d in error state\n",
+		    qp->ibqp.qp_num, qp->remote_qpn);
+
+	spin_lock(&dev->pending_lock);
+	/* XXX What if its already removed by the timeout code? */
+	if (qp->timerwait.next != LIST_POISON1)
+		list_del(&qp->timerwait);
+	if (qp->piowait.next != LIST_POISON1)
+		list_del(&qp->piowait);
+	spin_unlock(&dev->pending_lock);
+
+	wc.status = IB_WC_WR_FLUSH_ERR;
+	wc.vendor_err = 0;
+	wc.byte_len = 0;
+	wc.imm_data = 0;
+	wc.qp_num = qp->ibqp.qp_num;
+	wc.src_qp = 0;
+	wc.wc_flags = 0;
+	wc.pkey_index = 0;
+	wc.slid = 0;
+	wc.sl = 0;
+	wc.dlid_path_bits = 0;
+	wc.port_num = 0;
+
+	while (qp->s_last != qp->s_head) {
+		struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
+
+		wc.wr_id = wqe->wr.wr_id;
+		wc.opcode = wc_opcode[wqe->wr.opcode];
+		if (++qp->s_last >= qp->s_size)
+			qp->s_last = 0;
+		ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 1);
+	}
+	qp->s_cur = qp->s_tail = qp->s_head;
+	qp->s_hdrwords = 0;
+	qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+
+	wc.opcode = IB_WC_RECV;
+	while (qp->r_rq.tail != qp->r_rq.head) {
+		wc.wr_id = get_rwqe_ptr(&qp->r_rq, qp->r_rq.tail)->wr_id;
+		if (++qp->r_rq.tail >= qp->r_rq.size)
+			qp->r_rq.tail = 0;
+		ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
+	}
+}
+
+static int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+			   int attr_mask)
+{
+	struct ipath_qp *qp = to_iqp(ibqp);
+	enum ib_qp_state cur_state, new_state;
+	u32 req_param, opt_param;
+	unsigned long flags;
+
+	if (attr_mask & IB_QP_CUR_STATE) {
+		cur_state = attr->cur_qp_state;
+		if (cur_state != IB_QPS_RTR &&
+		    cur_state != IB_QPS_RTS &&
+		    cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
+			return -EINVAL;
+		spin_lock_irqsave(&qp->r_rq.lock, flags);
+		spin_lock(&qp->s_lock);
+	} else {
+		spin_lock_irqsave(&qp->r_rq.lock, flags);
+		spin_lock(&qp->s_lock);
+		cur_state = qp->state;
+	}
+
+	if (attr_mask & IB_QP_STATE) {
+		new_state = attr->qp_state;
+		if (new_state < 0 || new_state > IB_QPS_ERR)
+			goto inval;
+	} else
+		new_state = cur_state;
+
+	switch (qp_state_table[cur_state][new_state].trans) {
+	case IPATH_TRANS_INVALID:
+		goto inval;
+
+	case IPATH_TRANS_ANY2RST:
+		ipath_reset_qp(qp);
+		break;
+
+	case IPATH_TRANS_ANY2ERR:
+		ipath_error_qp(qp);
+		break;
+
+	}
+
+	req_param =
+	    qp_state_table[cur_state][new_state].req_param[qp->ibqp.qp_type];
+	opt_param =
+	    qp_state_table[cur_state][new_state].opt_param[qp->ibqp.qp_type];
+
+	if ((req_param & attr_mask) != req_param)
+		goto inval;
+
+	if (attr_mask & ~(req_param | opt_param | IB_QP_STATE))
+		goto inval;
+
+	if (attr_mask & IB_QP_PKEY_INDEX) {
+		struct ipath_ibdev *dev = to_idev(ibqp->device);
+
+		if (attr->pkey_index >= ipath_layer_get_npkeys(dev->ib_unit))
+			goto inval;
+		qp->s_pkey_index = attr->pkey_index;
+	}
+
+	if (attr_mask & IB_QP_DEST_QPN)
+		qp->remote_qpn = attr->dest_qp_num;
+
+	if (attr_mask & IB_QP_SQ_PSN) {
+		qp->s_next_psn = attr->sq_psn;
+		qp->s_last_psn = qp->s_next_psn - 1;
+	}
+
+	if (attr_mask & IB_QP_RQ_PSN)
+		qp->r_psn = attr->rq_psn;
+
+	if (attr_mask & IB_QP_ACCESS_FLAGS)
+		qp->qp_access_flags = attr->qp_access_flags;
+
+	if (attr_mask & IB_QP_AV)
+		qp->remote_ah_attr = attr->ah_attr;
+
+	if (attr_mask & IB_QP_PATH_MTU)
+		qp->path_mtu = attr->path_mtu;
+
+	if (attr_mask & IB_QP_RETRY_CNT)
+		qp->s_retry = qp->s_retry_cnt = attr->retry_cnt;
+
+	if (attr_mask & IB_QP_RNR_RETRY) {
+		qp->s_rnr_retry = attr->rnr_retry;
+		if (qp->s_rnr_retry > 7)
+			qp->s_rnr_retry = 7;
+		qp->s_rnr_retry_cnt = qp->s_rnr_retry;
+	}
+
+	if (attr_mask & IB_QP_MIN_RNR_TIMER)
+		qp->s_min_rnr_timer = attr->min_rnr_timer & 0x1F;
+
+	if (attr_mask & IB_QP_QKEY)
+		qp->qkey = attr->qkey;
+
+	if (attr_mask & IB_QP_PKEY_INDEX)
+		qp->s_pkey_index = attr->pkey_index;
+
+	qp->state = new_state;
+	spin_unlock(&qp->s_lock);
+	spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+
+	/*
+	 * Try to move to ARMED if QP1 changed to the RTS state.
+	 */
+	if (qp->ibqp.qp_num == 1 && new_state == IB_QPS_RTS) {
+		struct ipath_ibdev *dev = to_idev(ibqp->device);
+
+		/*
+		 * Bounce the link even if it was active so the SM will
+		 * reinitialize the SMA's state.
+		 */
+		ipath_kset_linkstate((dev->ib_unit << 16) | IPATH_IB_LINKDOWN);
+		ipath_kset_linkstate((dev->ib_unit << 16) | IPATH_IB_LINKARM);
+	}
+	return 0;
+
+inval:
+	spin_unlock(&qp->s_lock);
+	spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+	return -EINVAL;
+}
+
+/*
+ * Compute the AETH (syndrome + MSN).
+ * The QP s_lock should be held.
+ */
+static u32 ipath_compute_aeth(struct ipath_qp *qp)
+{
+	u32 aeth = atomic_read(&qp->msn) & 0xFFFFFF;
+
+	if (qp->s_nak_state) {
+		aeth |= qp->s_nak_state << 24;
+	} else if (qp->ibqp.srq) {
+		/* Shared receive queues don't generate credits. */
+		aeth |= 0x1F << 24;
+	} else {
+		u32 min, max, x;
+		u32 credits;
+
+		/*
+		 * Compute the number of credits available (RWQEs).
+		 * XXX Not holding the r_rq.lock here so there is a small
+		 * chance that the pair of reads are not atomic.
+		 */
+		credits = qp->r_rq.head - qp->r_rq.tail;
+		if ((int)credits < 0)
+			credits += qp->r_rq.size;
+		/* Binary search the credit table to find the code to use. */
+		min = 0;
+		max = 31;
+		for (;;) {
+			x = (min + max) / 2;
+			if (credit_table[x] == credits)
+				break;
+			if (credit_table[x] > credits)
+				max = x;
+			else if (min == x)
+				break;
+			else
+				min = x;
+		}
+		aeth |= x << 24;
+	}
+	return cpu_to_be32(aeth);
+}
+
+
+static void no_bufs_available(struct ipath_qp *qp, struct ipath_ibdev *dev)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->pending_lock, flags);
+	if (qp->piowait.next == LIST_POISON1)
+		list_add_tail(&qp->piowait, &dev->piowait);
+	spin_unlock_irqrestore(&dev->pending_lock, flags);
+	/*
+	 * Note that as soon as ipath_layer_want_buffer() is called and
+	 * possibly before it returns, ipath_ib_piobufavail()
+	 * could be called.  If we are still in the tasklet function,
+	 * tasklet_schedule() will not call us until the next time
+	 * tasklet_schedule() is called.
+	 * We clear the tasklet flag now since we are committing to return
+	 * from the tasklet function.
+	 */
+	tasklet_unlock(&qp->s_task);
+	ipath_layer_want_buffer(dev->ib_unit);
+	dev->n_piowait++;
+}
+
+/*
+ * Process entries in the send work queue until the queue is exhausted.
+ * Only allow one CPU to send a packet per QP (tasklet).
+ * Otherwise, after we drop the QP lock, two threads could send
+ * packets out of order.
+ * This is similar to do_rc_send() below except we don't have timeouts or
+ * resends.
+ */
+static void do_uc_send(unsigned long data)
+{
+	struct ipath_qp *qp = (struct ipath_qp *)data;
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ipath_swqe *wqe;
+	unsigned long flags;
+	u16 lrh0;
+	u32 hwords;
+	u32 nwords;
+	u32 extra_bytes;
+	u32 bth0;
+	u32 bth2;
+	u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);
+	u32 len;
+	struct ipath_other_headers *ohdr;
+	struct ib_wc wc;
+
+	if (test_and_set_bit(IPATH_S_BUSY, &qp->s_flags))
+		return;
+
+	if (unlikely(qp->remote_ah_attr.dlid ==
+		     ipath_layer_get_lid(dev->ib_unit))) {
+		/* Pass in an uninitialized ib_wc to save stack space. */
+		ipath_ruc_loopback(qp, &wc);
+		clear_bit(IPATH_S_BUSY, &qp->s_flags);
+		return;
+	}
+
+	ohdr = &qp->s_hdr.u.oth;
+	if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
+		ohdr = &qp->s_hdr.u.l.oth;
+
+again:
+	/* Check for a constructed packet to be sent. */
+	if (qp->s_hdrwords != 0) {
+			/*
+			 * If no PIO bufs are available, return.
+			 * An interrupt will call ipath_ib_piobufavail()
+			 * when one is available.
+			 */
+			if (ipath_verbs_send(dev->ib_unit, qp->s_hdrwords,
+					     (uint32_t *) &qp->s_hdr,
+					     qp->s_cur_size, qp->s_cur_sge)) {
+				no_bufs_available(qp, dev);
+				return;
+			}
+		/* Record that we sent the packet and s_hdr is empty. */
+		qp->s_hdrwords = 0;
+	}
+
+	lrh0 = IPS_LRH_BTH;
+	/* header size in 32-bit words LRH+BTH = (8+12)/4. */
+	hwords = 5;
+
+	/*
+	 * The lock is needed to synchronize between
+	 * setting qp->s_ack_state and post_send().
+	 */
+	spin_lock_irqsave(&qp->s_lock, flags);
+
+	if (!(state_ops[qp->state] & IPATH_PROCESS_SEND_OK))
+		goto done;
+
+	bth0 = ipath_layer_get_pkey(dev->ib_unit, qp->s_pkey_index);
+
+	/* Send a request. */
+	wqe = get_swqe_ptr(qp, qp->s_last);
+	switch (qp->s_state) {
+	default:
+		/* Signal the completion of the last send (if there is one). */
+		if (qp->s_last != qp->s_tail) {
+			if (++qp->s_last == qp->s_size)
+				qp->s_last = 0;
+			if (!test_bit(IPATH_S_SIGNAL_REQ_WR, &qp->s_flags) ||
+			    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
+				wc.wr_id = wqe->wr.wr_id;
+				wc.status = IB_WC_SUCCESS;
+				wc.opcode = wc_opcode[wqe->wr.opcode];
+				wc.vendor_err = 0;
+				wc.byte_len = wqe->length;
+				wc.qp_num = qp->ibqp.qp_num;
+				wc.src_qp = qp->remote_qpn;
+				wc.pkey_index = 0;
+				wc.slid = qp->remote_ah_attr.dlid;
+				wc.sl = qp->remote_ah_attr.sl;
+				wc.dlid_path_bits = 0;
+				wc.port_num = 0;
+				ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc,
+					       0);
+			}
+			wqe = get_swqe_ptr(qp, qp->s_last);
+		}
+		/* Check if send work queue is empty. */
+		if (qp->s_tail == qp->s_head)
+			goto done;
+		/*
+		 * Start a new request.
+		 */
+		qp->s_psn = wqe->psn = qp->s_next_psn;
+		qp->s_sge.sge = wqe->sg_list[0];
+		qp->s_sge.sg_list = wqe->sg_list + 1;
+		qp->s_sge.num_sge = wqe->wr.num_sge;
+		qp->s_len = len = wqe->length;
+		switch (wqe->wr.opcode) {
+		case IB_WR_SEND:
+		case IB_WR_SEND_WITH_IMM:
+			if (len > pmtu) {
+				qp->s_state = IB_OPCODE_UC_SEND_FIRST;
+				len = pmtu;
+				break;
+			}
+			if (wqe->wr.opcode == IB_WR_SEND) {
+				qp->s_state = IB_OPCODE_UC_SEND_ONLY;
+			} else {
+				qp->s_state =
+				    IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE;
+				/* Immediate data comes after the BTH */
+				ohdr->u.imm_data = wqe->wr.imm_data;
+				hwords += 1;
+			}
+			if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+				bth0 |= 1 << 23;
+			break;
+
+		case IB_WR_RDMA_WRITE:
+		case IB_WR_RDMA_WRITE_WITH_IMM:
+			ohdr->u.rc.reth.vaddr =
+			    cpu_to_be64(wqe->wr.wr.rdma.remote_addr);
+			ohdr->u.rc.reth.rkey =
+			    cpu_to_be32(wqe->wr.wr.rdma.rkey);
+			ohdr->u.rc.reth.length = cpu_to_be32(len);
+			hwords += sizeof(struct ib_reth) / 4;
+			if (len > pmtu) {
+				qp->s_state = IB_OPCODE_UC_RDMA_WRITE_FIRST;
+				len = pmtu;
+				break;
+			}
+			if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
+				qp->s_state = IB_OPCODE_UC_RDMA_WRITE_ONLY;
+			} else {
+				qp->s_state =
+				    IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE;
+				/* Immediate data comes after the RETH */
+				ohdr->u.rc.imm_data = wqe->wr.imm_data;
+				hwords += 1;
+				if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+					bth0 |= 1 << 23;
+			}
+			break;
+
+		default:
+			goto done;
+		}
+		if (++qp->s_tail >= qp->s_size)
+			qp->s_tail = 0;
+		break;
+
+	case IB_OPCODE_UC_SEND_FIRST:
+		qp->s_state = IB_OPCODE_UC_SEND_MIDDLE;
+		/* FALLTHROUGH */
+	case IB_OPCODE_UC_SEND_MIDDLE:
+		len = qp->s_len;
+		if (len > pmtu) {
+			len = pmtu;
+			break;
+		}
+		if (wqe->wr.opcode == IB_WR_SEND)
+			qp->s_state = IB_OPCODE_UC_SEND_LAST;
+		else {
+			qp->s_state = IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE;
+			/* Immediate data comes after the BTH */
+			ohdr->u.imm_data = wqe->wr.imm_data;
+			hwords += 1;
+		}
+		if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+			bth0 |= 1 << 23;
+		break;
+
+	case IB_OPCODE_UC_RDMA_WRITE_FIRST:
+		qp->s_state = IB_OPCODE_UC_RDMA_WRITE_MIDDLE;
+		/* FALLTHROUGH */
+	case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:
+		len = qp->s_len;
+		if (len > pmtu) {
+			len = pmtu;
+			break;
+		}
+		if (wqe->wr.opcode == IB_WR_RDMA_WRITE)
+			qp->s_state = IB_OPCODE_UC_RDMA_WRITE_LAST;
+		else {
+			qp->s_state =
+			    IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE;
+			/* Immediate data comes after the BTH */
+			ohdr->u.imm_data = wqe->wr.imm_data;
+			hwords += 1;
+			if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+				bth0 |= 1 << 23;
+		}
+		break;
+	}
+	bth2 = qp->s_next_psn++ & 0xFFFFFF;
+	qp->s_len -= len;
+	bth0 |= qp->s_state << 24;
+
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+
+	/* Construct the header. */
+	extra_bytes = (4 - len) & 3;
+	nwords = (len + extra_bytes) >> 2;
+	if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
+		/* Header size in 32-bit words. */
+		hwords += 10;
+		lrh0 = IPS_LRH_GRH;
+		qp->s_hdr.u.l.grh.version_tclass_flow =
+		    cpu_to_be32((6 << 28) |
+				(qp->remote_ah_attr.grh.traffic_class << 20) |
+				qp->remote_ah_attr.grh.flow_label);
+		qp->s_hdr.u.l.grh.paylen =
+		    cpu_to_be16(((hwords - 12) + nwords + SIZE_OF_CRC) << 2);
+		qp->s_hdr.u.l.grh.next_hdr = 0x1B;
+		qp->s_hdr.u.l.grh.hop_limit = qp->remote_ah_attr.grh.hop_limit;
+		/* The SGID is 32-bit aligned. */
+		qp->s_hdr.u.l.grh.sgid.global.subnet_prefix = dev->gid_prefix;
+		qp->s_hdr.u.l.grh.sgid.global.interface_id =
+		    ipath_layer_get_guid(dev->ib_unit);
+		qp->s_hdr.u.l.grh.dgid = qp->remote_ah_attr.grh.dgid;
+	}
+	qp->s_hdrwords = hwords;
+	qp->s_cur_sge = &qp->s_sge;
+	qp->s_cur_size = len;
+	lrh0 |= qp->remote_ah_attr.sl << 4;
+	qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
+	/* DEST LID */
+	qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
+	qp->s_hdr.lrh[2] = cpu_to_be16(hwords + nwords + SIZE_OF_CRC);
+	qp->s_hdr.lrh[3] = cpu_to_be16(ipath_layer_get_lid(dev->ib_unit));
+	bth0 |= extra_bytes << 20;
+	ohdr->bth[0] = cpu_to_be32(bth0);
+	ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
+	ohdr->bth[2] = cpu_to_be32(bth2);
+
+	/* Check for more work to do. */
+	goto again;
+
+done:
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+	clear_bit(IPATH_S_BUSY, &qp->s_flags);
+}
+
+/*
+ * Process entries in the send work queue until credit or queue is exhausted.
+ * Only allow one CPU to send a packet per QP (tasklet).
+ * Otherwise, after we drop the QP s_lock, two threads could send
+ * packets out of order.
+ */
+static void do_rc_send(unsigned long data)
+{
+	struct ipath_qp *qp = (struct ipath_qp *)data;
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ipath_swqe *wqe;
+	struct ipath_sge_state *ss;
+	unsigned long flags;
+	u16 lrh0;
+	u32 hwords;
+	u32 nwords;
+	u32 extra_bytes;
+	u32 bth0;
+	u32 bth2;
+	u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);
+	u32 len;
+	struct ipath_other_headers *ohdr;
+	char newreq;
+
+	if (test_and_set_bit(IPATH_S_BUSY, &qp->s_flags))
+		return;
+
+	if (unlikely(qp->remote_ah_attr.dlid ==
+		     ipath_layer_get_lid(dev->ib_unit))) {
+		struct ib_wc wc;
+
+		/*
+		 * Pass in an uninitialized ib_wc to be consistent with
+		 * other places where ipath_ruc_loopback() is called.
+		 */
+		ipath_ruc_loopback(qp, &wc);
+		clear_bit(IPATH_S_BUSY, &qp->s_flags);
+		return;
+	}
+
+	ohdr = &qp->s_hdr.u.oth;
+	if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
+		ohdr = &qp->s_hdr.u.l.oth;
+
+again:
+	/* Check for a constructed packet to be sent. */
+	if (qp->s_hdrwords != 0) {
+			/*
+			 * If no PIO bufs are available, return.
+			 * An interrupt will call ipath_ib_piobufavail()
+			 * when one is available.
+			 */
+			if (ipath_verbs_send(dev->ib_unit, qp->s_hdrwords,
+					     (uint32_t *) &qp->s_hdr,
+					     qp->s_cur_size, qp->s_cur_sge)) {
+				no_bufs_available(qp, dev);
+				return;
+			}
+		/* Record that we sent the packet and s_hdr is empty. */
+		qp->s_hdrwords = 0;
+	}
+
+	lrh0 = IPS_LRH_BTH;
+	/* header size in 32-bit words LRH+BTH = (8+12)/4. */
+	hwords = 5;
+
+	/*
+	 * The lock is needed to synchronize between
+	 * setting qp->s_ack_state, resend timer, and post_send().
+	 */
+	spin_lock_irqsave(&qp->s_lock, flags);
+
+	bth0 = ipath_layer_get_pkey(dev->ib_unit, qp->s_pkey_index);
+
+	/* Sending responses has higher priority over sending requests. */
+	if (qp->s_ack_state != IB_OPCODE_RC_ACKNOWLEDGE) {
+		/*
+		 * Send a response.
+		 * Note that we are in the responder's side of the QP context.
+		 */
+		switch (qp->s_ack_state) {
+		case IB_OPCODE_RC_RDMA_READ_REQUEST:
+			ss = &qp->s_rdma_sge;
+			len = qp->s_rdma_len;
+			if (len > pmtu) {
+				len = pmtu;
+				qp->s_ack_state =
+				    IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST;
+			} else {
+				qp->s_ack_state =
+				    IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY;
+			}
+			qp->s_rdma_len -= len;
+			bth0 |= qp->s_ack_state << 24;
+			ohdr->u.aeth = ipath_compute_aeth(qp);
+			hwords++;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST:
+			qp->s_ack_state =
+			    IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE;
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:
+			ss = &qp->s_rdma_sge;
+			len = qp->s_rdma_len;
+			if (len > pmtu) {
+				len = pmtu;
+			} else {
+				ohdr->u.aeth = ipath_compute_aeth(qp);
+				hwords++;
+				qp->s_ack_state =
+				    IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST;
+			}
+			qp->s_rdma_len -= len;
+			bth0 |= qp->s_ack_state << 24;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST:
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY:
+			/*
+			 * We have to prevent new requests from changing
+			 * the r_sge state while a ipath_verbs_send()
+			 * is in progress.
+			 * Changing r_state allows the receiver
+			 * to continue processing new packets.
+			 * We do it here now instead of above so
+			 * that we are sure the packet was sent before
+			 * changing the state.
+			 */
+			qp->r_state = IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST;
+			qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+			goto send_req;
+
+		case IB_OPCODE_RC_COMPARE_SWAP:
+		case IB_OPCODE_RC_FETCH_ADD:
+			ss = NULL;
+			len = 0;
+			qp->r_state = IB_OPCODE_RC_SEND_LAST;
+			qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+			bth0 |= IB_OPCODE_ATOMIC_ACKNOWLEDGE << 24;
+			ohdr->u.at.aeth = ipath_compute_aeth(qp);
+			ohdr->u.at.atomic_ack_eth =
+			    cpu_to_be64(qp->s_ack_atomic);
+			hwords += sizeof(ohdr->u.at) / 4;
+			break;
+
+		default:
+			/* Send a regular ACK. */
+			ss = NULL;
+			len = 0;
+			qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+			bth0 |= qp->s_ack_state << 24;
+			ohdr->u.aeth = ipath_compute_aeth(qp);
+			hwords++;
+		}
+		bth2 = qp->s_ack_psn++ & 0xFFFFFF;
+	} else {
+	      send_req:
+		if (!(state_ops[qp->state] & IPATH_PROCESS_SEND_OK) ||
+		    qp->s_rnr_timeout)
+			goto done;
+
+		/* Send a request. */
+		wqe = get_swqe_ptr(qp, qp->s_cur);
+		switch (qp->s_state) {
+		default:
+			/*
+			 * Resend an old request or start a new one.
+			 *
+			 * We keep track of the current SWQE so that
+			 * we don't reset the "furthest progress" state
+			 * if we need to back up.
+			 */
+			newreq = 0;
+			if (qp->s_cur == qp->s_tail) {
+				/* Check if send work queue is empty. */
+				if (qp->s_tail == qp->s_head)
+					goto done;
+				qp->s_psn = wqe->psn = qp->s_next_psn;
+				newreq = 1;
+			}
+			/*
+			 * Note that we have to be careful not to modify the
+			 * original work request since we may need to resend
+			 * it.
+			 */
+			qp->s_sge.sge = wqe->sg_list[0];
+			qp->s_sge.sg_list = wqe->sg_list + 1;
+			qp->s_sge.num_sge = wqe->wr.num_sge;
+			qp->s_len = len = wqe->length;
+			ss = &qp->s_sge;
+			bth2 = 0;
+			switch (wqe->wr.opcode) {
+			case IB_WR_SEND:
+			case IB_WR_SEND_WITH_IMM:
+				/* If no credit, return. */
+				if (qp->s_lsn != (u32) -1 &&
+				    cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {
+					goto done;
+				}
+				wqe->lpsn = wqe->psn;
+				if (len > pmtu) {
+					wqe->lpsn += (len - 1) / pmtu;
+					qp->s_state = IB_OPCODE_RC_SEND_FIRST;
+					len = pmtu;
+					break;
+				}
+				if (wqe->wr.opcode == IB_WR_SEND) {
+					qp->s_state = IB_OPCODE_RC_SEND_ONLY;
+				} else {
+					qp->s_state =
+					    IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE;
+					/* Immediate data comes after the BTH */
+					ohdr->u.imm_data = wqe->wr.imm_data;
+					hwords += 1;
+				}
+				if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+					bth0 |= 1 << 23;
+				bth2 = 1 << 31;	/* Request ACK. */
+				if (++qp->s_cur == qp->s_size)
+					qp->s_cur = 0;
+				break;
+
+			case IB_WR_RDMA_WRITE:
+				if (newreq)
+					qp->s_lsn++;
+				/* FALLTHROUGH */
+			case IB_WR_RDMA_WRITE_WITH_IMM:
+				/* If no credit, return. */
+				if (qp->s_lsn != (u32) -1 &&
+				    cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {
+					goto done;
+				}
+				ohdr->u.rc.reth.vaddr =
+				    cpu_to_be64(wqe->wr.wr.rdma.remote_addr);
+				ohdr->u.rc.reth.rkey =
+				    cpu_to_be32(wqe->wr.wr.rdma.rkey);
+				ohdr->u.rc.reth.length = cpu_to_be32(len);
+				hwords += sizeof(struct ib_reth) / 4;
+				wqe->lpsn = wqe->psn;
+				if (len > pmtu) {
+					wqe->lpsn += (len - 1) / pmtu;
+					qp->s_state =
+					    IB_OPCODE_RC_RDMA_WRITE_FIRST;
+					len = pmtu;
+					break;
+				}
+				if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
+					qp->s_state =
+					    IB_OPCODE_RC_RDMA_WRITE_ONLY;
+				} else {
+					qp->s_state =
+					    IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE;
+					/* Immediate data comes after RETH */
+					ohdr->u.rc.imm_data = wqe->wr.imm_data;
+					hwords += 1;
+					if (wqe->wr.
+					    send_flags & IB_SEND_SOLICITED)
+						bth0 |= 1 << 23;
+				}
+				bth2 = 1 << 31;	/* Request ACK. */
+				if (++qp->s_cur == qp->s_size)
+					qp->s_cur = 0;
+				break;
+
+			case IB_WR_RDMA_READ:
+				ohdr->u.rc.reth.vaddr =
+				    cpu_to_be64(wqe->wr.wr.rdma.remote_addr);
+				ohdr->u.rc.reth.rkey =
+				    cpu_to_be32(wqe->wr.wr.rdma.rkey);
+				ohdr->u.rc.reth.length = cpu_to_be32(len);
+				qp->s_state = IB_OPCODE_RC_RDMA_READ_REQUEST;
+				hwords += sizeof(ohdr->u.rc.reth) / 4;
+				if (newreq) {
+					qp->s_lsn++;
+					/*
+					 * Adjust s_next_psn to count the
+					 * expected number of responses.
+					 */
+					if (len > pmtu)
+						qp->s_next_psn +=
+						    (len - 1) / pmtu;
+					wqe->lpsn = qp->s_next_psn++;
+				}
+				ss = NULL;
+				len = 0;
+				if (++qp->s_cur == qp->s_size)
+					qp->s_cur = 0;
+				break;
+
+			case IB_WR_ATOMIC_CMP_AND_SWP:
+			case IB_WR_ATOMIC_FETCH_AND_ADD:
+				qp->s_state =
+				    wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ?
+				    IB_OPCODE_RC_COMPARE_SWAP :
+				    IB_OPCODE_RC_FETCH_ADD;
+				ohdr->u.atomic_eth.vaddr =
+				    cpu_to_be64(wqe->wr.wr.atomic.remote_addr);
+				ohdr->u.atomic_eth.rkey =
+				    cpu_to_be32(wqe->wr.wr.atomic.rkey);
+				ohdr->u.atomic_eth.swap_data =
+				    cpu_to_be64(wqe->wr.wr.atomic.swap);
+				ohdr->u.atomic_eth.compare_data =
+				    cpu_to_be64(wqe->wr.wr.atomic.compare_add);
+				hwords += sizeof(struct ib_atomic_eth) / 4;
+				if (newreq) {
+					qp->s_lsn++;
+					wqe->lpsn = wqe->psn;
+				}
+				if (++qp->s_cur == qp->s_size)
+					qp->s_cur = 0;
+				ss = NULL;
+				len = 0;
+				break;
+
+			default:
+				goto done;
+			}
+			if (newreq) {
+				if (++qp->s_tail >= qp->s_size)
+					qp->s_tail = 0;
+			}
+			bth2 |= qp->s_psn++ & 0xFFFFFF;
+			if ((int)(qp->s_psn - qp->s_next_psn) > 0)
+				qp->s_next_psn = qp->s_psn;
+			spin_lock(&dev->pending_lock);
+			if (qp->timerwait.next == LIST_POISON1) {
+				list_add_tail(&qp->timerwait,
+					      &dev->pending[dev->
+							    pending_index]);
+			}
+			spin_unlock(&dev->pending_lock);
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST:
+			/*
+			 * This case can only happen if a send is
+			 * restarted.  See ipath_restart_rc().
+			 */
+			ipath_init_restart(qp, wqe);
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_SEND_FIRST:
+			qp->s_state = IB_OPCODE_RC_SEND_MIDDLE;
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_SEND_MIDDLE:
+			bth2 = qp->s_psn++ & 0xFFFFFF;
+			if ((int)(qp->s_psn - qp->s_next_psn) > 0)
+				qp->s_next_psn = qp->s_psn;
+			ss = &qp->s_sge;
+			len = qp->s_len;
+			if (len > pmtu) {
+				/*
+				 * Request an ACK every 1/2 MB to avoid
+				 * retransmit timeouts.
+				 */
+				if (((wqe->length - len) % (512 * 1024)) == 0)
+					bth2 |= 1 << 31;
+				len = pmtu;
+				break;
+			}
+			if (wqe->wr.opcode == IB_WR_SEND)
+				qp->s_state = IB_OPCODE_RC_SEND_LAST;
+			else {
+				qp->s_state =
+				    IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE;
+				/* Immediate data comes after the BTH */
+				ohdr->u.imm_data = wqe->wr.imm_data;
+				hwords += 1;
+			}
+			if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+				bth0 |= 1 << 23;
+			bth2 |= 1 << 31;	/* Request ACK. */
+			if (++qp->s_cur >= qp->s_size)
+				qp->s_cur = 0;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST:
+			/*
+			 * This case can only happen if a RDMA write is
+			 * restarted.  See ipath_restart_rc().
+			 */
+			ipath_init_restart(qp, wqe);
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_RDMA_WRITE_FIRST:
+			qp->s_state = IB_OPCODE_RC_RDMA_WRITE_MIDDLE;
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:
+			bth2 = qp->s_psn++ & 0xFFFFFF;
+			if ((int)(qp->s_psn - qp->s_next_psn) > 0)
+				qp->s_next_psn = qp->s_psn;
+			ss = &qp->s_sge;
+			len = qp->s_len;
+			if (len > pmtu) {
+				/*
+				 * Request an ACK every 1/2 MB to avoid
+				 * retransmit timeouts.
+				 */
+				if (((wqe->length - len) % (512 * 1024)) == 0)
+					bth2 |= 1 << 31;
+				len = pmtu;
+				break;
+			}
+			if (wqe->wr.opcode == IB_WR_RDMA_WRITE)
+				qp->s_state = IB_OPCODE_RC_RDMA_WRITE_LAST;
+			else {
+				qp->s_state =
+				    IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE;
+				/* Immediate data comes after the BTH */
+				ohdr->u.imm_data = wqe->wr.imm_data;
+				hwords += 1;
+				if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+					bth0 |= 1 << 23;
+			}
+			bth2 |= 1 << 31;	/* Request ACK. */
+			if (++qp->s_cur >= qp->s_size)
+				qp->s_cur = 0;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:
+			/*
+			 * This case can only happen if a RDMA read is
+			 * restarted.  See ipath_restart_rc().
+			 */
+			ipath_init_restart(qp, wqe);
+			len = ((qp->s_psn - wqe->psn) & 0xFFFFFF) * pmtu;
+			ohdr->u.rc.reth.vaddr =
+			    cpu_to_be64(wqe->wr.wr.rdma.remote_addr + len);
+			ohdr->u.rc.reth.rkey =
+			    cpu_to_be32(wqe->wr.wr.rdma.rkey);
+			ohdr->u.rc.reth.length = cpu_to_be32(qp->s_len);
+			qp->s_state = IB_OPCODE_RC_RDMA_READ_REQUEST;
+			hwords += sizeof(ohdr->u.rc.reth) / 4;
+			bth2 = qp->s_psn++ & 0xFFFFFF;
+			if ((int)(qp->s_psn - qp->s_next_psn) > 0)
+				qp->s_next_psn = qp->s_psn;
+			ss = NULL;
+			len = 0;
+			if (++qp->s_cur == qp->s_size)
+				qp->s_cur = 0;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_REQUEST:
+		case IB_OPCODE_RC_COMPARE_SWAP:
+		case IB_OPCODE_RC_FETCH_ADD:
+			/*
+			 * We shouldn't start anything new until this request
+			 * is finished.  The ACK will handle rescheduling us.
+			 * XXX The number of outstanding ones is negotiated
+			 * at connection setup time (see pg. 258,289)?
+			 * XXX Also, if we support multiple outstanding
+			 * requests, we need to check the WQE IB_SEND_FENCE
+			 * flag and not send a new request if a RDMA read or
+			 * atomic is pending.
+			 */
+			goto done;
+		}
+		qp->s_len -= len;
+		bth0 |= qp->s_state << 24;
+		/* XXX queue resend timeout. */
+	}
+	/* Make sure it is non-zero before dropping the lock. */
+	qp->s_hdrwords = hwords;
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+
+	/* Construct the header. */
+	extra_bytes = (4 - len) & 3;
+	nwords = (len + extra_bytes) >> 2;
+	if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
+		/* Header size in 32-bit words. */
+		hwords += 10;
+		lrh0 = IPS_LRH_GRH;
+		qp->s_hdr.u.l.grh.version_tclass_flow =
+		    cpu_to_be32((6 << 28) |
+				(qp->remote_ah_attr.grh.traffic_class << 20) |
+				qp->remote_ah_attr.grh.flow_label);
+		qp->s_hdr.u.l.grh.paylen =
+		    cpu_to_be16(((hwords - 12) + nwords + SIZE_OF_CRC) << 2);
+		qp->s_hdr.u.l.grh.next_hdr = 0x1B;
+		qp->s_hdr.u.l.grh.hop_limit = qp->remote_ah_attr.grh.hop_limit;
+		/* The SGID is 32-bit aligned. */
+		qp->s_hdr.u.l.grh.sgid.global.subnet_prefix = dev->gid_prefix;
+		qp->s_hdr.u.l.grh.sgid.global.interface_id =
+		    ipath_layer_get_guid(dev->ib_unit);
+		qp->s_hdr.u.l.grh.dgid = qp->remote_ah_attr.grh.dgid;
+		qp->s_hdrwords = hwords;
+	}
+	qp->s_cur_sge = ss;
+	qp->s_cur_size = len;
+	lrh0 |= qp->remote_ah_attr.sl << 4;
+	qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
+	/* DEST LID */
+	qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
+	qp->s_hdr.lrh[2] = cpu_to_be16(hwords + nwords + SIZE_OF_CRC);
+	qp->s_hdr.lrh[3] = cpu_to_be16(ipath_layer_get_lid(dev->ib_unit));
+	bth0 |= extra_bytes << 20;
+	ohdr->bth[0] = cpu_to_be32(bth0);
+	ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
+	ohdr->bth[2] = cpu_to_be32(bth2);
+
+	/* Check for more work to do. */
+	goto again;
+
+done:
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+	clear_bit(IPATH_S_BUSY, &qp->s_flags);
+}
+
+static void send_rc_ack(struct ipath_qp *qp)
+{
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	u16 lrh0;
+	u32 bth0;
+	u32 hwords;
+	struct ipath_other_headers *ohdr;
+
+	/* Construct the header. */
+	ohdr = &qp->s_hdr.u.oth;
+	lrh0 = IPS_LRH_BTH;
+	/* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4. */
+	hwords = 6;
+	if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
+		ohdr = &qp->s_hdr.u.l.oth;
+		/* Header size in 32-bit words. */
+		hwords += 10;
+		lrh0 = IPS_LRH_GRH;
+		qp->s_hdr.u.l.grh.version_tclass_flow =
+		    cpu_to_be32((6 << 28) |
+				(qp->remote_ah_attr.grh.traffic_class << 20) |
+				qp->remote_ah_attr.grh.flow_label);
+		qp->s_hdr.u.l.grh.paylen =
+		    cpu_to_be16(((hwords - 12) + SIZE_OF_CRC) << 2);
+		qp->s_hdr.u.l.grh.next_hdr = 0x1B;
+		qp->s_hdr.u.l.grh.hop_limit = qp->remote_ah_attr.grh.hop_limit;
+		/* The SGID is 32-bit aligned. */
+		qp->s_hdr.u.l.grh.sgid.global.subnet_prefix = dev->gid_prefix;
+		qp->s_hdr.u.l.grh.sgid.global.interface_id =
+		    ipath_layer_get_guid(dev->ib_unit);
+		qp->s_hdr.u.l.grh.dgid = qp->remote_ah_attr.grh.dgid;
+	}
+	bth0 = ipath_layer_get_pkey(dev->ib_unit, qp->s_pkey_index);
+	ohdr->u.aeth = ipath_compute_aeth(qp);
+	if (qp->s_ack_state >= IB_OPCODE_RC_COMPARE_SWAP) {
+		bth0 |= IB_OPCODE_ATOMIC_ACKNOWLEDGE << 24;
+		ohdr->u.at.atomic_ack_eth = cpu_to_be64(qp->s_ack_atomic);
+		hwords += sizeof(ohdr->u.at.atomic_ack_eth) / 4;
+	} else {
+		bth0 |= IB_OPCODE_RC_ACKNOWLEDGE << 24;
+	}
+	lrh0 |= qp->remote_ah_attr.sl << 4;
+	qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
+	/* DEST LID */
+	qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
+	qp->s_hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC);
+	qp->s_hdr.lrh[3] = cpu_to_be16(ipath_layer_get_lid(dev->ib_unit));
+	ohdr->bth[0] = cpu_to_be32(bth0);
+	ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
+	ohdr->bth[2] = cpu_to_be32(qp->s_ack_psn & 0xFFFFFF);
+
+	/*
+	 * If we can send the ACK, clear the ACK state.
+	 */
+	if (ipath_verbs_send(dev->ib_unit, hwords, (uint32_t *) &qp->s_hdr,
+			     0, NULL) == 0) {
+		qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+		dev->n_rc_qacks++;
+	}
+}
+
+/*
+ * Back up the requester to resend the last un-ACKed request.
+ * The QP s_lock should be held.
+ */
+static void ipath_restart_rc(struct ipath_qp *qp, u32 psn, struct ib_wc *wc)
+{
+	struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
+	struct ipath_ibdev *dev;
+	u32 n;
+
+	/*
+	 * If there are no requests pending, we are done.
+	 */
+	if (cmp24(psn, qp->s_next_psn) >= 0 || qp->s_last == qp->s_tail)
+		goto done;
+
+	if (qp->s_retry == 0) {
+		wc->wr_id = wqe->wr.wr_id;
+		wc->status = IB_WC_RETRY_EXC_ERR;
+		wc->opcode = wc_opcode[wqe->wr.opcode];
+		wc->vendor_err = 0;
+		wc->byte_len = 0;
+		wc->qp_num = qp->ibqp.qp_num;
+		wc->src_qp = qp->remote_qpn;
+		wc->pkey_index = 0;
+		wc->slid = qp->remote_ah_attr.dlid;
+		wc->sl = qp->remote_ah_attr.sl;
+		wc->dlid_path_bits = 0;
+		wc->port_num = 0;
+		ipath_sqerror_qp(qp, wc);
+		return;
+	}
+	qp->s_retry--;
+
+	/*
+	 * Remove the QP from the timeout queue.
+	 * Note: it may already have been removed by ipath_ib_timer().
+	 */
+	dev = to_idev(qp->ibqp.device);
+	spin_lock(&dev->pending_lock);
+	if (qp->timerwait.next != LIST_POISON1)
+		list_del(&qp->timerwait);
+	spin_unlock(&dev->pending_lock);
+
+	if (wqe->wr.opcode == IB_WR_RDMA_READ)
+		dev->n_rc_resends++;
+	else
+		dev->n_rc_resends += (int)qp->s_psn - (int)psn;
+
+	/*
+	 * If we are starting the request from the beginning, let the
+	 * normal send code handle initialization.
+	 */
+	qp->s_cur = qp->s_last;
+	if (cmp24(psn, wqe->psn) <= 0) {
+		qp->s_state = IB_OPCODE_RC_SEND_LAST;
+		qp->s_psn = wqe->psn;
+	} else {
+		n = qp->s_cur;
+		for (;;) {
+			if (++n == qp->s_size)
+				n = 0;
+			if (n == qp->s_tail) {
+				if (cmp24(psn, qp->s_next_psn) >= 0) {
+					qp->s_cur = n;
+					wqe = get_swqe_ptr(qp, n);
+				}
+				break;
+			}
+			wqe = get_swqe_ptr(qp, n);
+			if (cmp24(psn, wqe->psn) < 0)
+				break;
+			qp->s_cur = n;
+		}
+		qp->s_psn = psn;
+
+		/*
+		 * Reset the state to restart in the middle of a request.
+		 * Don't change the s_sge, s_cur_sge, or s_cur_size.
+		 * See do_rc_send().
+		 */
+		switch (wqe->wr.opcode) {
+		case IB_WR_SEND:
+		case IB_WR_SEND_WITH_IMM:
+			qp->s_state = IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST;
+			break;
+
+		case IB_WR_RDMA_WRITE:
+		case IB_WR_RDMA_WRITE_WITH_IMM:
+			qp->s_state = IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST;
+			break;
+
+		case IB_WR_RDMA_READ:
+			qp->s_state = IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE;
+			break;
+
+		default:
+			/*
+			 * This case shouldn't happen since its only
+			 * one PSN per req.
+			 */
+			qp->s_state = IB_OPCODE_RC_SEND_LAST;
+		}
+	}
+
+done:
+	tasklet_schedule(&qp->s_task);
+}
+
+/*
+ * Handle RC and UC post sends.
+ */
+static int ipath_post_rc_send(struct ipath_qp *qp, struct ib_send_wr *wr)
+{
+	struct ipath_swqe *wqe;
+	unsigned long flags;
+	u32 next;
+	int i, j;
+	int acc;
+
+	/*
+	 * Don't allow RDMA reads or atomic operations on UC or
+	 * undefined operations.
+	 * Make sure buffer is large enough to hold the result for atomics.
+	 */
+	if (qp->ibqp.qp_type == IB_QPT_UC) {
+		if ((unsigned) wr->opcode >= IB_WR_RDMA_READ)
+			return -EINVAL;
+	} else if ((unsigned) wr->opcode > IB_WR_ATOMIC_FETCH_AND_ADD)
+		return -EINVAL;
+	else if (wr->opcode >= IB_WR_ATOMIC_CMP_AND_SWP &&
+		 (wr->num_sge == 0 || wr->sg_list[0].length < sizeof(u64) ||
+		  wr->sg_list[0].addr & 0x7))
+		return -EINVAL;
+
+	/* IB spec says that num_sge == 0 is OK. */
+	if (wr->num_sge > qp->s_max_sge)
+		return -ENOMEM;
+
+	spin_lock_irqsave(&qp->s_lock, flags);
+	next = qp->s_head + 1;
+	if (next >= qp->s_size)
+		next = 0;
+	if (next == qp->s_last) {
+		spin_unlock_irqrestore(&qp->s_lock, flags);
+		return -EINVAL;
+	}
+
+	wqe = get_swqe_ptr(qp, qp->s_head);
+	wqe->wr = *wr;
+	wqe->ssn = qp->s_ssn++;
+	wqe->sg_list[0].mr = NULL;
+	wqe->sg_list[0].vaddr = NULL;
+	wqe->sg_list[0].length = 0;
+	wqe->sg_list[0].sge_length = 0;
+	wqe->length = 0;
+	acc = wr->opcode >= IB_WR_RDMA_READ ? IB_ACCESS_LOCAL_WRITE : 0;
+	for (i = 0, j = 0; i < wr->num_sge; i++) {
+		if (to_ipd(qp->ibqp.pd)->user && wr->sg_list[i].lkey == 0) {
+			spin_unlock_irqrestore(&qp->s_lock, flags);
+			return -EINVAL;
+		}
+		if (wr->sg_list[i].length == 0)
+			continue;
+		if (!ipath_lkey_ok(&to_idev(qp->ibqp.device)->lk_table,
+				   &wqe->sg_list[j], &wr->sg_list[i], acc)) {
+			spin_unlock_irqrestore(&qp->s_lock, flags);
+			return -EINVAL;
+		}
+		wqe->length += wr->sg_list[i].length;
+		j++;
+	}
+	wqe->wr.num_sge = j;
+	qp->s_head = next;
+	/*
+	 * Wake up the send tasklet if the QP is not waiting
+	 * for an RNR timeout.
+	 */
+	next = qp->s_rnr_timeout;
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+
+	if (next == 0) {
+		if (qp->ibqp.qp_type == IB_QPT_UC)
+			do_uc_send((unsigned long) qp);
+		else
+			do_rc_send((unsigned long) qp);
+	}
+	return 0;
+}
+
+/*
+ * Note that we actually send the data as it is posted instead of putting
+ * the request into a ring buffer.  If we wanted to use a ring buffer,
+ * we would need to save a reference to the destination address in the SWQE.
+ */
+static int ipath_post_ud_send(struct ipath_qp *qp, struct ib_send_wr *wr)
+{
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ipath_other_headers *ohdr;
+	struct ib_ah_attr *ah_attr;
+	struct ipath_sge_state ss;
+	struct ipath_sge *sg_list;
+	struct ib_wc wc;
+	u32 hwords;
+	u32 nwords;
+	u32 len;
+	u32 extra_bytes;
+	u32 bth0;
+	u16 lrh0;
+	u16 lid;
+	int i;
+
+	if (!(state_ops[qp->state] & IPATH_PROCESS_SEND_OK))
+		return 0;
+
+	/* IB spec says that num_sge == 0 is OK. */
+	if (wr->num_sge > qp->s_max_sge)
+		return -EINVAL;
+
+	if (wr->num_sge > 1) {
+		sg_list = kmalloc((qp->s_max_sge - 1) * sizeof(*sg_list),
+				  GFP_ATOMIC);
+		if (!sg_list)
+			return -ENOMEM;
+	} else
+		sg_list = NULL;
+
+	/* Check the buffer to send. */
+	ss.sg_list = sg_list;
+	ss.sge.mr = NULL;
+	ss.sge.vaddr = NULL;
+	ss.sge.length = 0;
+	ss.sge.sge_length = 0;
+	ss.num_sge = 0;
+	len = 0;
+	for (i = 0; i < wr->num_sge; i++) {
+		/* Check LKEY */
+		if (to_ipd(qp->ibqp.pd)->user && wr->sg_list[i].lkey == 0)
+			return -EINVAL;
+
+		if (wr->sg_list[i].length == 0)
+			continue;
+		if (!ipath_lkey_ok(&dev->lk_table, ss.num_sge ?
+				   sg_list + ss.num_sge : &ss.sge,
+				   &wr->sg_list[i], 0)) {
+			return -EINVAL;
+		}
+		len += wr->sg_list[i].length;
+		ss.num_sge++;
+	}
+	extra_bytes = (4 - len) & 3;
+	nwords = (len + extra_bytes) >> 2;
+
+	/* Construct the header. */
+	ah_attr = &to_iah(wr->wr.ud.ah)->attr;
+	if (ah_attr->dlid >= 0xC000 && ah_attr->dlid < 0xFFFF)
+		dev->n_multicast_xmit++;
+	if (unlikely(ah_attr->dlid == ipath_layer_get_lid(dev->ib_unit))) {
+		/* Pass in an uninitialized ib_wc to save stack space. */
+		ipath_ud_loopback(qp, &ss, len, wr, &wc);
+		goto done;
+	}
+	if (ah_attr->ah_flags & IB_AH_GRH) {
+		/* Header size in 32-bit words. */
+		hwords = 17;
+		lrh0 = IPS_LRH_GRH;
+		ohdr = &qp->s_hdr.u.l.oth;
+		qp->s_hdr.u.l.grh.version_tclass_flow =
+		    cpu_to_be32((6 << 28) |
+				(ah_attr->grh.traffic_class << 20) |
+				ah_attr->grh.flow_label);
+		qp->s_hdr.u.l.grh.paylen =
+		    cpu_to_be16(((wr->opcode ==
+				  IB_WR_SEND_WITH_IMM ? 6 : 5) + nwords +
+				 SIZE_OF_CRC) << 2);
+		qp->s_hdr.u.l.grh.next_hdr = 0x1B;
+		qp->s_hdr.u.l.grh.hop_limit = ah_attr->grh.hop_limit;
+		/* The SGID is 32-bit aligned. */
+		qp->s_hdr.u.l.grh.sgid.global.subnet_prefix = dev->gid_prefix;
+		qp->s_hdr.u.l.grh.sgid.global.interface_id =
+		    ipath_layer_get_guid(dev->ib_unit);
+		qp->s_hdr.u.l.grh.dgid = ah_attr->grh.dgid;
+		/*
+		 * Don't worry about sending to locally attached
+		 * multicast QPs.  It is unspecified by the spec. what happens.
+		 */
+	} else {
+		/* Header size in 32-bit words. */
+		hwords = 7;
+		lrh0 = IPS_LRH_BTH;
+		ohdr = &qp->s_hdr.u.oth;
+	}
+	if (wr->opcode == IB_WR_SEND_WITH_IMM) {
+		ohdr->u.ud.imm_data = wr->imm_data;
+		wc.imm_data = wr->imm_data;
+		hwords += 1;
+		bth0 = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE << 24;
+	} else if (wr->opcode == IB_WR_SEND) {
+		wc.imm_data = 0;
+		bth0 = IB_OPCODE_UD_SEND_ONLY << 24;
+	} else
+		return -EINVAL;
+	lrh0 |= ah_attr->sl << 4;
+	if (qp->ibqp.qp_type == IB_QPT_SMI)
+		lrh0 |= 0xF000;	/* Set VL */
+	qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
+	qp->s_hdr.lrh[1] = cpu_to_be16(ah_attr->dlid);	/* DEST LID */
+	qp->s_hdr.lrh[2] = cpu_to_be16(hwords + nwords + SIZE_OF_CRC);
+	lid = ipath_layer_get_lid(dev->ib_unit);
+	qp->s_hdr.lrh[3] = lid ? cpu_to_be16(lid) : IB_LID_PERMISSIVE;
+	if (wr->send_flags & IB_SEND_SOLICITED)
+		bth0 |= 1 << 23;
+	bth0 |= extra_bytes << 20;
+	bth0 |= qp->ibqp.qp_type == IB_QPT_SMI ? IPS_DEFAULT_P_KEY :
+	    ipath_layer_get_pkey(dev->ib_unit, qp->s_pkey_index);
+	ohdr->bth[0] = cpu_to_be32(bth0);
+	ohdr->bth[1] = cpu_to_be32(wr->wr.ud.remote_qpn);
+	/* XXX Could lose a PSN count but not worth locking */
+	ohdr->bth[2] = cpu_to_be32(qp->s_psn++ & 0xFFFFFF);
+	/*
+	 * Qkeys with the high order bit set mean use the
+	 * qkey from the QP context instead of the WR.
+	 */
+	ohdr->u.ud.deth[0] = cpu_to_be32((int)wr->wr.ud.remote_qkey < 0 ?
+					 qp->qkey : wr->wr.ud.remote_qkey);
+	ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num);
+	if (ipath_verbs_send(dev->ib_unit, hwords, (uint32_t *) &qp->s_hdr,
+			     len, &ss))
+		dev->n_no_piobuf++;
+
+done:
+	/* Queue the completion status entry. */
+	if (!test_bit(IPATH_S_SIGNAL_REQ_WR, &qp->s_flags) ||
+	    (wr->send_flags & IB_SEND_SIGNALED)) {
+		wc.wr_id = wr->wr_id;
+		wc.status = IB_WC_SUCCESS;
+		wc.vendor_err = 0;
+		wc.opcode = IB_WC_SEND;
+		wc.byte_len = len;
+		wc.qp_num = qp->ibqp.qp_num;
+		wc.src_qp = 0;
+		wc.wc_flags = 0;
+		/* XXX initialize other fields? */
+		ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0);
+	}
+	kfree(sg_list);
+
+	return 0;
+}
+
+/*
+ * This may be called from interrupt context.
+ */
+static int ipath_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
+			   struct ib_send_wr **bad_wr)
+{
+	struct ipath_qp *qp = to_iqp(ibqp);
+	int err = 0;
+
+	/* Check that state is OK to post send. */
+	if (!(state_ops[qp->state] & IPATH_POST_SEND_OK)) {
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+
+	for (; wr; wr = wr->next) {
+		switch (qp->ibqp.qp_type) {
+		case IB_QPT_UC:
+		case IB_QPT_RC:
+			err = ipath_post_rc_send(qp, wr);
+			break;
+
+		case IB_QPT_SMI:
+		case IB_QPT_GSI:
+		case IB_QPT_UD:
+			err = ipath_post_ud_send(qp, wr);
+			break;
+
+		default:
+			err = -EINVAL;
+		}
+		if (err) {
+			*bad_wr = wr;
+			break;
+		}
+	}
+	return err;
+}
+
+/*
+ * This may be called from interrupt context.
+ */
+static int ipath_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
+			      struct ib_recv_wr **bad_wr)
+{
+	struct ipath_qp *qp = to_iqp(ibqp);
+	unsigned long flags;
+
+	/* Check that state is OK to post receive. */
+	if (!(state_ops[qp->state] & IPATH_POST_RECV_OK)) {
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+
+	for (; wr; wr = wr->next) {
+		struct ipath_rwqe *wqe;
+		u32 next;
+		int i, j;
+
+		if (wr->num_sge > qp->r_rq.max_sge) {
+			*bad_wr = wr;
+			return -ENOMEM;
+		}
+
+		spin_lock_irqsave(&qp->r_rq.lock, flags);
+		next = qp->r_rq.head + 1;
+		if (next >= qp->r_rq.size)
+			next = 0;
+		if (next == qp->r_rq.tail) {
+			spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+			*bad_wr = wr;
+			return -ENOMEM;
+		}
+
+		wqe = get_rwqe_ptr(&qp->r_rq, qp->r_rq.head);
+		wqe->wr_id = wr->wr_id;
+		wqe->sg_list[0].mr = NULL;
+		wqe->sg_list[0].vaddr = NULL;
+		wqe->sg_list[0].length = 0;
+		wqe->sg_list[0].sge_length = 0;
+		wqe->length = 0;
+		for (i = 0, j = 0; i < wr->num_sge; i++) {
+			/* Check LKEY */
+			if (to_ipd(qp->ibqp.pd)->user &&
+			    wr->sg_list[i].lkey == 0) {
+				spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+				*bad_wr = wr;
+				return -EINVAL;
+			}
+			if (wr->sg_list[i].length == 0)
+				continue;
+			if (!ipath_lkey_ok(&to_idev(qp->ibqp.device)->lk_table,
+					   &wqe->sg_list[j], &wr->sg_list[i],
+					   IB_ACCESS_LOCAL_WRITE)) {
+				spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+				*bad_wr = wr;
+				return -EINVAL;
+			}
+			wqe->length += wr->sg_list[i].length;
+			j++;
+		}
+		wqe->num_sge = j;
+		qp->r_rq.head = next;
+		spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+	}
+	return 0;
+}
+
+/*
+ * This may be called from interrupt context.
+ */
+static int ipath_post_srq_receive(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
+				  struct ib_recv_wr **bad_wr)
+{
+	struct ipath_srq *srq = to_isrq(ibsrq);
+	struct ipath_ibdev *dev = to_idev(ibsrq->device);
+	unsigned long flags;
+
+	for (; wr; wr = wr->next) {
+		struct ipath_rwqe *wqe;
+		u32 next;
+		int i, j;
+
+		if (wr->num_sge > srq->rq.max_sge) {
+			*bad_wr = wr;
+			return -ENOMEM;
+		}
+
+		spin_lock_irqsave(&srq->rq.lock, flags);
+		next = srq->rq.head + 1;
+		if (next >= srq->rq.size)
+			next = 0;
+		if (next == srq->rq.tail) {
+			spin_unlock_irqrestore(&srq->rq.lock, flags);
+			*bad_wr = wr;
+			return -ENOMEM;
+		}
+
+		wqe = get_rwqe_ptr(&srq->rq, srq->rq.head);
+		wqe->wr_id = wr->wr_id;
+		wqe->sg_list[0].mr = NULL;
+		wqe->sg_list[0].vaddr = NULL;
+		wqe->sg_list[0].length = 0;
+		wqe->sg_list[0].sge_length = 0;
+		wqe->length = 0;
+		for (i = 0, j = 0; i < wr->num_sge; i++) {
+			/* Check LKEY */
+			if (to_ipd(srq->ibsrq.pd)->user &&
+			    wr->sg_list[i].lkey == 0) {
+				spin_unlock_irqrestore(&srq->rq.lock, flags);
+				*bad_wr = wr;
+				return -EINVAL;
+			}
+			if (wr->sg_list[i].length == 0)
+				continue;
+			if (!ipath_lkey_ok(&dev->lk_table,
+					   &wqe->sg_list[j], &wr->sg_list[i],
+					   IB_ACCESS_LOCAL_WRITE)) {
+				spin_unlock_irqrestore(&srq->rq.lock, flags);
+				*bad_wr = wr;
+				return -EINVAL;
+			}
+			wqe->length += wr->sg_list[i].length;
+			j++;
+		}
+		wqe->num_sge = j;
+		srq->rq.head = next;
+		spin_unlock_irqrestore(&srq->rq.lock, flags);
+	}
+	return 0;
+}
+
+/*
+ * This is called from ipath_qp_rcv() to process an incomming UD packet
+ * for the given QP.
+ * Called at interrupt level.
+ */
+static void ipath_ud_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
+			 int has_grh, void *data, u32 tlen, struct ipath_qp *qp)
+{
+	struct ipath_other_headers *ohdr;
+	int opcode;
+	u32 hdrsize;
+	u32 pad;
+	unsigned long flags;
+	struct ib_wc wc;
+	u32 qkey;
+	u32 src_qp;
+	struct ipath_rq *rq;
+	struct ipath_srq *srq;
+	struct ipath_rwqe *wqe;
+
+	/* Check for GRH */
+	if (!has_grh) {
+		ohdr = &hdr->u.oth;
+		hdrsize = 8 + 12 + 8;	/* LRH + BTH + DETH */
+		qkey = be32_to_cpu(ohdr->u.ud.deth[0]);
+		src_qp = be32_to_cpu(ohdr->u.ud.deth[1]);
+	} else {
+		ohdr = &hdr->u.l.oth;
+		hdrsize = 8 + 40 + 12 + 8;	/* LRH + GRH + BTH + DETH */
+		/*
+		 * The header with GRH is 68 bytes and the
+		 * core driver sets the eager header buffer
+		 * size to 56 bytes so the last 12 bytes of
+		 * the IB header is in the data buffer.
+		 */
+		qkey = be32_to_cpu(((u32 *) data)[1]);
+		src_qp = be32_to_cpu(((u32 *) data)[2]);
+		data += 12;
+	}
+	src_qp &= 0xFFFFFF;
+
+	/* Check that the qkey matches. */
+	if (unlikely(qkey != qp->qkey)) {
+		/* XXX OK to lose a count once in a while. */
+		dev->qkey_violations++;
+		dev->n_pkt_drops++;
+		return;
+	}
+
+	/* Get the number of bytes the message was padded by. */
+	pad = (ohdr->bth[0] >> 12) & 3;
+	if (unlikely(tlen < (hdrsize + pad + 4))) {
+		/* Drop incomplete packets. */
+		dev->n_pkt_drops++;
+		return;
+	}
+
+	/*
+	 * A GRH is expected to preceed the data even if not
+	 * present on the wire.
+	 */
+	wc.byte_len = tlen - (hdrsize + pad + 4) + sizeof(struct ib_grh);
+
+	/*
+	 * The opcode is in the low byte when its in network order
+	 * (top byte when in host order).
+	 */
+	opcode = *(u8 *) (&ohdr->bth[0]);
+	if (opcode == IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE) {
+		if (has_grh) {
+			wc.imm_data = *(u32 *) data;
+			data += sizeof(u32);
+		} else
+			wc.imm_data = ohdr->u.ud.imm_data;
+		wc.wc_flags = IB_WC_WITH_IMM;
+		hdrsize += sizeof(u32);
+	} else if (opcode == IB_OPCODE_UD_SEND_ONLY) {
+		wc.imm_data = 0;
+		wc.wc_flags = 0;
+	} else {
+		dev->n_pkt_drops++;
+		return;
+	}
+
+	/*
+	 * Get the next work request entry to find where to put the data.
+	 * Note that it is safe to drop the lock after changing rq->tail
+	 * since ipath_post_receive() won't fill the empty slot.
+	 */
+	if (qp->ibqp.srq) {
+		srq = to_isrq(qp->ibqp.srq);
+		rq = &srq->rq;
+	} else {
+		srq = NULL;
+		rq = &qp->r_rq;
+	}
+	spin_lock_irqsave(&rq->lock, flags);
+	if (rq->tail == rq->head) {
+		spin_unlock_irqrestore(&rq->lock, flags);
+		dev->n_pkt_drops++;
+		return;
+	}
+	/* Silently drop packets which are too big. */
+	wqe = get_rwqe_ptr(rq, rq->tail);
+	if (wc.byte_len > wqe->length) {
+		spin_unlock_irqrestore(&rq->lock, flags);
+		dev->n_pkt_drops++;
+		return;
+	}
+	wc.wr_id = wqe->wr_id;
+	qp->r_sge.sge = wqe->sg_list[0];
+	qp->r_sge.sg_list = wqe->sg_list + 1;
+	qp->r_sge.num_sge = wqe->num_sge;
+	if (++rq->tail >= rq->size)
+		rq->tail = 0;
+	if (srq && srq->ibsrq.event_handler) {
+		u32 n;
+
+		if (rq->head < rq->tail)
+			n = rq->size + rq->head - rq->tail;
+		else
[… more lines in the raw message]

Lines of this file are not shown. View the raw message.

The rest of this diff is not shown. View the raw message.

Signature · 6 lines
0.99.9n
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/

Part of the diff is not shown here. View the raw message.

#24 lore raw

Re: [openib-general] Re: [PATCH 10/13] [RFC] ipath verbs, part 1

Quoted 5 lines · Paul E. McKenney
Quoted 1 line · Roland Dreier

First half of ipath verbs driver

Some RCU-related questions interspersed. Basic question is "where is the lock-free read-side traversal?"

Good question. I'll take a closer look.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#25 lore raw

Re: [openib-general] Re: [PATCH 10/13] [RFC] ipath verbs, part 1

In reply to Paul E. McKenney

The quick answer is the qp_list is traversed w/o the lock held in ipath_ib_rcv(). The intent is to be able to do a lookup on the GID to get a reference to the struct ipath_mcast and then walk the qp_list w/o locks being held while processing the received packets at interrupt level.

Quoted 13 lines · Paul E. McKenney
Quoted 1 line · Roland Dreier

First half of ipath verbs driver

Some RCU-related questions interspersed. Basic question is "where is the lock-free read-side traversal?"

Thanx, Paul

Quoted 5 lines

---

 drivers/infiniband/hw/ipath/ipath_verbs.c | 3244 +++++++++++++++++++++++++++++
 1 files changed, 3244 insertions(+), 0 deletions(-)
 create mode 100644 drivers/infiniband/hw/ipath/ipath_verbs.c

...

Quoted 31 lines
Quoted 31 lines
drivers/infiniband/hw/ipath/ipath_verbs.c new file +31 -0
+/*
+ * Insert the multicast GID into the table and
+ * attach the QP structure.
+ * Return zero if both were added.
+ * Return EEXIST if the GID was already in the table but the QP was added.
+ * Return ESRCH if the QP was already attached and neither structure was added.
+ */
+static int ipath_mcast_add(struct ipath_mcast *mcast,
+			   struct ipath_mcast_qp *mqp)
+{
+	struct rb_node **n = &mcast_tree.rb_node;
+	struct rb_node *pn = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mcast_lock, flags);
+
+	while (*n) {
+		struct ipath_mcast *tmcast;
+		struct ipath_mcast_qp *p;
+		int ret;
+
+		pn = *n;
+		tmcast = rb_entry(pn, struct ipath_mcast, rb_node);
+
+		ret = memcmp(mcast->mgid.raw, tmcast->mgid.raw,
+			     sizeof(union ib_gid));
+		if (ret < 0) {
+			n = &pn->rb_left;
+			continue;
+		}
+		if (ret > 0) {
Quoted 8 lines

drivers/infiniband/hw/ipath/ipath_verbs.c line 377

Quoted 6 lines
drivers/infiniband/hw/ipath/ipath_verbs.c new file +6 -0
+			n = &pn->rb_right;
+			continue;
+		}
+
+		/* Search the QP list to see if this is already there. */
+		list_for_each_entry_rcu(p, &tmcast->qp_list, list) {

Given that we hold the global mcast_lock, how is RCU helping here?

Its not really. I'm just trying to be consistent where ever the qp_list is traversed.

Quoted 2 lines

Is there a lock-free read-side traversal path somewhere that I am missing?

The lock free traversal is in ipath_ib_rcv() which is an interrupt routine.

Quoted 27 lines

drivers/infiniband/hw/ipath/ipath_verbs.c line 383

Quoted 6 lines
drivers/infiniband/hw/ipath/ipath_verbs.c new file +6 -0
+			if (p->qp == mqp->qp) {
+				spin_unlock_irqrestore(&mcast_lock, flags);
+				return ESRCH;
+			}
+		}
+		list_add_tail_rcu(&mqp->list, &tmcast->qp_list);

Ditto...

drivers/infiniband/hw/ipath/ipath_verbs.c line 388

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_verbs.c new file +5 -0
+		spin_unlock_irqrestore(&mcast_lock, flags);
+		return EEXIST;
+	}
+
+	list_add_tail_rcu(&mqp->list, &mcast->qp_list);

Ditto...

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_verbs.c new file +10 -0
+		spin_unlock_irqrestore(&mcast_lock, flags);
+
+	atomic_inc(&mcast->refcount);
+	rb_link_node(&mcast->rb_node, pn, n);
+	rb_insert_color(&mcast->rb_node, &mcast_tree);
+
+	spin_unlock_irqrestore(&mcast_lock, flags);
+
+	return 0;
+}
Signature · 1 line
Ralph Campbell <[email protected]>
#26 lore raw

Re: [PATCH 08/13] [RFC] ipath core last bit

In reply to Roland Dreier
Quoted 11 lines · Roland Dreier
drivers/infiniband/hw/ipath/ipath_layer.c new file +9 -0
+EXPORT_SYMBOL(ipath_kset_linkstate);
+EXPORT_SYMBOL(ipath_kset_mtu);
+EXPORT_SYMBOL(ipath_layer_close);
+EXPORT_SYMBOL(ipath_layer_get_bcast);
+EXPORT_SYMBOL(ipath_layer_get_cr_errpkey);
+EXPORT_SYMBOL(ipath_layer_get_deviceid);
+EXPORT_SYMBOL(ipath_layer_get_flags);
+EXPORT_SYMBOL(ipath_layer_get_guid);
+EXPORT_SYMBOL(ipath_layer_get_ibmtu);

etc

EXPORT_SMBOL_GPL?

#27 lore raw

Re: [PATCH 08/13] [RFC] ipath core last bit

Quoted 14 lines · Andrew Morton
Quoted 11 lines · Roland Dreier
drivers/infiniband/hw/ipath/ipath_layer.c new file +9 -0
+EXPORT_SYMBOL(ipath_kset_linkstate);
+EXPORT_SYMBOL(ipath_kset_mtu);
+EXPORT_SYMBOL(ipath_layer_close);
+EXPORT_SYMBOL(ipath_layer_get_bcast);
+EXPORT_SYMBOL(ipath_layer_get_cr_errpkey);
+EXPORT_SYMBOL(ipath_layer_get_deviceid);
+EXPORT_SYMBOL(ipath_layer_get_flags);
+EXPORT_SYMBOL(ipath_layer_get_guid);
+EXPORT_SYMBOL(ipath_layer_get_ibmtu);

etc

EXPORT_SMBOL_GPL?

Hmm, well, nothing else in the infiniband directory uses this, probably because of the dual GPL/BSD license that all files in there have. For consistency, I'll leave it as EXPORT_SYMBOL, but I don't have any real problems with it either way.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043
#28 lore raw

Re: [PATCH 07/13] [RFC] ipath core misc files

In reply to Roland Dreier
Quoted 21 lines · Roland Dreier

...

drivers/infiniband/hw/ipath/ipath_lib.c new file +19 -0
+/*
+ * This isn't perfect, but it's close enough for timing work. We want this
+ * to work on systems where the cycle counter isn't the same as the clock
+ * frequency.  The one msec spin is OK, since we execute this only once
+ * when first loaded.  We don't use CURRENT_TIME because on some systems
+ * it only has jiffy resolution; we just assume udelay is well calibrated
+ * and that we aren't likely to be rescheduled.  Do it multiple times,
+ * with a yield in between, to try to make sure we get the "true minimum"
+ * value.
+ * _ipath_pico_per_cycle isn't going to lead to completely accurate
+ * conversions from timestamps to nanoseconds, but it's close enough
+ * for our purposes, which is mainly to allow people to show events with
+ * nsecs or usecs if desired, rather than cycles.
+ */
+void ipath_init_picotime(void)
+{
+	int i;
+	u_int64_t ts, te, delta = -1ULL;
+

drivers/infiniband/hw/ipath/ipath_lib.c line 92

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_lib.c new file +10 -0
+	for (i = 0; i < 5; i++) {
+		ts = get_cycles();
+		udelay(250);
+		te = get_cycles();
+		if ((te - ts) < delta)
+			delta = te - ts;
+		yield();
+	}
+	_ipath_pico_per_cycle = 250000000 / delta;
+}

hm, I hope this is debug code which is going away. If not, we should take a look at what it's trying to do here.

Quoted 19 lines
drivers/infiniband/hw/ipath/ipath_mlock.c new file +19 -0
+/*
+ * Our version of the kernel mlock function.  This function is no longer
+ * exposed, so we need to do it ourselves.  It takes a given start page
+ * (page aligned user virtual address) and pins it and the following specified
+ * number of pages.
+ * For now, num_pages is always 1, but that will probably change at some
+ * point (because caller is doing expected sends on a single virtually
+ * contiguous buffer, so we can do all pages at once).
+ */
+int ipath_mlock(unsigned long start_page, size_t num_pages, struct page **p)
+{
+	int n;
+
+	_IPATH_VDBG("pin %lx pages from vaddr %lx\n", num_pages, start_page);
+	down_read(&current->mm->mmap_sem);
+	n = get_user_pages(current, current->mm, start_page, num_pages, 1, 1,
+			   p, NULL);
+	up_read(&current->mm->mmap_sem);
+	if (n != num_pages) {

drivers/infiniband/hw/ipath/ipath_mlock.c line 84

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_mlock.c new file +10 -0
+		_IPATH_INFO
+		    ("get_user_pages (0x%lx pages starting at 0x%lx failed with %d\n",
+		     num_pages, start_page, n);
+		if (n < 0)	/* it's an errno */
+			return n;
+		return -ENOMEM;	/* no way to know actual error */
+	}
+
+	return 0;
+}

OK. It's perhaps not a very well named function.

Quoted 17 lines
drivers/infiniband/hw/ipath/ipath_mlock.c new file +17 -0
+/*
+ * this is similar to ipath_mlock, but it's always one page, and we mark
+ * the page as locked for i/o, and shared.  This is used for the user process
+ * page that contains the destination address for the rcvhdrq tail update,
+ * so we need to have the vma.  If we don't do this, the page can be taken
+ * away from us on fork, even if the child never touches it, and then
+ * the user process never sees the tail register updates.
+ */
+int ipath_mlock_nocopy(unsigned long start_page, struct page **p)
+{
+	int n;
+	struct vm_area_struct *vm = NULL;
+
+	down_read(&current->mm->mmap_sem);
+	n = get_user_pages(current, current->mm, start_page, 1, 1, 1, p, &vm);
+	up_read(&current->mm->mmap_sem);
+	if (n != 1) {

drivers/infiniband/hw/ipath/ipath_mlock.c line 112

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_mlock.c new file +10 -0
+		_IPATH_INFO("get_user_pages for 0x%lx failed with %d\n",
+			    start_page, n);
+		if (n < 0)	/* it's an errno */
+			return n;
+		return -ENOMEM;	/* no way to know actual error */
+	}
+	vm->vm_flags |= VM_SHM | VM_LOCKED;
+
+	return 0;
+}

I don't think we want to be setting the user's VMA's vm_flags in this manner. This is purely to retain the physical page across fork?

Quoted 7 lines
drivers/infiniband/hw/ipath/ipath_mlock.c new file +7 -0
+/*
+ * Our version of the kernel munlock function.  This function is no longer
+ * exposed, so we need to do it ourselves.  It unpins the start page
+ * (a page aligned full user virtual address, not a page number)
+ * and pins it and the following specified number of pages.
+ */
+int ipath_munlock(size_t num_pages, struct page **p)

drivers/infiniband/hw/ipath/ipath_mlock.c line 130

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_mlock.c new file +10 -0
+{
+	int i;
+
+	for (i = 0; i < num_pages; i++) {
+		_IPATH_MMDBG("%u/%lu put_page %p\n", i, num_pages, p[i]);
+		SetPageDirty(p[i]);
+		put_page(p[i]);
+	}
+	return 0;
+}

Nope, SetPageDirty() doesn't tell the VM that the page is dirty - it'll never get written out. Use set_page_dirty_lock().

#29 lore raw

Re: [PATCH 07/13] [RFC] ipath core misc files

Quoted 10 lines
Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_lib.c new file +10 -0
+void ipath_init_picotime(void)
+{
+	int i;
+	u_int64_t ts, te, delta = -1ULL;
+
+	for (i = 0; i < 5; i++) {
+		ts = get_cycles();
+		udelay(250);
+		te = get_cycles();
+		if ((te - ts) < delta)
Quoted 8 lines

drivers/infiniband/hw/ipath/ipath_lib.c line 92

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_lib.c new file +5 -0
+			delta = te - ts;
+		yield();
+	}
+	_ipath_pico_per_cycle = 250000000 / delta;
+}

hm, I hope this is debug code which is going away. If not, we should take a look at what it's trying to do here.

This isn't debug code. It's used to calculate the roughly number picoseconds per cycle. This is used in the driver to make sure HyperTransport reads haven't timed out (see ipath_snap_cntr in ipath_driver.c) when reading chip counters. If you can think of a better way of figuring out whether a read took greater than a certain length of time, I'd be interested in knowing it.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#30 lore raw

Re: [PATCH 07/13] [RFC] ipath core misc files

In reply to Andrew Morton
Quoted 2 lines
Quoted 1 line

+int ipath_mlock(unsigned long start_page, size_t num_pages, struct page **p)

OK. It's perhaps not a very well named function.

Really? Suggestion for a better name?

Quoted 8 lines

drivers/infiniband/hw/ipath/ipath_mlock.c line 112

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_mlock.c new file +5 -0
+	}
+	vm->vm_flags |= VM_SHM | VM_LOCKED;
+
+	return 0;
+}

I don't think we want to be setting the user's VMA's vm_flags in this manner. This is purely to retain the physical page across fork?

I didn't write this bit of the driver, but I believe this is the case. Is there a better way of doing this?

Quoted 6 lines
Quoted 6 lines
drivers/infiniband/hw/ipath/ipath_mlock.c new file +6 -0
+int ipath_munlock(size_t num_pages, struct page **p)
+{
+	int i;
+
+	for (i = 0; i < num_pages; i++) {
+		_IPATH_MMDBG("%u/%lu put_page %p\n", i, num_pages, p[i]);
Quoted 8 lines

drivers/infiniband/hw/ipath/ipath_mlock.c line 130

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_mlock.c new file +5 -0
+		SetPageDirty(p[i]);
+		put_page(p[i]);
+	}
+	return 0;
+}

Nope, SetPageDirty() doesn't tell the VM that the page is dirty - it'll never get written out. Use set_page_dirty_lock().

OK.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#31 lore raw

Re: [PATCH 07/13] [RFC] ipath core misc files

Quoted 6 lines · Robert Walsh
Quoted 3 lines
Quoted 2 lines

+int ipath_mlock(unsigned long start_page, size_t num_pages, struct page **p)

OK. It's perhaps not a very well named function.

Really? Suggestion for a better name?

ipath_get_user_pages() would cause the least surprise.

Quoted 11 lines
Quoted 8 lines

drivers/infiniband/hw/ipath/ipath_mlock.c line 112

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_mlock.c new file +5 -0
+	}
+	vm->vm_flags |= VM_SHM | VM_LOCKED;
+
+	return 0;
+}

I don't think we want to be setting the user's VMA's vm_flags in this manner. This is purely to retain the physical page across fork?

I didn't write this bit of the driver, but I believe this is the case. Is there a better way of doing this?

This stuff has been churning a bit lately. I've drawn Hugh Dickins's attention to the patch - he'd have a better handle on what the best approach would be.

#32 lore raw

Re: [PATCH 07/13] [RFC] ipath core misc files

Quoted 9 lines · Andrew Morton
Quoted 6 lines · Robert Walsh
Quoted 3 lines
Quoted 2 lines

+int ipath_mlock(unsigned long start_page, size_t num_pages, struct page **p)

OK. It's perhaps not a very well named function.

Really? Suggestion for a better name?

ipath_get_user_pages() would cause the least surprise.

Seems reasonable. I'll look at the related functions, too.

Quoted 9 lines
Quoted 5 lines
Quoted 2 lines

I don't think we want to be setting the user's VMA's vm_flags in this manner. This is purely to retain the physical page across fork?

I didn't write this bit of the driver, but I believe this is the case. Is there a better way of doing this?

This stuff has been churning a bit lately. I've drawn Hugh Dickins's attention to the patch - he'd have a better handle on what the best approach would be.

OK then - I'll wait and see.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#33 lore raw

Re: [PATCH 04/13] [RFC] ipath LLD core, part 1

In reply to Roland Dreier

drivers/infiniband/hw/ipath/ipath_driver.c line 2279

Quoted 5 lines · Roland Dreier
drivers/infiniband/hw/ipath/ipath_driver.c new file +4 -0
+	if ((ret = copy_from_user(&rpkt, p, sizeof rpkt))) {
+		_IPATH_DBG("Failed to copy in pkt struct (%d)\n", ret);
+		return ret;
+	}

The driver does this quite a lot. copy_from_user() will return the number of bytes remaining to copy. So I think we'll be wanting `return -EFAULT;' in lots of places rather than this.

#34 lore raw

Re: [PATCH 04/13] [RFC] ipath LLD core, part 1

Quoted 10 lines · Andrew Morton

drivers/infiniband/hw/ipath/ipath_driver.c line 2279

Quoted 5 lines · Roland Dreier
drivers/infiniband/hw/ipath/ipath_driver.c new file +4 -0
+	if ((ret = copy_from_user(&rpkt, p, sizeof rpkt))) {
+		_IPATH_DBG("Failed to copy in pkt struct (%d)\n", ret);
+		return ret;
+	}

The driver does this quite a lot. copy_from_user() will return the number of bytes remaining to copy. So I think we'll be wanting `return -EFAULT;' in lots of places rather than this.

Thanks. Will fix.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#35 lore raw

Re: [PATCH 03/13] [RFC] ipath copy routines

In reply to Roland Dreier

drivers/infiniband/hw/ipath/ipath_copy.c line 63

Quoted 2 lines · Roland Dreier
drivers/infiniband/hw/ipath/ipath_copy.c new file +2 -0
+#define TRUE 1
+#define FALSE 0

Please kill these.

Pekka

#36 lore raw

Re: [PATCH 03/13] [RFC] ipath copy routines

Quoted 5 lines · Pekka Enberg

drivers/infiniband/hw/ipath/ipath_copy.c line 63

Quoted 2 lines · Roland Dreier
drivers/infiniband/hw/ipath/ipath_copy.c new file +2 -0
+#define TRUE 1
+#define FALSE 0

Please kill these.

OK.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#38 lore raw

Re: [PATCH 03/13] [RFC] ipath copy routines

In reply to Roland Dreier
Quoted 4 lines · Roland Dreier
drivers/infiniband/hw/ipath/ipath_dwordcpy.S new file +3 -0
+ 	.globl ipath_dwordcpy
+/* rdi	destination, rsi source, rdx count */
+ipath_dwordcpy:

drivers/infiniband/hw/ipath/ipath_dwordcpy.S line 62

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_dwordcpy.S new file +10 -0
+	movl %edx,%ecx
+	shrl $1,%ecx
+	andl $1,%edx	
+	cld
+	rep 
+	movsq 
+	movl %edx,%ecx
+	rep
+	movsd
+	ret

err, we have a portability problem.

#39 lore raw

Re: [PATCH 03/13] [RFC] ipath copy routines

Quoted 17 lines · Andrew Morton
Quoted 4 lines · Roland Dreier
drivers/infiniband/hw/ipath/ipath_dwordcpy.S new file +3 -0
+ 	.globl ipath_dwordcpy
+/* rdi	destination, rsi source, rdx count */
+ipath_dwordcpy:

drivers/infiniband/hw/ipath/ipath_dwordcpy.S line 62

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_dwordcpy.S new file +10 -0
+	movl %edx,%ecx
+	shrl $1,%ecx
+	andl $1,%edx	
+	cld
+	rep 
+	movsq 
+	movl %edx,%ecx
+	rep
+	movsd
+	ret

err, we have a portability problem.

Any chance we could get these moved into the x86_64 arch directory, then? We have to do double-word copies, or our chip gets unhappy.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#40 lore raw

Re: [PATCH 03/13] [RFC] ipath copy routines

Quoted 14 lines · Robert Walsh
Quoted 13 lines

drivers/infiniband/hw/ipath/ipath_dwordcpy.S line 62

Quoted 11 lines
drivers/infiniband/hw/ipath/ipath_dwordcpy.S new file +10 -0
+	movl %edx,%ecx
+	shrl $1,%ecx
+	andl $1,%edx	
+	cld
+	rep 
+	movsq 
+	movl %edx,%ecx
+	rep
+	movsd
+	ret

err, we have a portability problem.

Quoted 2 lines

Any chance we could get these moved into the x86_64 arch directory, then?

That would make sense. Give it a non-ipath-related name and require that all architectures which wish to run this driver must implement that (documented) function.

And, in Kconfig, make sure that architectures which don't implement that library function do not attempt to build this driver. To avoid breaking `make allmodconfig'.

Quoted 1 line

We have to do double-word copies, or our chip gets unhappy.

In what form is this chip available? As a standard PCI/PCIX card which people will want to plug into power4/ia64/x86 machines? Or is it in some way exclusively tied to x86_64?

#41 lore raw

Re: [PATCH 03/13] [RFC] ipath copy routines

Quoted 4 lines · Andrew Morton

...
In what form is this chip available? As a standard PCI/PCIX card which people will want to plug into power4/ia64/x86 machines? Or is it in some way exclusively tied to x86_64?

Hardware can hardly be exclusively tied to x86_64 without also being available on x86 machines since i386 kernels run on x86_64 hardware.

cu
Adrian

Signature · 4 lines
       "Is there not promise of rain?" Ling Tan asked suddenly out
        of the darkness. There had been need of rain for many days.
       "Only a promise," Lao Er said.
                                       Pearl S. Buck - Dragon Seed
#42 lore raw

Re: [PATCH 03/13] [RFC] ipath copy routines

In reply to Andrew Morton
Quoted 10 lines
Quoted 2 lines

Any chance we could get these moved into the x86_64 arch directory, then?

That would make sense. Give it a non-ipath-related name and require that all architectures which wish to run this driver must implement that (documented) function.

And, in Kconfig, make sure that architectures which don't implement that library function do not attempt to build this driver. To avoid breaking `make allmodconfig'.

Sounds good. I'll get something together next week.

Quoted 5 lines
Quoted 1 line

We have to do double-word copies, or our chip gets unhappy.

In what form is this chip available? As a standard PCI/PCIX card which people will want to plug into power4/ia64/x86 machines? Or is it in some way exclusively tied to x86_64?

It's a HyperTransport card, not PCI/PCIe/PCIX. It plugs into the HTX slot on a suitably-equipped motherboard. On some machines, it's available on the motherboard itself (e.g. the Linux Networx LS/X.)

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#43 lore raw

Re: [PATCH 03/13] [RFC] ipath copy routines

In reply to Andrew Morton

From: Andrew Morton <[email protected]>
Date: Sat, 17 Dec 2005 19:19:32 -0800

Quoted 7 lines

That would make sense. Give it a non-ipath-related name and require that all architectures which wish to run this driver must implement that (documented) function.

And, in Kconfig, make sure that architectures which don't implement that library function do not attempt to build this driver. To avoid breaking `make allmodconfig'.

How about we implement a portable version in C that you get by default if you don't implement the assembler routine? Pretty please? :-)

#44 lore raw

Re: [PATCH 03/13] [RFC] ipath copy routines

Quoted 11 lines
Quoted 7 lines

That would make sense. Give it a non-ipath-related name and require that all architectures which wish to run this driver must implement that (documented) function.

And, in Kconfig, make sure that architectures which don't implement that library function do not attempt to build this driver. To avoid breaking `make allmodconfig'.

How about we implement a portable version in C that you get by default if you don't implement the assembler routine? Pretty please? :-)

Sure. :-)

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#45 lore raw

Re: [PATCH 03/13] [RFC] ipath copy routines

In reply to Robert Walsh
Quoted 3 lines · Robert Walsh

Any chance we could get these moved into the x86_64 arch directory, then? We have to do double-word copies, or our chip gets unhappy.

Standard memcpy will do double word copies if everything is suitably aligned. Just use that.

-Andi

#46 lore raw

Re: [PATCH 03/13] [RFC] ipath copy routines

Quoted 7 lines · Andi Kleen
Quoted 3 lines · Robert Walsh

Any chance we could get these moved into the x86_64 arch directory, then? We have to do double-word copies, or our chip gets unhappy.

Standard memcpy will do double word copies if everything is suitably aligned. Just use that.

This is dealing with buffers that may be passed in from user space, so there's no guarantee of alignment for either the start address or the length.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#47 lore raw

Re: [PATCH 03/13] [RFC] ipath copy routines

Quoted 6 lines · Robert Walsh
Quoted 5 lines · Andi Kleen
Quoted 3 lines · Robert Walsh

Any chance we could get these moved into the x86_64 arch directory, then? We have to do double-word copies, or our chip gets unhappy.

Quoted 6 lines
Quoted 2 lines

Standard memcpy will do double word copies if everything is suitably aligned. Just use that.

This is dealing with buffers that may be passed in from user space, so there's no guarantee of alignment for either the start address or the length.

So how can you do double word access when the length is not a multiple of four?

The current x86-64 copy_from_user will use double work access even in that case, except for the end of course.

But what you're doing is so deeply unportable it's not funny. I am not sure such a unportable driver even belongs in the kernel.

If the code was really intended to run on user space addresses it was totally broken btw because it didn't handle exceptions.

-Andi

#48 lore raw

Re: [PATCH 03/13] [RFC] ipath copy routines

In reply to Andi Kleen
Quoted 7 lines · Andi Kleen
Quoted 3 lines · Robert Walsh

Any chance we could get these moved into the x86_64 arch directory, then? We have to do double-word copies, or our chip gets unhappy.

Standard memcpy will do double word copies if everything is suitably aligned. Just use that.

Sorry I have to disagree with that. The standard memcpy may change in future to have different properties. If you really need specific unusual properties then you want to be sure that it is obvious they are there.

I'd also like to see Dave's generic C implementation as I don't believe you can create one the compiler isn't allowed at least technically to do differently.

#49 lore raw

Re: [PATCH 01/13] [RFC] ipath basic headers

In reply to Roland Dreier

Hi Roland,

Quoted 30 lines · Roland Dreier
drivers/infiniband/hw/ipath/ipath_common.h new file +30 -0
+/*
+ * This file contains defines, structures, etc. that are used
+ * to communicate between kernel and user code.
+ */
+
+#ifdef __KERNEL__
+#include <linux/ioctl.h>
+#include <linux/uio.h>
+#include <asm/atomic.h>
+#else                          /* !__KERNEL__; user mode */
+#include <sys/ioctl.h>
+#include <sys/uio.h>
+#include <sys/types.h>
+#include <stdint.h>
+
+/* these aren't implemented for user mode, which is OK until we multi-thread */
+typedef struct _atomic {
+       uint32_t counter;
+} atomic_t;                    /* no atomic_t type in user-land */
+#define atomic_set(a,v) ((a)->counter = (v))
+#define atomic_inc_return(a)  (++(a)->counter)
+#define likely(x) (x)
+#define unlikely(x) (x)
+
+#define yield() sched_yield()
+
+/*
+ * too horrible to try and use the kernel get_cycles() or equivalent,
+ * so define and inline it here
+ */

drivers/infiniband/hw/ipath/ipath_common.h line 81

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +10 -0
+
+#if !defined(rdtscll)
+#if defined(__x86_64) || defined(__i386)
+#define rdtscll(v) do {uint32_t a,d;asm volatile("rdtsc" : "=a" (a), "=d" (d)); \
+         (v) = ((uint64_t)a) | (((uint64_t)d)<<32); \
+} while(0)
+#else
+#error "No cycle counter routine implemented yet for this platform"
+#endif
+#endif                         /* !defined(rdtscll) */

Do we really need this ugly userspace emulation code in the kernel?

Quoted 12 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +12 -0
+/*
+ * this is used for very short copies, usually 1 - 8 bytes,
+ * *NEVER* to the PIO buffers!!!!!!!  use ipath_dwordcpy for longer
+ * copies, or any copy to the PIO buffers.  Works for 32 and 64 bit
+ * gcc and pathcc
+ */
+static __inline__ void ipath_shortcopy(void *dest, void *src, uint32_t cnt)
+{
+       void *ssv, *dsv;
+       uint32_t csv;
+       __asm__ __volatile__("cld\n\trep\n\tmovsb":"=&c"(csv), "=&D"(dsv),
+                            "=&S"(ssv)

drivers/infiniband/hw/ipath/ipath_common.h line 126

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +10 -0
+                            :"0"(cnt), "1"(dest), "2"(src)
+                            :"memory");
+}
+
+/*
+ * optimized word copy; good for rev C and later opterons.  Among the best for
+ * short copies, and does as well or slightly better than the optimizization
+ * guide copies 6 and 8 at 2KB.
+ */
+void ipath_dwordcpy(uint32_t * dest, uint32_t * src, uint32_t ndwords);

What is this used for? Why can't yo use memcpy?

Quoted 1 line

+#define round_up(v,sz) (((v) + (sz)-1) & ~((sz)-1))

Please use ALIGN().

drivers/infiniband/hw/ipath/ipath_common.h line 151

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +10 -0
+/* These are used in the driver, don't use them elsewhere */
+#define _IPATH_SIMFUNC_IOCTL_LOW 1
+#define _IPATH_SIMFUNC_IOCTL_HIGH 7
+
+/*
+ * These tell the driver which ioctl's belong to the diags interface.
+ * As above, don't use them elsewhere.
+ */
+#define _IPATH_DIAG_IOCTL_LOW 100
+#define _IPATH_DIAG_IOCTL_HIGH 109

[snip, snip]

You seem to be introducing loads of new ioctls. Any reason you can't use sysfs and/or configfs?

Quoted 5 lines
drivers/infiniband/hw/ipath/ips_common.h new file +5 -0
+/* macros for processing rcvhdrq entries */
+#define ips_get_hdr_err_flags(StartOfBuffer) *(((uint32_t *)(StartOfBuffer))+1)
+#define ips_get_index(StartOfBuffer) (((*((uint32_t *)(StartOfBuffer))) >> \
+        INFINIPATH_RHF_EGRINDEX_SHIFT) & INFINIPATH_RHF_EGRINDEX_MASK)
+#define ips_get_rcv_type(StartOfBuffer)  ((*(((uint32_t *)(StartOfBuffer))) >> \

drivers/infiniband/hw/ipath/ips_common.h line 220

Quoted 10 lines
drivers/infiniband/hw/ipath/ips_common.h new file +10 -0
+        INFINIPATH_RHF_RCVTYPE_SHIFT) & INFINIPATH_RHF_RCVTYPE_MASK)
+#define ips_get_length_in_bytes(StartOfBuffer) \
+        (uint32_t)(((*(((uint32_t *)(StartOfBuffer))) >> \
+        INFINIPATH_RHF_LENGTH_SHIFT) & INFINIPATH_RHF_LENGTH_MASK) << 2)
+#define ips_get_first_protocol_header(StartOfBuffer) (void *) \
+        ((uint32_t *)(StartOfBuffer) + 2)
+#define ips_get_ips_header(StartOfBuffer) ((ips_message_header_typ *) \
+        ((uint32_t *)(StartOfBuffer) + 2))
+#define ips_get_ipath_ver(ipath_header) (((ipath_header) >> INFINIPATH_I_VERS_SHIFT) \
+        & INFINIPATH_I_VERS_MASK)

Please use static inlines instead for readability.

#50 lore raw

Re: [PATCH 01/13] [RFC] ipath basic headers

Quoted 1 line

Do we really need this ugly userspace emulation code in the kernel?

Probably not. I'll look into removing it.

Quoted 1 line

What is this used for? Why can't yo use memcpy?

Our chip can only handle double-word copies.

Quoted 3 lines
Quoted 1 line

+#define round_up(v,sz) (((v) + (sz)-1) & ~((sz)-1))

Please use ALIGN().

Fair enough.

Quoted 2 lines

You seem to be introducing loads of new ioctls. Any reason you can't use sysfs and/or configfs?

I'll see what people here think of that idea.

Quoted 4 lines

drivers/infiniband/hw/ipath/ips_common.h line 220

Quoted 2 lines
drivers/infiniband/hw/ipath/ips_common.h new file +2 -0
+#define ips_get_ipath_ver(ipath_header) (((ipath_header) >> INFINIPATH_I_VERS_SHIFT) \
+        & INFINIPATH_I_VERS_MASK)

Please use static inlines instead for readability.

OK.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#51 lore raw

Re: [PATCH 01/13] [RFC] ipath basic headers

In reply to Roland Dreier
Quoted 1 line

+ * $Id: ipath_common.h 4491 2005-12-15 22:20:31Z rjwalsh $

please remove RCSIDs everywhere.

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +10 -0
+#ifdef __KERNEL__
+#include <linux/ioctl.h>
+#include <linux/uio.h>
+#include <asm/atomic.h>
+#else				/* !__KERNEL__; user mode */
+#include <sys/ioctl.h>
+#include <sys/uio.h>
+#include <sys/types.h>
+#include <stdint.h>
+

drivers/infiniband/hw/ipath/ipath_common.h line 66

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +10 -0
+/* these aren't implemented for user mode, which is OK until we multi-thread */
+typedef struct _atomic {
+	uint32_t counter;
+} atomic_t;			/* no atomic_t type in user-land */
+#define atomic_set(a,v) ((a)->counter = (v))
+#define atomic_inc_return(a)  (++(a)->counter)
+#define likely(x) (x)
+#define unlikely(x) (x)
+
+#define yield() sched_yield()

Please push this out. It's fine if they reuse kernel-code in userspace this way, but please move the compat wrappers to a separate file that's not in the kernel tree.

Quoted 1 line

+typedef uint8_t ipath_type;

totally meaningless typedef

drivers/infiniband/hw/ipath/ipath_common.h line 100

Quoted 3 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +3 -0
+#ifndef _BITS_PER_BYTE
+#define _BITS_PER_BYTE 8
+#endif

WTF?

drivers/infiniband/hw/ipath/ipath_common.h line 111

Quoted 13 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +10 -0
+
+static __inline__ void ipath_shortcopy(void *dest, void *src, uint32_t cnt)
+    __attribute__ ((always_inline));
 
 
 
+/*
+ * this is used for very short copies, usually 1 - 8 bytes,
+ * *NEVER* to the PIO buffers!!!!!!!  use ipath_dwordcpy for longer
+ * copies, or any copy to the PIO buffers.  Works for 32 and 64 bit
+ * gcc and pathcc
+ */
+static __inline__ void ipath_shortcopy(void *dest, void *src, uint32_t cnt)

in kernel land we __inline__ includes always_inline. Also no need for a separate prototype for a just following inline function.

drivers/infiniband/hw/ipath/ipath_common.h line 119

Quoted 8 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +8 -0
+{
+	void *ssv, *dsv;
+	uint32_t csv;
+	__asm__ __volatile__("cld\n\trep\n\tmovsb":"=&c"(csv), "=&D"(dsv),
+			     "=&S"(ssv)
+			     :"0"(cnt), "1"(dest), "2"(src)
+			     :"memory");
+}

No way we're gonna put assembler code into such a driver.

drivers/infiniband/hw/ipath/ipath_common.h line 165

Quoted 4 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +4 -0
+struct ipath_int_vec {
+	int long long addr;
+	uint32_t info;
+};

please always used fixes-size types for user communication. also please avoid ioctls like the rest of the IB codebase.

drivers/infiniband/hw/ipath/ipath_common.h line 557

Quoted 12 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +12 -0
+/* Similarly, this is the kernel version going back to the user.  It's slightly
+ * different, in that we want to tell if the driver was built as part of a
+ * PathScale release, or from the driver from the OpenIB, kernel.org, or a
+ * standard distribution, for support reasons.  The high bit is 0 for
+ * non-PathScale, and 1 for PathScale-built/supplied.  That bit is defined
+ * in Makefiles, rather than this file.
+ *
+ * It's returned by the driver to the user code during initialization
+ * in the spi_sw_version field of ipath_base_info, so the user code can
+ * in turn check for compatibility with the kernel.
+*/
+#define IPATH_KERN_SWVERSION ((IPATH_KERN_TYPE<<31) | IPATH_USER_SWVERSION)

NACK, there's no way we're gonna put in a way to identify an "official" version. The official version is the last one in mainline always.

drivers/infiniband/hw/ipath/ipath_kernel.h line 46

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +5 -0
+#ifndef PCI_VENDOR_ID_PATHSCALE	/* not in pci.ids yet */
+#define PCI_VENDOR_ID_PATHSCALE 0x1fc1
+#define PCI_DEVICE_ID_PATHSCALE_INFINIPATH1 0xa
+#define PCI_DEVICE_ID_PATHSCALE_INFINIPATH2 0xd
+#endif

so move it there?

Quoted 1 line

+typedef struct _ipath_portdata {

please avoid typedefs for struct types.

Quoted 9 lines

+/*
+ * these should be somewhat dynamic someday, although they are fixed
+ * for all users of the device on any given load.
+ *
+ * NOTE: There is a VM bug in the 2.4 Kernels similar to the one Dave
+ * fixed in the 2.6 Kernel. When using large or discontinuous memory,
+ * we get random kernel oops. So, in 2.4, we are just going to stick
+ * with 4k chunks instead of 64k chunks.
+ */

No one cares about 2.4 kernels here.

drivers/infiniband/hw/ipath/ipath_kernel.h line 607

Quoted 10 lines

+ * these function similarly to the mlock/munlock system calls.
+ * ipath_mlock() is used to pin an address range (if not already pinned),
+ * and optionally return the list of physical addresses
+ * ipath_munlock() does the obvious, and ipath_mlock() cleans up all
+ * private memory, used at driver unload.
+ * ipath_mlock_nocopy() is similar to mlock, but only one page, and marks
+ * the vm so the page isn't taken away on a fork.
+ */

drivers/infiniband/hw/ipath/ipath_kernel.h new file +2 -0
+int ipath_mlock(unsigned long, size_t, struct page **);
+int ipath_mlock_nocopy(unsigned long, struct page **);

this kind of thing definitly doesn't belong into an LLDD. or maybe it's just stale prototypes?

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +5 -0
+#ifdef IPATH_COSIM
+extern __u32 sim_readl(const volatile void __iomem * addr);
+extern __u64 sim_readq(const volatile void __iomem * addr);
+extern void sim_writel(__u32 val, volatile void __iomem * addr);
+extern void sim_writeq(__u64 val, volatile void __iomem * addr);

drivers/infiniband/hw/ipath/ipath_kernel.h line 678

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +10 -0
+#define ipath_readl(addr) sim_readl(addr)
+#define ipath_readq(addr) sim_readq(addr)
+#define ipath_writel(val, addr) sim_writel(val, addr)
+#define ipath_writeq(val, addr) sim_writeq(val, addr)
+#else
+#define ipath_readl(addr) readl(addr)
+#define ipath_readq(addr) readq(addr)
+#define ipath_writel(val, addr) writel(val, addr)
+#define ipath_writeq(val, addr) writeq(val, addr)
+#endif

Please use the proper functions directly. Your simulator can override them if nessecary.

drivers/infiniband/hw/ipath/ipath_kernel.h line 732

Quoted 9 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +9 -0
+static __inline__ uint32_t ipath_kget_kreg32(const ipath_type stype,
+					     ipath_kreg regno)
+{
+	volatile uint32_t *kreg32;
+
+	if (!devdata[stype].ipath_kregbase)
+		return ~0;
+
+	kreg32 = (volatile uint32_t *)&devdata[stype].ipath_kregbase[regno];

volatile use is probably always wrong. but this whole functions looks like a very odd wrapper anyway?

#52 lore raw

Re: [PATCH 01/13] [RFC] ipath basic headers

Quoted 2 lines · Christoph Hellwig

please always used fixes-size types for user communication. also please avoid ioctls like the rest of the IB codebase.

Could someone please explain to me how the uverbs abuse of write is better that ioctl?

Every single command seems to have a __u64 response fields that is a pointer into user space. When you write your commands and read your responses like the netlink layer does I can see the sense of it. But making write an ioctl by another name...

One of the scarier comments I have seen lately from ib_user_verbs.h /*
* Make sure that all structs defined in this file remain laid out so
* that they pack the same way on 32-bit and 64-bit architectures (to
* avoid incompatibility between 32-bit userspace and 64-bit kernels).
* Specifically:
* - Do not use pointer types -- pass pointers in __u64 instead.
* - Make sure that any structure larger than 4 bytes is padded to a
* multiple of 8 bytes. Otherwise the structure size will be
* different between 32-bit and 64-bit architectures.
*/

The two points that get called out.
- Embedded pointers are a large part of what make ioctl a maintenance nightmare. I admit we are 15-20 years away before big machines exhaust the capability of 64bit pointers so we aren't likely to run into size issues soon. But a write that changes your address space is ugly, and unexpected.

What looks like a reimplementation of readv/writev using this technique is also scary.

- 64bit compilers will not pad every structure to 8 bytes. This only will happen if you happen to have an 8 byte element in your structure that is only aligned to 32bits by a 32bit structure. Unfortunately the 32bit gcc only aligns long long to 32bits on x86, which triggers the described behavior.

Eric

#53 lore raw

Re: [PATCH 01/13] [RFC] ipath basic headers

Quoted 7 lines · [email protected] (Eric W. Biederman) writes:
Quoted 2 lines · Christoph Hellwig

please always used fixes-size types for user communication. also please avoid ioctls like the rest of the IB codebase.

Could someone please explain to me how the uverbs abuse of write is better that ioctl?

It's actually worse because if they have a 32bit compat issue then ioctl can be fixed up, but read/write can't.

I wish the people arguing against ioctl all the time would just stop that because the alternatives are usually worse.

Quoted 5 lines

- 64bit compilers will not pad every structure to 8 bytes. This only will happen if you happen to have an 8 byte element in your structure that is only aligned to 32bits by a 32bit structure. Unfortunately the 32bit gcc only aligns long long to 32bits on x86, which triggers the described behavior.

Exactly - and driver writers usually don't get that right so we need to have a tool to fix it up in the end. And with ioctl that's easiest.

-Andi

#54 lore raw

Re: [PATCH 01/13] [RFC] ipath basic headers

Quoted 7 lines · Andi Kleen
Quoted 5 lines · [email protected] (Eric W. Biederman) writes:
Quoted 2 lines · Christoph Hellwig

please always used fixes-size types for user communication. also please avoid ioctls like the rest of the IB codebase.

Quoted 8 lines
Quoted 2 lines

Could someone please explain to me how the uverbs abuse of write is better that ioctl?

It's actually worse because if they have a 32bit compat issue then ioctl can be fixed up, but read/write can't.

I wish the people arguing against ioctl all the time would just stop that because the alternatives are usually worse.

Some of the suggestions like using sysfs aren't too bad. One value per file in text format is clean and not going to change when you switch architectures :)

The rule should really be that you can't just argue against ioctl but instead you must argue for something.

Quoted 9 lines
Quoted 5 lines

- 64bit compilers will not pad every structure to 8 bytes. This only will happen if you happen to have an 8 byte element in your structure that is only aligned to 32bits by a 32bit structure. Unfortunately the 32bit gcc only aligns long long to 32bits on x86, which triggers the described behavior.

Exactly - and driver writers usually don't get that right so we need to have a tool to fix it up in the end. And with ioctl that's easiest.

In this case I don't see any current problems. But I don't think this is a pattern we want to encourage, and if there is a more maintainable pattern now would be the time to fix it.

Eric

#55 lore raw

Re: [PATCH 01/13] [RFC] ipath basic headers

In reply to Christoph Hellwig
Quoted 3 lines
Quoted 1 line

+ * $Id: ipath_common.h 4491 2005-12-15 22:20:31Z rjwalsh $

please remove RCSIDs everywhere.

These are everywhere in the OpenIB code. I was actually asked by one of the OpenIB developers to include them. I'm happy to remove them again, but what do the OpenIB folks think?

Quoted 5 lines
Quoted 1 line

+#define yield() sched_yield()

Please push this out. It's fine if they reuse kernel-code in userspace this way, but please move the compat wrappers to a separate file that's not in the kernel tree.

I will do this.

Quoted 3 lines
Quoted 1 line

+typedef uint8_t ipath_type;

totally meaningless typedef

In what way?

Quoted 5 lines

drivers/infiniband/hw/ipath/ipath_common.h line 100

Quoted 3 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +3 -0
+#ifndef _BITS_PER_BYTE
+#define _BITS_PER_BYTE 8
+#endif

WTF?

Hmm. That is odd. I'll ask the folks here if we can remove this.

Quoted 2 lines

in kernel land we __inline__ includes always_inline. Also no need for a separate prototype for a just following inline function.

Fine.

Quoted 10 lines

drivers/infiniband/hw/ipath/ipath_common.h line 119

Quoted 8 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +8 -0
+{
+	void *ssv, *dsv;
+	uint32_t csv;
+	__asm__ __volatile__("cld\n\trep\n\tmovsb":"=&c"(csv), "=&D"(dsv),
+			     "=&S"(ssv)
+			     :"0"(cnt), "1"(dest), "2"(src)
+			     :"memory");
+}

No way we're gonna put assembler code into such a driver.

Why not? The chip (and therefore the driver) only works with Opterons. It's tied to the HT bus, but PCI or anything like that.

Quoted 7 lines

drivers/infiniband/hw/ipath/ipath_common.h line 165

Quoted 4 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +4 -0
+struct ipath_int_vec {
+	int long long addr;
+	uint32_t info;
+};

please always used fixes-size types for user communication.

OK.

Quoted 2 lines

also please
avoid ioctls like the rest of the IB codebase.

More complex, but I'll look into it.

Quoted 7 lines
Quoted 7 lines

+/* Similarly, this is the kernel version going back to the user. It's slightly
+ * different, in that we want to tell if the driver was built as part of a
+ * PathScale release, or from the driver from the OpenIB, kernel.org, or a
+ * standard distribution, for support reasons. The high bit is 0 for
+ * non-PathScale, and 1 for PathScale-built/supplied. That bit is defined
+ * in Makefiles, rather than this file.
+ *

Quoted 8 lines

drivers/infiniband/hw/ipath/ipath_common.h line 557

Quoted 5 lines

+ * It's returned by the driver to the user code during initialization
+ * in the spi_sw_version field of ipath_base_info, so the user code can
+ * in turn check for compatibility with the kernel.

drivers/infiniband/hw/ipath/ipath_common.h new file +2 -0
+*/
+#define IPATH_KERN_SWVERSION ((IPATH_KERN_TYPE<<31) | IPATH_USER_SWVERSION)

NACK, there's no way we're gonna put in a way to identify an "official" version. The official version is the last one in mainline always.

Why make this hard for vendors? You may only care about the latest mainline, but if we want to sell chips, we have to support this all the way back to 2.6.9 (RHEL).

Quoted 7 lines

drivers/infiniband/hw/ipath/ipath_kernel.h line 46

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +5 -0
+#ifndef PCI_VENDOR_ID_PATHSCALE	/* not in pci.ids yet */
+#define PCI_VENDOR_ID_PATHSCALE 0x1fc1
+#define PCI_DEVICE_ID_PATHSCALE_INFINIPATH1 0xa
+#define PCI_DEVICE_ID_PATHSCALE_INFINIPATH2 0xd
+#endif

so move it there?

Sounds like a good idea. I'll submit a separate patch.

Quoted 3 lines
Quoted 1 line

+typedef struct _ipath_portdata {

please avoid typedefs for struct types.

I thought I had, but I must have missed this one.

Quoted 11 lines
Quoted 9 lines

+/*
+ * these should be somewhat dynamic someday, although they are fixed
+ * for all users of the device on any given load.
+ *
+ * NOTE: There is a VM bug in the 2.4 Kernels similar to the one Dave
+ * fixed in the 2.6 Kernel. When using large or discontinuous memory,
+ * we get random kernel oops. So, in 2.4, we are just going to stick
+ * with 4k chunks instead of 64k chunks.
+ */

No one cares about 2.4 kernels here.

Fine.

Quoted 5 lines
Quoted 5 lines

+ * these function similarly to the mlock/munlock system calls.
+ * ipath_mlock() is used to pin an address range (if not already pinned),
+ * and optionally return the list of physical addresses
+ * ipath_munlock() does the obvious, and ipath_mlock() cleans up all
+ * private memory, used at driver unload.

Quoted 8 lines

drivers/infiniband/hw/ipath/ipath_kernel.h line 607

Quoted 5 lines

+ * ipath_mlock_nocopy() is similar to mlock, but only one page, and marks
+ * the vm so the page isn't taken away on a fork.
+ */

drivers/infiniband/hw/ipath/ipath_kernel.h new file +2 -0
+int ipath_mlock(unsigned long, size_t, struct page **);
+int ipath_mlock_nocopy(unsigned long, struct page **);

this kind of thing definitly doesn't belong into an LLDD. or maybe it's just stale prototypes?

No - they're used. Why do you say they don't belong?

Quoted 10 lines
Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +10 -0
+#ifdef IPATH_COSIM
+extern __u32 sim_readl(const volatile void __iomem * addr);
+extern __u64 sim_readq(const volatile void __iomem * addr);
+extern void sim_writel(__u32 val, volatile void __iomem * addr);
+extern void sim_writeq(__u64 val, volatile void __iomem * addr);
+#define ipath_readl(addr) sim_readl(addr)
+#define ipath_readq(addr) sim_readq(addr)
+#define ipath_writel(val, addr) sim_writel(val, addr)
+#define ipath_writeq(val, addr) sim_writeq(val, addr)
+#else
Quoted 8 lines

drivers/infiniband/hw/ipath/ipath_kernel.h line 678

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +5 -0
+#define ipath_readl(addr) readl(addr)
+#define ipath_readq(addr) readq(addr)
+#define ipath_writel(val, addr) writel(val, addr)
+#define ipath_writeq(val, addr) writeq(val, addr)
+#endif

Please use the proper functions directly. Your simulator can override them if nessecary.

Fine.

Quoted 4 lines
Quoted 4 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +4 -0
+static __inline__ uint32_t ipath_kget_kreg32(const ipath_type stype,
+					     ipath_kreg regno)
+{
+	volatile uint32_t *kreg32;
Quoted 8 lines

drivers/infiniband/hw/ipath/ipath_kernel.h line 732

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +5 -0
+
+	if (!devdata[stype].ipath_kregbase)
+		return ~0;
+
+	kreg32 = (volatile uint32_t *)&devdata[stype].ipath_kregbase[regno];

volatile use is probably always wrong. but this whole functions looks like a very odd wrapper anyway?

The volatile is there so the compiler doesn't optimize away the read. This is important, because reads of our hardware have side-effects and cannot be optimized out.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#56 lore raw

Re: [PATCH 01/13] [RFC] ipath basic headers

Quoted 11 lines · Robert Walsh
Quoted 10 lines

drivers/infiniband/hw/ipath/ipath_common.h line 119

Quoted 8 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +8 -0
+{
+	void *ssv, *dsv;
+	uint32_t csv;
+	__asm__ __volatile__("cld\n\trep\n\tmovsb":"=&c"(csv), "=&D"(dsv),
+			     "=&S"(ssv)
+			     :"0"(cnt), "1"(dest), "2"(src)
+			     :"memory");
+}

No way we're gonna put assembler code into such a driver.

Quoted 2 lines

Why not? The chip (and therefore the driver) only works with Opterons. It's tied to the HT bus, but PCI or anything like that.

and opterons can already run 2 architectures. And the HT bus is a generic bus.. with public specs. Others than just AMD use it as well.

also.. what is wrong with memcpy and co ?

Quoted 13 lines
Quoted 4 lines
Quoted 4 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +4 -0
+static __inline__ uint32_t ipath_kget_kreg32(const ipath_type stype,
+					     ipath_kreg regno)
+{
+	volatile uint32_t *kreg32;
Quoted 8 lines

drivers/infiniband/hw/ipath/ipath_kernel.h line 732

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +5 -0
+
+	if (!devdata[stype].ipath_kregbase)
+		return ~0;
+
+	kreg32 = (volatile uint32_t *)&devdata[stype].ipath_kregbase[regno];

volatile use is probably always wrong. but this whole functions looks like a very odd wrapper anyway?

Quoted 3 lines

The volatile is there so the compiler doesn't optimize away the read. This is important, because reads of our hardware have side-effects and cannot be optimized out.

then you need to use readl() and family most like; they already take care of this anyway.

#57 lore raw

Re: [PATCH 01/13] [RFC] ipath basic headers

Quoted 4 lines

and opterons can already run 2 architectures. And the HT bus is a generic bus.. with public specs. Others than just AMD use it as well.

also.. what is wrong with memcpy and co ?

Our chips can only handle double-word writes. memcpy isn't guaranteed to do this.

Quoted 2 lines

then you need to use readl() and family most like; they already take care of this anyway.

Oh, OK then.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#58 lore raw

Re: [PATCH 01/13] [RFC] ipath basic headers

In reply to Roland Dreier
Quoted 13 lines · Roland Dreier

...

drivers/infiniband/hw/ipath/ipath_common.h new file +10 -0
+#ifdef __KERNEL__
+#include <linux/ioctl.h>
+#include <linux/uio.h>
+#include <asm/atomic.h>
+#else				/* !__KERNEL__; user mode */
+#include <sys/ioctl.h>
+#include <sys/uio.h>
+#include <sys/types.h>
+#include <stdint.h>
+

drivers/infiniband/hw/ipath/ipath_common.h line 66

Quoted 10 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +10 -0
+/* these aren't implemented for user mode, which is OK until we multi-thread */
+typedef struct _atomic {
+	uint32_t counter;
+} atomic_t;			/* no atomic_t type in user-land */
+#define atomic_set(a,v) ((a)->counter = (v))
+#define atomic_inc_return(a)  (++(a)->counter)
+#define likely(x) (x)
+#define unlikely(x) (x)
+
+#define yield() sched_yield()

Some might get upset about what I assume is userspace test harness code or what _is_ this doing?) in a driver. But if the maintainers find it useful we can live with it,

drivers/infiniband/hw/ipath/ipath_common.h line 100

Quoted 3 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +3 -0
+#ifndef _BITS_PER_BYTE
+#define _BITS_PER_BYTE 8
+#endif

I'd be inclined to stick BITS_PER_BYTE into include/linux/types.h.

drivers/infiniband/hw/ipath/ipath_common.h line 103

Quoted 2 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +2 -0
+static __inline__ void ipath_shortcopy(void *dest, void *src, uint32_t cnt)
+    __attribute__ ((always_inline));

s/__inline__/inline/ throughout.

Quoted 1 line

+#define round_up(v,sz) (((v) + (sz)-1) & ~((sz)-1))

We have ALIGN()

drivers/infiniband/hw/ipath/ipath_common.h line 163

Quoted 2 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +2 -0
+struct ipath_int_vec {
+	int long long addr;

long long

drivers/infiniband/hw/ipath/ipath_common.h line 167

Quoted 4 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +4 -0
+	uint32_t info;
+};
+struct ipath_eeprom_req {
+	long long addr;

Like this.

drivers/infiniband/hw/ipath/ipath_common.h line 195

Quoted 9 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +8 -0
+	uint16_t len;
+	uint16_t offset;
+};
...
+#define IPATH_USERINIT      _IOW('s', 16, struct ipath_user_info)
+/* init;  kernel/chip params to user */
+#define IPATH_BASEINFO      _IOR('s', 17, struct ipath_base_info)
+/* send a packet */
+#define IPATH_SENDPKT       _IOW('s', 18, struct ipath_sendpkt)

uh-oh. ioctls. Do we have compat conversions for them all, if needed?

drivers/infiniband/hw/ipath/ipath_kernel.h line 421

Quoted 9 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +9 -0
+/*
+ * A segment is a linear region of low physical memory.
+ * XXX Maybe we should use phys addr here and kmap()/kunmap()
+ * Used by the verbs layer.
+ */
+struct ipath_seg {
+	void *vaddr;
+	u64 length;
+};

Suggest `long' for the length. We don't need 64 bits on 32-bit machines.

drivers/infiniband/hw/ipath/ipath_kernel.h line 434

Quoted 2 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +2 -0
+struct ipath_mregion {
+	u64 user_base;		/* User's address for this region */

void *.

drivers/infiniband/hw/ipath/ipath_kernel.h line 435

Quoted 1 line
drivers/infiniband/hw/ipath/ipath_kernel.h new file +1 -0
+	u64 iova;		/* IB start address of this region */

Maybe here too.

Quoted 1 line

+int ipath_mlock(unsigned long, size_t, struct page **);

Sometimes it does int foo()' and sometimes extern int foo()'. I tend to think the `extern' is a waste of space.

drivers/infiniband/hw/ipath/ipath_kernel.h line 624

Quoted 8 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +8 -0
+#define ipath_func_krecord(a)
+#define ipath_func_urecord(a, b)
+#define ipath_func_mrecord(a, b)
+#define ipath_func_rkrecord(a)
+#define ipath_func_rurecord(a, b)
+#define ipath_func_rmrecord(a, b)
+#define ipath_func_rsrecord(a)
+#define ipath_func_rcrecord(a)

What are all these doing? Might need do{}while(0) for safety.

drivers/infiniband/hw/ipath/ipath_kernel.h line 666

Quoted 3 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +3 -0
+#ifdef IPATH_COSIM
+extern __u32 sim_readl(const volatile void __iomem * addr);
+extern __u64 sim_readq(const volatile void __iomem * addr);

The driver has a strange mixture of int32_t, s32 and __s32. s32 is preferred.

drivers/infiniband/hw/ipath/ipath_kernel.h line 704

Quoted 11 lines

+ */

drivers/infiniband/hw/ipath/ipath_kernel.h new file +10 -0
+static __inline__ uint32_t ipath_kget_ureg32(const ipath_type stype,
+					     ipath_ureg regno, int port)
+{
+	uint64_t *ubase;
+
+	ubase = (uint64_t *) (devdata[stype].ipath_uregbase
+			      + (char *)devdata[stype].ipath_kregbase
+			      + devdata[stype].ipath_palign * port);
+	return ubase ? ipath_readl(ubase + regno) : 0;
+}

Are all these u64's needed on 32-bit?

drivers/infiniband/hw/ipath/ipath_kernel.h line 740

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +5 -0
+static __inline__ uint64_t ipath_kget_kreg64(const ipath_type stype,
+					     ipath_kreg regno)
+{
+	if (!devdata[stype].ipath_kregbase)
+		return ~0ULL;

We don't know that the architecture implements u64 as unsigned long long. Some use unsigned long. Best way of implmenting the all-ones pattern is just `-1'.

Gee. Big driver.

#59 lore raw

Re: [PATCH 01/13] [RFC] ipath basic headers

Quoted 5 lines
Quoted 1 line

+#define yield() sched_yield()

Some might get upset about what I assume is userspace test harness code or what _is_ this doing?) in a driver. But if the maintainers find it useful we can live with it,

That is cosimulator code. It's easy enough to remove. I'll look into it.

Quoted 5 lines

drivers/infiniband/hw/ipath/ipath_common.h line 100

Quoted 3 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +3 -0
+#ifndef _BITS_PER_BYTE
+#define _BITS_PER_BYTE 8
+#endif

I'd be inclined to stick BITS_PER_BYTE into include/linux/types.h.

Really? I was just going to suggest removing it, but if sticking it in types.h works for you, then fine.

Quoted 4 lines

drivers/infiniband/hw/ipath/ipath_common.h line 103

Quoted 2 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +2 -0
+static __inline__ void ipath_shortcopy(void *dest, void *src, uint32_t cnt)
+    __attribute__ ((always_inline));

s/__inline__/inline/ throughout.

OK.

Quoted 3 lines
Quoted 1 line

+#define round_up(v,sz) (((v) + (sz)-1) & ~((sz)-1))

We have ALIGN()

Yup.

Quoted 4 lines

drivers/infiniband/hw/ipath/ipath_common.h line 163

Quoted 2 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +2 -0
+struct ipath_int_vec {
+	int long long addr;

long long

OK.

Quoted 7 lines

drivers/infiniband/hw/ipath/ipath_common.h line 195

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_common.h new file +5 -0
+#define IPATH_USERINIT      _IOW('s', 16, struct ipath_user_info)
+/* init;  kernel/chip params to user */
+#define IPATH_BASEINFO      _IOR('s', 17, struct ipath_base_info)
+/* send a packet */
+#define IPATH_SENDPKT       _IOW('s', 18, struct ipath_sendpkt)

uh-oh. ioctls. Do we have compat conversions for them all, if needed?

For those that are needed, I believe we covered them all. Some have suggested removing ioctls. I'm willing to look into alternatives, but if you think they're OK, I'd rather leave them.

Quoted 11 lines

drivers/infiniband/hw/ipath/ipath_kernel.h line 421

Quoted 9 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +9 -0
+/*
+ * A segment is a linear region of low physical memory.
+ * XXX Maybe we should use phys addr here and kmap()/kunmap()
+ * Used by the verbs layer.
+ */
+struct ipath_seg {
+	void *vaddr;
+	u64 length;
+};

Suggest `long' for the length. We don't need 64 bits on 32-bit machines.

OK.

Quoted 8 lines

drivers/infiniband/hw/ipath/ipath_kernel.h line 434

Quoted 2 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +2 -0
+struct ipath_mregion {
+	u64 user_base;		/* User's address for this region */

void *.

drivers/infiniband/hw/ipath/ipath_kernel.h line 435

Quoted 1 line
drivers/infiniband/hw/ipath/ipath_kernel.h new file +1 -0
+	u64 iova;		/* IB start address of this region */

Maybe here too.

OK.

Quoted 4 lines
Quoted 1 line

+int ipath_mlock(unsigned long, size_t, struct page **);

Sometimes it does int foo()' and sometimes extern int foo()'. I tend to think the `extern' is a waste of space.

Yup.

Quoted 10 lines

drivers/infiniband/hw/ipath/ipath_kernel.h line 624

Quoted 8 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +8 -0
+#define ipath_func_krecord(a)
+#define ipath_func_urecord(a, b)
+#define ipath_func_mrecord(a, b)
+#define ipath_func_rkrecord(a)
+#define ipath_func_rurecord(a, b)
+#define ipath_func_rmrecord(a, b)
+#define ipath_func_rsrecord(a)
+#define ipath_func_rcrecord(a)

What are all these doing? Might need do{}while(0) for safety.

I'll look at cleaning them out. Probably left-overs from some earlier experiment.

Quoted 6 lines

drivers/infiniband/hw/ipath/ipath_kernel.h line 666

Quoted 3 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +3 -0
+#ifdef IPATH_COSIM
+extern __u32 sim_readl(const volatile void __iomem * addr);
+extern __u64 sim_readq(const volatile void __iomem * addr);

The driver has a strange mixture of int32_t, s32 and __s32. s32 is preferred.

Yea - I'll clean that up.

Quoted 5 lines
Quoted 5 lines

+ */

drivers/infiniband/hw/ipath/ipath_kernel.h new file +4 -0
+static __inline__ uint32_t ipath_kget_ureg32(const ipath_type stype,
+					     ipath_ureg regno, int port)
+{
+	uint64_t *ubase;
Quoted 8 lines

drivers/infiniband/hw/ipath/ipath_kernel.h line 704

Quoted 6 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +6 -0
+
+	ubase = (uint64_t *) (devdata[stype].ipath_uregbase
+			      + (char *)devdata[stype].ipath_kregbase
+			      + devdata[stype].ipath_palign * port);
+	return ubase ? ipath_readl(ubase + regno) : 0;
+}

Are all these u64's needed on 32-bit?

Don't know - I'll ask around. We don't support the hardware in 32-bit anyway, so...

Quoted 9 lines

drivers/infiniband/hw/ipath/ipath_kernel.h line 740

Quoted 5 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +5 -0
+static __inline__ uint64_t ipath_kget_kreg64(const ipath_type stype,
+					     ipath_kreg regno)
+{
+	if (!devdata[stype].ipath_kregbase)
+		return ~0ULL;

We don't know that the architecture implements u64 as unsigned long long. Some use unsigned long. Best way of implmenting the all-ones pattern is just `-1'.

OK.

Quoted 1 line

Gee. Big driver.

Tell me about it :-) Basically, we're doing infiniband in software: no offload.

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043.
#60 lore raw

Re: [PATCH 01/13] [RFC] ipath basic headers

Quoted 6 lines · Robert Walsh
Quoted 2 lines

I'd be inclined to stick BITS_PER_BYTE into include/linux/types.h.

Really? I was just going to suggest removing it, but if sticking it in types.h works for you, then fine.

I think it's a readbility thing.

	x += 8;			/* wtf? */

vs

	x += BITS_PER_BYTE;	/* ah! */
#61 lore raw

Re: [PATCH 01/13] [RFC] ipath basic headers

In reply to Andrew Morton
Quoted 6 lines

drivers/infiniband/hw/ipath/ipath_kernel.h line 666

Quoted 3 lines
drivers/infiniband/hw/ipath/ipath_kernel.h new file +3 -0
+#ifdef IPATH_COSIM
+extern __u32 sim_readl(const volatile void __iomem * addr);
+extern __u64 sim_readq(const volatile void __iomem * addr);

The driver has a strange mixture of int32_t, s32 and __s32. s32 is preferred.

The cosim stuff has been nuked, as it was old code anyway. With those functions gone, we now use int32_t (and related 8-, 16-, 32- and 64-bit signed and unsigned versions) consistently throughout the code. We'd prefer to keep it that way, instead of changing over to s32 and friends, as some of our header files are used by userland programs. Unless we put in magic typedef for s32 and friends in userland, that won't work, hence we use the C standard typedefs.

Is this a problem?

Regards,
Robert.

Signature · 4 lines
Robert Walsh                                 Email: [email protected]
PathScale, Inc.                              Phone: +1 650 934 8117
2071 Stierlin Court, Suite 200                 Fax: +1 650 428 1969
Mountain View, CA 94043
#62 lore raw

Re: [PATCH 00/13] [RFC] IB: PathScale InfiniPath driver

In reply to Roland Dreier
Quoted 6 lines · Roland Dreier

having sysctls that set values also settable through module parameters under /sys/module, code inside #ifndef __KERNEL__ so include files can be shared with other PathScale code, code in ipath_i2c.c that might be simplified by using drivers/i2c, etc. I'd like to try to get a sense of whether I'm being too picky or whether PathScale really does need to fix these up before the driver is merged.

Yes, please fix this stuff before. The current driver looks like a horrible mess.

Is there some political plot going where pathscale folks are forcing you to send this out in this scheme? Otherwise I couldn't explain the code quality magnitudes lower than normally expected from your merges.

#63 lore raw

Re: [PATCH 00/13] [RFC] IB: PathScale InfiniPath driver

Christoph> Is there some political plot going where pathscale Christoph> folks are forcing you to send this out in this scheme? Christoph> Otherwise I couldn't explain the code quality Christoph> magnitudes lower than normally expected from your Christoph> merges.

No political plot -- this posting was an RFC in the literal sense, with no expectation that the code is mergable as-is. I just want to get comments early so that we have a better idea of what needs to be fixed. For example, what's your feeling about sysctls in drivers?

BTW, Pathscale people -- please respond to the comments that are made about your driver...

- R.