ipath - PathScale InfiniPath driver

diff -r 9e8d017ed298 -r ffbd416f30d4 drivers/infiniband/hw/ipath/ipath_copy.c
@@ -0,0 +1,612 @@
+/*
+ * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Patent licenses, if any, provided herein do not apply to
+ * combinations of this program with other software, or any other
+ * product whatsoever.
+ */
+
+/*
+ * This file provides support for doing sk_buff buffer swapping between
+ * the low level driver eager buffers, and the network layer.  It's part
+ * of the core driver, rather than the ether driver, because it relies
+ * on variables and functions in the core driver.  It exports a single
+ * entry point for use in the ipath_ether module.
+ */
+
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <asm/io.h>
+#include <asm/byteorder.h>
+#include <asm/bitops.h>
+#include <linux/skbuff.h>
+#include <linux/netdevice.h>
+
+#include <linux/crc32.h>        /* we can generate our own crc's for testing */
+
+#include "ipath_kernel.h"
+#include "ips_common.h"
+#include "ipath_layer.h"
+
+/*
+ * Allocate a PIO send buffer, initialize the header and copy it out.
+ */
+static int layer_send_getpiobuf(struct copy_data_s *cdp)
+{
+	uint32_t device = cdp->device;
+	uint32_t extra_bytes;
+	uint32_t len, nwords;
+	uint32_t __iomem *piobuf;
+
+	if (!(piobuf = ipath_getpiobuf(device, NULL))) {
+		cdp->error = -EBUSY;
+		return cdp->error;
+	}
+
+	/*
+	 * Compute the max amount of data that can fit into a PIO buffer.
+	 * buffer size - header size - trigger qword length & flags - CRC
+	 */
+	len = devdata[device].ipath_ibmaxlen -
+		sizeof(struct ether_header_typ) - 8 - (SIZE_OF_CRC << 2);
+	if (len > devdata[device].ipath_rcvegrbufsize)
+		len = devdata[device].ipath_rcvegrbufsize;
+	if (len > (cdp->len + cdp->extra))
+		len = (cdp->len + cdp->extra);
+	/* Compute word aligment (i.e., (len & 3) ? 4 - (len & 3) : 0) */
+	extra_bytes = (4 - len) & 3;
+	nwords = (sizeof(struct ether_header_typ) + len + extra_bytes) >> 2;
+	cdp->hdr->lrh[2] = htons(nwords + SIZE_OF_CRC);
+	cdp->hdr->bth[0] = htonl((OPCODE_ITH4X << 24) + (extra_bytes << 20) +
+		IPS_DEFAULT_P_KEY);
+	cdp->hdr->sub_opcode = OPCODE_ENCAP;
+
+	cdp->hdr->bth[2] = 0;
+	/* Generate an interrupt on the receive side for the last fragment. */
+	cdp->hdr->iph.pkt_flags = ((cdp->len+cdp->extra) == len) ? INFINIPATH_KPF_INTR : 0;
+	cdp->hdr->iph.chksum = (uint16_t) IPS_LRH_BTH +
+		(uint16_t) (nwords + SIZE_OF_CRC) -
+		(uint16_t) ((cdp->hdr->iph.ver_port_tid_offset >> 16)&0xFFFF) -
+		(uint16_t) (cdp->hdr->iph.ver_port_tid_offset & 0xFFFF) -
+		(uint16_t) cdp->hdr->iph.pkt_flags;
+
+	_IPATH_VDBG("send %d (%x %x %x %x %x %x %x)\n", nwords,
+		cdp->hdr->lrh[0], cdp->hdr->lrh[1],
+		cdp->hdr->lrh[2], cdp->hdr->lrh[3],
+		cdp->hdr->bth[0], cdp->hdr->bth[1], cdp->hdr->bth[2]);
+	/*
+	 * Write len to control qword, no flags.
+	 * +1 is for the qword padding of pbc.
+	 */
+	writeq(nwords + 1ULL, (uint64_t __iomem *) piobuf);
+	/* we have to flush after the PBC for correctness on some cpus
+	 * or WC buffer can be written out of order */
+	mb();
+	piobuf += 2;
+	memcpy_toio32(piobuf, cdp->hdr, sizeof(struct ether_header_typ) >> 2);
+	cdp->csum_pio = &((struct ether_header_typ __iomem *) piobuf)->csum;
+	cdp->to = piobuf + (sizeof(struct ether_header_typ) >> 2);
+	cdp->flen = nwords - (sizeof(struct ether_header_typ) >> 2);
+	cdp->hdr->frag_num++;
+	return 0;
+}
+
+/*
+ * copy the last full dword when that's the "extra" word, preceding it
+ * with a memory fence, so that all prior data is written to the PIO
+ * buffer before the trigger word, to enforce the correct bus ordering
+ * of the WC buffer contents on the bus.
+ */
+static inline unsigned copy_extra_dword(struct copy_data_s *cdp, unsigned dosum)
+{
+	if (!cdp->flen && layer_send_getpiobuf(cdp) < 0)
+		return 1;
+	/* write the checksum before the last PIO write, if requested. */
+	if (dosum && cdp->flen == 1)
+		writel(csum_fold(cdp->csum), cdp->csum_pio);
+	mb();
+	writel(cdp->u.w, cdp->to++);
+	mb();
+	cdp->extra = 0;
+	cdp->flen -= 1;
+	return 0;
+}
+
+/*
+ * copy a PIO buffer's worth (or the skb fragment, at least) to the PIO
+ * buffer, adding a memory fence before the last word.  We need the fence
+ * as part of forcing the WC ordering on some cpus, for the cases where
+ * it will be the trigger word.  The final fence after the trigger word
+ * will be done either at the next chunk, or on final return from the caller
+ * Takes max byte count, returns byte count actually done (always rounded
+ * to dword multiple).
+ */
+static uint32_t copy_a_buffer(struct copy_data_s *cdp, void *p, uint32_t n,
+	unsigned dosum)
+{
+	uint32_t *p32;
+
+	if (!cdp->flen && layer_send_getpiobuf(cdp) < 0)
+		return -1;
+	if (n > cdp->flen)
+		n = cdp->flen;
+	if (dosum && cdp->flen == n)
+		writel(csum_fold(cdp->csum), cdp->csum_pio);
+	p32 = p;
+	memcpy_toio32(cdp->to, p32, n-1);
+	cdp->to += n-1;
+	mb();
+	writel(p32[n-1], cdp->to++);
+	mb();
+	_IPATH_PDBG("trigger write to pio %p\n", &p32[n-1]);
+	cdp->flen -= n;
+	n <<= 2;
+	cdp->offset += n;
+	cdp->len -= n;
+	return n;
+}
+
+/*
+ * Copy data out of one or a chain of sk_buffs, into the PIO buffer.
+ * Fragment an sk_buff into multiple IB packets if the amount of data is
+ * more than a single eager send.
+ * Offset and len are in bytes.
+ * Note that this function is recursive!
+ */
+static void copy_bits(const struct sk_buff *skb, unsigned int offset,
+    unsigned int len, struct copy_data_s *cdp)
+{
+	unsigned int start = skb_headlen(skb);
+	unsigned int i, copy;
+	uint32_t n;
+	uint8_t *p;
+
+	/* Copy header. */
+	if ((int)(copy = start - offset) > 0) {
+		if (copy > len)
+			copy = len;
+		p = skb->data + offset;
+		offset += copy;
+		len -= copy;
+		/* If the alignment buffer is not empty, fill it and write it out. */
+		if (cdp->extra) {
+			if (cdp->extra == 4) {
+				if (copy_extra_dword(cdp, 0))
+					return;
+			}
+			else while (copy != 0) {
+				cdp->u.buf[cdp->extra] = *p++;
+				copy--;
+				cdp->offset++;
+				cdp->len--;
+
+				if (++cdp->extra == 4) {
+					if (copy_extra_dword(cdp, 0))
+						return;
+					break;
+				}
+			}
+		}
+		while (copy >= 4) {
+			n = copy_a_buffer(cdp, p, copy>>2, 0);
+			if (n == -1)
+				return;
+			p += n;
+			copy -= n;
+		}
+		/*
+		 * Either cdp->extra is zero or copy is zero which means that
+		 * the loop here can't cause the alignment buffer to fill up.
+		 */
+		while (copy != 0) {
+			cdp->u.buf[cdp->extra++] = *p++;
+			copy--;
+			cdp->offset++;
+			cdp->len--;
+
+		}
+		if (len == 0)
+			return;
+	}
+
+	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+		unsigned int end;
+
+		end = start + frag->size;
+		if ((int)(copy = end - offset) > 0) {
+			uint8_t *vaddr;
+
+			if (copy > len)
+				copy = len;
+			vaddr = kmap_skb_frag(frag);
+			p = vaddr + frag->page_offset + offset - start;
+			offset += copy;
+			len -= copy;
+			/* If the alignment buffer is not empty, fill it and write it out. */
+			if (cdp->extra) {
+				if (cdp->extra == 4) {
+					if (copy_extra_dword(cdp, 0))
+						return;
+				}
+				else while (copy != 0) {
+					cdp->u.buf[cdp->extra] = *p++;
+					copy--;
+					cdp->offset++;
+					cdp->len--;
+
+					if (++cdp->extra == 4) {
+						if (copy_extra_dword(cdp, 0))
+							return;
+						break;
+					}
+				}
+			}
+			while (copy >= 4) {
+				n = copy_a_buffer(cdp, p, copy>>2, 0);
+				if (n == -1)
+					return;
+				p += n;
+				copy -= n;
+			}
+			/*
+			 * Either cdp->extra is zero or copy is zero which means that
+			 * the loop here can't cause the alignment buffer to fill up.
+			 */
+			while (copy != 0) {
+				cdp->u.buf[cdp->extra++] = *p++;
+				copy--;
+				cdp->offset++;
+				cdp->len--;
+			}
+			kunmap_skb_frag(vaddr);
+
+			if (len == 0)
+				return;
+		}
+		start = end;
+	}
+
+	if (skb_shinfo(skb)->frag_list) {
+		struct sk_buff *list = skb_shinfo(skb)->frag_list;
+
+		for (; list; list = list->next) {
+			unsigned int end;
+
+			end = start + list->len;
+			if ((int)(copy = end - offset) > 0) {
+				if (copy > len)
+					copy = len;
+				copy_bits(list, offset - start, copy, cdp);
+				if (cdp->error || (len -= copy) == 0)
+					return;
+			}
+			start = end;
+		}
+	}
+	if (len)
+		cdp->error = -EFAULT;
+}
+
+/*
+ * Copy data out of one or a chain of sk_buffs, into the PIO buffer, generating
+ * the checksum as we go.
+ * Fragment an sk_buff into multiple IB packets if the amount of data is
+ * more than a single eager send.
+ * Offset and len are in bytes.
+ * Note that this function is recursive!
+ */
+static void copy_and_csum_bits(const struct sk_buff *skb, unsigned int offset,
+    unsigned int len, struct copy_data_s *cdp)
+{
+	unsigned int start = skb_headlen(skb);
+	unsigned int i, copy;
+	unsigned int csum2;
+	uint32_t n;
+	uint8_t *p;
+
+	/* Copy header. */
+	if ((int)(copy = start - offset) > 0) {
+		if (copy > len)
+			copy = len;
+		p = skb->data + offset;
+		offset += copy;
+		len -= copy;
+		if (!cdp->checksum_calc) {
+			cdp->checksum_calc = 1;
+
+			csum2 = csum_partial(p, copy, 0);
+			cdp->csum = csum_block_add(cdp->csum, csum2, cdp->pos);
+			cdp->pos += copy;
+		}
+		/* If the alignment buffer is not empty, fill it and write it out. */
+		if (cdp->extra) {
+			if (cdp->extra == 4) {
+				if (copy_extra_dword(cdp, 1))
+					goto done;
+			}
+			else while (copy != 0) {
+				cdp->u.buf[cdp->extra] = *p++;
+				copy--;
+				cdp->offset++;
+				cdp->len--;
+				if (++cdp->extra == 4) {
+					if (copy_extra_dword(cdp, 1))
+						goto done;
+					break;
+				}
+			}
+		}
+
+		while (copy >= 4) {
+			n = copy_a_buffer(cdp, p, copy>>2, 1);
+			if (n == -1)
+				goto done;
+			p += n;
+			copy -= n;
+		}
+		/*
+		 * Either cdp->extra is zero or copy is zero which means that
+		 * the loop here can't cause the alignment buffer to fill up.
+		 */
+		while (copy != 0) {
+			cdp->u.buf[cdp->extra++] = *p++;
+			copy--;
+			cdp->offset++;
+			cdp->len--;
+		}
+
+		cdp->checksum_calc = 0;
+
+		if (len == 0)
+			goto done;
+	}
+
+	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+		unsigned int end;
+
+		end = start + frag->size;
+		if ((int)(copy = end - offset) > 0) {
+			uint8_t *vaddr;
+
+			if (copy > len)
+				copy = len;
+			vaddr = kmap_skb_frag(frag);
+			p = vaddr + frag->page_offset + offset - start;
+			offset += copy;
+			len -= copy;
+
+			if (!cdp->checksum_calc) {
+				cdp->checksum_calc = 1;
+
+				csum2 = csum_partial(p, copy, 0);
+				cdp->csum = csum_block_add(cdp->csum, csum2,
+					cdp->pos);
+				cdp->pos += copy;
+			}
+			/* If the alignment buffer is not empty, fill it and write it out. */
+			if (cdp->extra) {
+				if (cdp->extra == 4) {
+					if (copy_extra_dword(cdp, 1)) {
+						kunmap_skb_frag(vaddr);
+						goto done;
+					}
+				}
+				else while (copy != 0) {
+					cdp->u.buf[cdp->extra] = *p++;
+					copy--;
+					cdp->offset++;
+					cdp->len--;
+
+					if (++cdp->extra == 4) {
+						if (copy_extra_dword(cdp, 1)) {
+							kunmap_skb_frag(vaddr);
+							goto done;
+						}
+						break;
+					}
+				}
+			}
+			while (copy >= 4) {
+				n = copy_a_buffer(cdp, p, copy>>2, 1);
+				if (n == -1) {
+					kunmap_skb_frag(vaddr);
+					goto done;
+				}
+				p += n;
+				copy -= n;
+			}
+			/*
+			 * Either cdp->extra is zero or copy is zero which means that
+			 * the loop here can't cause the alignment buffer to fill up.
+			 */
+			while (copy != 0) {
+				cdp->u.buf[cdp->extra++] = *p++;
+				copy--;
+				cdp->offset++;
+				cdp->len--;
+			}
+			kunmap_skb_frag(vaddr);
+
+			cdp->checksum_calc = 0;
+
+			if (len == 0)
+				goto done;
+		}
+		start = end;
+	}
+
+	if (skb_shinfo(skb)->frag_list) {
+		struct sk_buff *list = skb_shinfo(skb)->frag_list;
+
+		for (; list; list = list->next) {
+			unsigned int end;
+
+			end = start + list->len;
+			if ((int)(copy = end - offset) > 0) {
+				if (copy > len)
+					copy = len;
+				copy_and_csum_bits(list, offset - start, copy, cdp);
+				if (cdp->error || (len -= copy) == 0)
+					goto done;
+				offset += copy;
+			}
+			start = end;
+		}
+	}
+	if (len)
+		cdp->error = -EFAULT;
+done:
+	/* we have to flush after trigger word for correctness on some cpus
+	 * or WC buffer can be written out of order; needed even if
+	 * there was an error */
+	mb();
+}
+
+/*
+ * Note that the header should have the unchanging parts
+ * initialized but the rest of the header is computed as needed in
+ * order to break up skb data buffers larger than the hardware MTU.
+ * In other words, the Linux network stack MTU can be larger than the
+ * hardware MTU.
+ */
+int ipath_layer_send_skb(struct copy_data_s *cdata)
+{
+	int ret = 0;
+	uint16_t vlsllnh;
+	int device = cdata->device;
+
+	if (device >= infinipath_max) {
+		_IPATH_INFO("Invalid unit %u, failing\n", device);
+		return -EINVAL;
+	}
+	if (!(devdata[device].ipath_flags & IPATH_RCVHDRSZ_SET)) {
+		_IPATH_INFO("send while not open\n");
+		ret = -EINVAL;
+	}
+	else if ((devdata[device].ipath_flags & (IPATH_LINKUNK | IPATH_LINKDOWN))
+		|| devdata[device].ipath_lid == 0) {
+		/* lid check is for when sma hasn't yet configured */
+		ret = -ENETDOWN;
+		_IPATH_VDBG("send while not ready, mylid=%u, flags=0x%x\n",
+			devdata[device].ipath_lid, devdata[device].ipath_flags);
+	}
+	vlsllnh = *((uint16_t *) cdata->hdr);
+	if (vlsllnh != htons(IPS_LRH_BTH)) {
+		_IPATH_DBG("Warning: lrh[0] wrong (%x, not %x); not sending\n",
+			vlsllnh, htons(IPS_LRH_BTH));
+		ret = -EINVAL;
+	}
+	if (ret)
+		goto done;
+
+	cdata->error = 0;       /* clear last calls error  */
+
+	if (cdata->skb->ip_summed == CHECKSUM_HW) {
+		unsigned int csstart = cdata->skb->h.raw - cdata->skb->data;
+
+		/*
+		 * Computing the checksum is a bit tricky since if we fragment
+		 * the packet, the fragment that should contain the checksum
+		 * will have already been sent.  The solution is to store the checksum
+		 * in the header of the last fragment just before we write the
+		 * last data word which triggers the last fragment to be sent.
+		 * The receiver will check the header "tag" field, see that
+		 * there is a checksum, and store the checksum back into the packet.
+		 *
+		 * Save the offset of the two byte checksum.
+		 * Note that we have to add 2 to account for the two bytes of the
+		 * ethernet address we stripped from the packet and put in the header.
+		 */
+		cdata->hdr->csum_offset = csstart + cdata->skb->csum + 2;
+
+		if (cdata->offset < csstart)
+			copy_bits(cdata->skb, cdata->offset,
+				csstart - cdata->offset, cdata);
+
+		if (cdata->error) {
+			ret = cdata->error;
+			goto done;
+		}
+
+		if (cdata->offset < cdata->skb->len)
+			copy_and_csum_bits(cdata->skb, cdata->offset,
+				cdata->skb->len - cdata->offset, cdata);
+
+		if (cdata->error) {
+			ret = cdata->error;
+			goto done;
+		}
+
+		if (cdata->extra) {
+			while (cdata->extra < 4)
+				cdata->u.buf[cdata->extra++] = 0;
+			(void)copy_extra_dword(cdata, 1);
+		}
+	}
+	else {
+		copy_bits(cdata->skb, cdata->offset,
+			cdata->skb->len - cdata->offset, cdata);
+
+		if (cdata->error) {
+			ret = cdata->error;
+			goto done;
+		}
+
+		if (cdata->extra) {
+			while (cdata->extra < 4)
+				cdata->u.buf[cdata->extra++] = 0;
+			(void)copy_extra_dword(cdata, 1);
+		}
+	}
+
+	if (cdata->error) {
+		ret = cdata->error;
+		if (cdata->error != -EBUSY)
+			_IPATH_UNIT_ERROR(device,
+				"layer_send copy_bits failed with error %d\n",
+				-ret);
+	}
+
+	ipath_stats.sps_ether_spkts++;  /* another ether packet sent */
+
+done:
+	/* we have to flush after trigger word for correctness on some cpus
+	 * or WC buffer can be written out of order; needed even if
+	 * there was an error */
+	mb();
+	return ret;
+}
+
+EXPORT_SYMBOL(ipath_layer_send_skb);
+

diff -r 2d9a3f27a10c -r 9e8d017ed298 drivers/infiniband/hw/ipath/ipath_debug.h
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Patent licenses, if any, provided herein do not apply to
+ * combinations of this program with other software, or any other
+ * product whatsoever.
+ */
+
+#ifndef _IPATH_DEBUG_H
+#define _IPATH_DEBUG_H
+
+#ifndef _IPATH_DEBUGGING	/* debugging enabled or not */
+#define _IPATH_DEBUGGING 1
+#endif
+
+#if _IPATH_DEBUGGING
+
+/*
+ * Mask values for debugging.  The scheme allows us to compile out any
+ * of the debug tracing stuff, and if compiled in, to enable or disable
+ * dynamically.  This can be set at modprobe time also:
+ *      modprobe infinipath.ko infinipath_debug=7
+ */
+
+#define __IPATH_INFO        0x1 /* generic low verbosity stuff */
+#define __IPATH_DBG         0x2 /* generic debug */
+#define __IPATH_TRSAMPLE    0x8 /* generate trace buffer sample entries */
+/* leave some low verbosity spots open */
+#define __IPATH_VERBDBG     0x40        /* very verbose debug */
+#define __IPATH_PKTDBG      0x80        /* print packet data */
+/* print process startup (init)/exit messages */
+#define __IPATH_PROCDBG     0x100
+/* print mmap/nopage stuff, not using VDBG any more */
+#define __IPATH_MMDBG       0x200
+#define __IPATH_USER_SEND   0x1000      /* use user mode send */
+#define __IPATH_KERNEL_SEND 0x2000      /* use kernel mode send */
+#define __IPATH_EPKTDBG     0x4000      /* print ethernet packet data */
+#define __IPATH_SMADBG      0x8000      /* sma packet debug */
+#define __IPATH_IPATHDBG    0x10000     /* Ethernet (IPATH) general debug on */
+#define __IPATH_IPATHWARN   0x20000     /* Ethernet (IPATH) warnings on */
+#define __IPATH_IPATHERR    0x40000     /* Ethernet (IPATH) errors on */
+#define __IPATH_IPATHPD     0x80000     /* Ethernet (IPATH) packet dump on */
+#define __IPATH_IPATHTABLE  0x100000    /* Ethernet (IPATH) table dump on */
+
+#else /* _IPATH_DEBUGGING */
+
+/*
+ * define all of these even with debugging off, for the few places that do
+ * if(infinipath_debug & _IPATH_xyzzy), but in a way that will make the
+ * compiler eliminate the code
+ */
+
+#define __IPATH_INFO      0x0   /* generic low verbosity stuff */
+#define __IPATH_DBG       0x0   /* generic debug */
+#define __IPATH_TRSAMPLE  0x0   /* generate trace buffer sample entries */
+#define __IPATH_VERBDBG   0x0   /* very verbose debug */
+#define __IPATH_PKTDBG    0x0   /* print packet data */
+#define __IPATH_PROCDBG   0x0   /* print process startup (init)/exit messages */
+/* print mmap/nopage stuff, not using VDBG any more */
+#define __IPATH_MMDBG     0x0
+#define __IPATH_EPKTDBG   0x0   /* print ethernet packet data */
+#define __IPATH_SMADBG    0x0   /* print process startup (init)/exit messages */#define __IPATH_IPATHDBG  0x0   /* Ethernet (IPATH) table dump on */
+#define __IPATH_IPATHWARN 0x0   /* Ethernet (IPATH) warnings on   */
+#define __IPATH_IPATHERR  0x0   /* Ethernet (IPATH) errors on   */
+#define __IPATH_IPATHPD   0x0   /* Ethernet (IPATH) packet dump on   */
+#define __IPATH_IPATHTABLE 0x0  /* Ethernet (IPATH) packet dump on   */
+
+#endif /* _IPATH_DEBUGGING */
+
+#endif /* _IPATH_DEBUG_H */

109 lines not on this page · raw message

diff -r ef833f6712e7 -r 801287704e40 arch/x86_64/kernel/x8664_ksyms.c
@@ -164,6 +164,8 @@
 EXPORT_SYMBOL(memcpy);
 EXPORT_SYMBOL(__memcpy);
 
+EXPORT_SYMBOL_GPL(memcpy32);
+
 #ifdef CONFIG_RWSEM_XCHGADD_ALGORITHM
 /* prototypes are wrong, these are assembly with custom calling functions */
 extern void rwsem_down_read_failed_thunk(void);

diff -r ef833f6712e7 -r 801287704e40 include/asm-x86_64/string.h
@@ -45,6 +45,15 @@
 #define __HAVE_ARCH_MEMMOVE
 void * memmove(void * dest,const void *src,size_t count);
 
+/*
+ * memcpy32 - copy data, 32 bits at a time
+ *
+ * @dst: destination (must be 32-bit aligned)
+ * @src: source (must be 32-bit aligned)
+ * @count: number of 32-bit quantities to copy
+ */
+void memcpy32(void *dst, const void *src, size_t count);
+
 /* Use C out of line version for memcmp */ 
 #define memcmp __builtin_memcmp
 int memcmp(const void * cs,const void * ct,size_t count);

diff -r 26993cb5faee -r 471b7a7a005c drivers/infiniband/hw/ipath/ipath_verbs.c
@@ -0,0 +1,2307 @@
+/*
+ * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Patent licenses, if any, provided herein do not apply to
+ * combinations of this program with other software, or any other
+ * product whatsoever.
+ */
+
+#include <linux/config.h>
+#include <linux/version.h>
+#include <linux/pci.h>
+#include <linux/err.h>
+#include <rdma/ib_pack.h>
+#include <rdma/ib_smi.h>
+#include <rdma/ib_mad.h>
+#include <rdma/ib_user_verbs.h>
+
+#include <asm/uaccess.h>
+#include <asm-generic/bug.h>
+
+#include "ips_common.h"
+#include "ipath_layer.h"
+#include "ipath_verbs.h"
+
+/*
+ * Compare the lower 24 bits of the two values.
+ * Returns an integer <, ==, or > than zero.
+ */
+static inline int cmp24(u32 a, u32 b)
+{
+	return (((int) a) - ((int) b)) << 8;
+}
+
+#define MODNAME "ib_ipath"
+#define DRIVER_LOAD_MSG "PathScale " MODNAME " loaded: "
+#define PFX MODNAME ": "
+
+
+#define BITS_PER_PAGE           (PAGE_SIZE*BITS_PER_BYTE)
+#define BITS_PER_PAGE_MASK      (BITS_PER_PAGE-1)
+#define mk_qpn(qpt, map, off)   (((map) - (qpt)->map)*BITS_PER_PAGE + (off))
+#define find_next_offset(map, off)                                      \
+                find_next_zero_bit((map)->page, BITS_PER_PAGE, off)
+
+/* Not static, because we don't want the compiler removing it */
+const char ipath_verbs_version[] = "ipath_verbs " IPATH_IDSTR;
+
+unsigned int ib_ipath_qp_table_size = 251;
+module_param(ib_ipath_qp_table_size, uint, 0444);
+MODULE_PARM_DESC(ib_ipath_qp_table_size, "QP table size");
+
+unsigned int ib_ipath_lkey_table_size = 12;
+module_param(ib_ipath_lkey_table_size, uint, 0444);
+MODULE_PARM_DESC(ib_ipath_lkey_table_size,
+		 "LKEY table size in bits (2^n, 1 <= n <= 23)");
+
+unsigned int ib_ipath_debug;	/* debug mask */
+module_param(ib_ipath_debug, uint, 0644);
+MODULE_PARM_DESC(ib_ipath_debug, "Verbs debug mask");
+
+
+static void ipath_ud_loopback(struct ipath_qp *sqp, struct ipath_sge_state *ss,
+			      u32 len, struct ib_send_wr *wr, struct ib_wc *wc);
+static void ipath_ruc_loopback(struct ipath_qp *sqp, struct ib_wc *wc);
+static int ipath_destroy_qp(struct ib_qp *ibqp);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("PathScale <[email protected]>");
+MODULE_DESCRIPTION("Pathscale InfiniPath driver");
+
+enum {
+	IPATH_FAULT_RC_DROP_SEND_F = 1,
+	IPATH_FAULT_RC_DROP_SEND_M,
+	IPATH_FAULT_RC_DROP_SEND_L,
+	IPATH_FAULT_RC_DROP_SEND_O,
+	IPATH_FAULT_RC_DROP_RDMA_WRITE_F,
+	IPATH_FAULT_RC_DROP_RDMA_WRITE_M,
+	IPATH_FAULT_RC_DROP_RDMA_WRITE_L,
+	IPATH_FAULT_RC_DROP_RDMA_WRITE_O,
+	IPATH_FAULT_RC_DROP_RDMA_READ_RESP_F,
+	IPATH_FAULT_RC_DROP_RDMA_READ_RESP_M,
+	IPATH_FAULT_RC_DROP_RDMA_READ_RESP_L,
+	IPATH_FAULT_RC_DROP_RDMA_READ_RESP_O,
+	IPATH_FAULT_RC_DROP_ACK,
+};
+
+enum {
+	IPATH_TRANS_INVALID = 0,
+	IPATH_TRANS_ANY2RST,
+	IPATH_TRANS_RST2INIT,
+	IPATH_TRANS_INIT2INIT,
+	IPATH_TRANS_INIT2RTR,
+	IPATH_TRANS_RTR2RTS,
+	IPATH_TRANS_RTS2RTS,
+	IPATH_TRANS_SQERR2RTS,
+	IPATH_TRANS_ANY2ERR,
+	IPATH_TRANS_RTS2SQD,	/* XXX Wait for expected ACKs & signal event */
+	IPATH_TRANS_SQD2SQD,	/* error if not drained & parameter change */
+	IPATH_TRANS_SQD2RTS,	/* error if not drained */
+};
+
+enum {
+	IPATH_POST_SEND_OK = 0x0001,
+	IPATH_POST_RECV_OK = 0x0002,
+	IPATH_PROCESS_RECV_OK = 0x0004,
+	IPATH_PROCESS_SEND_OK = 0x0008,
+};
+
+static int state_ops[IB_QPS_ERR + 1] = {
+	[IB_QPS_RESET] = 0,
+	[IB_QPS_INIT] = IPATH_POST_RECV_OK,
+	[IB_QPS_RTR] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK,
+	[IB_QPS_RTS] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK |
+	    IPATH_POST_SEND_OK | IPATH_PROCESS_SEND_OK,
+	[IB_QPS_SQD] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK |
+	    IPATH_POST_SEND_OK,
+	[IB_QPS_SQE] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK,
+	[IB_QPS_ERR] = 0,
+};
+
+/*
+ * Convert the AETH credit code into the number of credits.
+ */
+static u32 credit_table[31] = {
+	0,			/* 0 */
+	1,			/* 1 */
+	2,			/* 2 */
+	3,			/* 3 */
+	4,			/* 4 */
+	6,			/* 5 */
+	8,			/* 6 */
+	12,			/* 7 */
+	16,			/* 8 */
+	24,			/* 9 */
+	32,			/* A */
+	48,			/* B */
+	64,			/* C */
+	96,			/* D */
+	128,			/* E */
+	192,			/* F */
+	256,			/* 10 */
+	384,			/* 11 */
+	512,			/* 12 */
+	768,			/* 13 */
+	1024,			/* 14 */
+	1536,			/* 15 */
+	2048,			/* 16 */
+	3072,			/* 17 */
+	4096,			/* 18 */
+	6144,			/* 19 */
+	8192,			/* 1A */
+	12288,			/* 1B */
+	16384,			/* 1C */
+	24576,			/* 1D */
+	32768			/* 1E */
+};
+
+/*
+ * Convert the AETH RNR timeout code into the number of milliseconds.
+ */
+static u32 rnr_table[32] = {
+	656,			/* 0 */
+	1,			/* 1 */
+	1,			/* 2 */
+	1,			/* 3 */
+	1,			/* 4 */
+	1,			/* 5 */
+	1,			/* 6 */
+	1,			/* 7 */
+	1,			/* 8 */
+	1,			/* 9 */
+	1,			/* A */
+	1,			/* B */
+	1,			/* C */
+	1,			/* D */
+	2,			/* E */
+	2,			/* F */
+	3,			/* 10 */
+	4,			/* 11 */
+	6,			/* 12 */
+	8,			/* 13 */
+	11,			/* 14 */
+	16,			/* 15 */
+	21,			/* 16 */
+	31,			/* 17 */
+	41,			/* 18 */
+	62,			/* 19 */
+	82,			/* 1A */
+	123,			/* 1B */
+	164,			/* 1C */
+	246,			/* 1D */
+	328,			/* 1E */
+	492			/* 1F */
+};
+
+/*
+ * Translate ib_wr_opcode into ib_wc_opcode.
+ */
+static enum ib_wc_opcode wc_opcode[] = {
+	[IB_WR_RDMA_WRITE] = IB_WC_RDMA_WRITE,
+	[IB_WR_RDMA_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE,
+	[IB_WR_SEND] = IB_WC_SEND,
+	[IB_WR_SEND_WITH_IMM] = IB_WC_SEND,
+	[IB_WR_RDMA_READ] = IB_WC_RDMA_READ,
+	[IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP,
+	[IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD
+};
+
+/*
+ * Array of device pointers.
+ */
+static uint32_t number_of_devices;
+static struct ipath_ibdev **ipath_devices;
+
+/*
+ * Global table of GID to attached QPs.
+ * The table is global to all ipath devices since a send from one QP/device
+ * needs to be locally routed to any locally attached QPs on the same
+ * or different device.
+ */
+static struct rb_root mcast_tree;
+static spinlock_t mcast_lock = SPIN_LOCK_UNLOCKED;
+
+/*
+ * Allocate a structure to link a QP to the multicast GID structure.
+ */
+static struct ipath_mcast_qp *ipath_mcast_qp_alloc(struct ipath_qp *qp)
+{
+	struct ipath_mcast_qp *mqp;
+
+	mqp = kmalloc(sizeof(*mqp), GFP_KERNEL);
+	if (!mqp)
+		return NULL;
+
+	mqp->qp = qp;
+	atomic_inc(&qp->refcount);
+
+	return mqp;
+}
+
+static void ipath_mcast_qp_free(struct ipath_mcast_qp *mqp)
+{
+	struct ipath_qp *qp = mqp->qp;
+
+	/* Notify ipath_destroy_qp() if it is waiting. */
+	if (atomic_dec_and_test(&qp->refcount))
+		wake_up(&qp->wait);
+
+	kfree(mqp);
+}
+
+/*
+ * Allocate a structure for the multicast GID.
+ * A list of QPs will be attached to this structure.
+ */
+static struct ipath_mcast *ipath_mcast_alloc(union ib_gid *mgid)
+{
+	struct ipath_mcast *mcast;
+
+	mcast = kmalloc(sizeof(*mcast), GFP_KERNEL);
+	if (!mcast)
+		return NULL;
+
+	mcast->mgid = *mgid;
+	INIT_LIST_HEAD(&mcast->qp_list);
+	init_waitqueue_head(&mcast->wait);
+	atomic_set(&mcast->refcount, 0);
+
+	return mcast;
+}
+
+static void ipath_mcast_free(struct ipath_mcast *mcast)
+{
+	struct ipath_mcast_qp *p, *tmp;
+
+	list_for_each_entry_safe(p, tmp, &mcast->qp_list, list)
+		ipath_mcast_qp_free(p);
+
+	kfree(mcast);
+}
+
+/*
+ * Search the global table for the given multicast GID.
+ * Return it or NULL if not found.
+ * The caller is responsible for decrementing the reference count if found.
+ */
+static struct ipath_mcast *ipath_mcast_find(union ib_gid *mgid)
+{
+	struct rb_node *n;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mcast_lock, flags);
+	n = mcast_tree.rb_node;
+	while (n) {
+		struct ipath_mcast *mcast;
+		int ret;
+
+		mcast = rb_entry(n, struct ipath_mcast, rb_node);
+
+		ret = memcmp(mgid->raw, mcast->mgid.raw, sizeof(union ib_gid));
+		if (ret < 0)
+			n = n->rb_left;
+		else if (ret > 0)
+			n = n->rb_right;
+		else {
+			atomic_inc(&mcast->refcount);
+			spin_unlock_irqrestore(&mcast_lock, flags);
+			return mcast;
+		}
+	}
+	spin_unlock_irqrestore(&mcast_lock, flags);
+
+	return NULL;
+}
+
+/*
+ * Insert the multicast GID into the table and
+ * attach the QP structure.
+ * Return zero if both were added.
+ * Return EEXIST if the GID was already in the table but the QP was added.
+ * Return ESRCH if the QP was already attached and neither structure was added.
+ */
+static int ipath_mcast_add(struct ipath_mcast *mcast,
+			   struct ipath_mcast_qp *mqp)
+{
+	struct rb_node **n = &mcast_tree.rb_node;
+	struct rb_node *pn = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mcast_lock, flags);
+
+	while (*n) {
+		struct ipath_mcast *tmcast;
+		struct ipath_mcast_qp *p;
+		int ret;
+
+		pn = *n;
+		tmcast = rb_entry(pn, struct ipath_mcast, rb_node);
+
+		ret = memcmp(mcast->mgid.raw, tmcast->mgid.raw,
+			     sizeof(union ib_gid));
+		if (ret < 0) {
+			n = &pn->rb_left;
+			continue;
+		}
+		if (ret > 0) {
+			n = &pn->rb_right;
+			continue;
+		}
+
+		/* Search the QP list to see if this is already there. */
+		list_for_each_entry_rcu(p, &tmcast->qp_list, list) {
+			if (p->qp == mqp->qp) {
+				spin_unlock_irqrestore(&mcast_lock, flags);
+				return ESRCH;
+			}
+		}
+		list_add_tail_rcu(&mqp->list, &tmcast->qp_list);
+		spin_unlock_irqrestore(&mcast_lock, flags);
+		return EEXIST;
+	}
+
+	list_add_tail_rcu(&mqp->list, &mcast->qp_list);
+
+	atomic_inc(&mcast->refcount);
+	rb_link_node(&mcast->rb_node, pn, n);
+	rb_insert_color(&mcast->rb_node, &mcast_tree);
+
+	spin_unlock_irqrestore(&mcast_lock, flags);
+
+	return 0;
+}
+
+static int ipath_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid,
+				  u16 lid)
+{
+	struct ipath_qp *qp = to_iqp(ibqp);
+	struct ipath_mcast *mcast;
+	struct ipath_mcast_qp *mqp;
+
+	/*
+	 * Allocate data structures since its better to do this outside of
+	 * spin locks and it will most likely be needed.
+	 */
+	mcast = ipath_mcast_alloc(gid);
+	if (mcast == NULL)
+		return -ENOMEM;
+	mqp = ipath_mcast_qp_alloc(qp);
+	if (mqp == NULL) {
+		ipath_mcast_free(mcast);
+		return -ENOMEM;
+	}
+	switch (ipath_mcast_add(mcast, mqp)) {
+	case ESRCH:
+		/* Neither was used: can't attach the same QP twice. */
+		ipath_mcast_qp_free(mqp);
+		ipath_mcast_free(mcast);
+		return -EINVAL;
+	case EEXIST:		/* The mcast wasn't used */
+		ipath_mcast_free(mcast);
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int ipath_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid,
+				  u16 lid)
+{
+	struct ipath_qp *qp = to_iqp(ibqp);
+	struct ipath_mcast *mcast = NULL;
+	struct ipath_mcast_qp *p, *tmp;
+	struct rb_node *n;
+	unsigned long flags;
+	int last = 0;
+
+	spin_lock_irqsave(&mcast_lock, flags);
+
+	/* Find the GID in the mcast table. */
+	n = mcast_tree.rb_node;
+	while (1) {
+		int ret;
+
+		if (n == NULL) {
+			spin_unlock_irqrestore(&mcast_lock, flags);
+			return 0;
+		}
+
+		mcast = rb_entry(n, struct ipath_mcast, rb_node);
+		ret = memcmp(gid->raw, mcast->mgid.raw, sizeof(union ib_gid));
+		if (ret < 0)
+			n = n->rb_left;
+		else if (ret > 0)
+			n = n->rb_right;
+		else
+			break;
+	}
+
+	/* Search the QP list. */
+	list_for_each_entry_safe(p, tmp, &mcast->qp_list, list) {
+		if (p->qp != qp)
+			continue;
+		/*
+		 * We found it, so remove it, but don't poison the forward link
+		 * until we are sure there are no list walkers.
+		 */
+		list_del_rcu(&p->list);
+
+		/* If this was the last attached QP, remove the GID too. */
+		if (list_empty(&mcast->qp_list)) {
+			rb_erase(&mcast->rb_node, &mcast_tree);
+			last = 1;
+		}
+		break;
+	}
+
+	spin_unlock_irqrestore(&mcast_lock, flags);
+
+	if (p) {
+		/*
+		 * Wait for any list walkers to finish before freeing the
+		 * list element.
+		 */
+		wait_event(mcast->wait, atomic_read(&mcast->refcount) <= 1);
+		ipath_mcast_qp_free(p);
+	}
+	if (last) {
+		atomic_dec(&mcast->refcount);
+		wait_event(mcast->wait, !atomic_read(&mcast->refcount));
+		ipath_mcast_free(mcast);
+	}
+
+	return 0;
+}
+
+/*
+ * Copy data to SGE memory.
+ */
+static void copy_sge(struct ipath_sge_state *ss, void *data, u32 length)
+{
+	struct ipath_sge *sge = &ss->sge;
+
+	while (length) {
+		u32 len = sge->length;
+
+		BUG_ON(len == 0);
+		if (len > length)
+			len = length;
+		memcpy(sge->vaddr, data, len);
+		sge->vaddr += len;
+		sge->length -= len;
+		sge->sge_length -= len;
+		if (sge->sge_length == 0) {
+			if (--ss->num_sge)
+				*sge = *ss->sg_list++;
+		} else if (sge->length == 0 && sge->mr != NULL) {
+			if (++sge->n >= IPATH_SEGSZ) {
+				if (++sge->m >= sge->mr->mapsz)
+					break;
+				sge->n = 0;
+			}
+			sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
+			sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
+		}
+		data += len;
+		length -= len;
+	}
+}
+
+/*
+ * Skip over length bytes of SGE memory.
+ */
+static void skip_sge(struct ipath_sge_state *ss, u32 length)
+{
+	struct ipath_sge *sge = &ss->sge;
+
+	while (length > sge->sge_length) {
+		length -= sge->sge_length;
+		ss->sge = *ss->sg_list++;
+	}
+	while (length) {
+		u32 len = sge->length;
+
+		BUG_ON(len == 0);
+		if (len > length)
+			len = length;
+		sge->vaddr += len;
+		sge->length -= len;
+		sge->sge_length -= len;
+		if (sge->sge_length == 0) {
+			if (--ss->num_sge)
+				*sge = *ss->sg_list++;
+		} else if (sge->length == 0 && sge->mr != NULL) {
+			if (++sge->n >= IPATH_SEGSZ) {
+				if (++sge->m >= sge->mr->mapsz)
+					break;
+				sge->n = 0;
+			}
+			sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
+			sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
+		}
+		length -= len;
+	}
+}
+
+static inline u32 alloc_qpn(struct ipath_qp_table *qpt)
+{
+	u32 i, offset, max_scan, qpn;
+	struct qpn_map *map;
+
+	qpn = qpt->last + 1;
+	if (qpn >= QPN_MAX)
+		qpn = 2;
+	offset = qpn & BITS_PER_PAGE_MASK;
+	map = &qpt->map[qpn / BITS_PER_PAGE];
+	max_scan = qpt->nmaps - !offset;
+	for (i = 0;;) {
+		if (unlikely(!map->page)) {
+			unsigned long page = get_zeroed_page(GFP_KERNEL);
+			unsigned long flags;
+
+			/*
+			 * Free the page if someone raced with us
+			 * installing it:
+			 */
+			spin_lock_irqsave(&qpt->lock, flags);
+			if (map->page)
+				free_page(page);
+			else
+				map->page = (void *)page;
+			spin_unlock_irqrestore(&qpt->lock, flags);
+			if (unlikely(!map->page))
+				break;
+		}
+		if (likely(atomic_read(&map->n_free))) {
+			do {
+				if (!test_and_set_bit(offset, map->page)) {
+					atomic_dec(&map->n_free);
+					qpt->last = qpn;
+					return qpn;
+				}
+				offset = find_next_offset(map, offset);
+				qpn = mk_qpn(qpt, map, offset);
+				/*
+				 * This test differs from alloc_pidmap().
+				 * If find_next_offset() does find a zero bit,
+				 * we don't need to check for QPN wrapping
+				 * around past our starting QPN.  We
+				 * just need to be sure we don't loop forever.
+				 */
+			} while (offset < BITS_PER_PAGE && qpn < QPN_MAX);
+		}
+		/*
+		 * In order to keep the number of pages allocated to a minimum,
+		 * we scan the all existing pages before increasing the size
+		 * of the bitmap table.
+		 */
+		if (++i > max_scan) {
+			if (qpt->nmaps == QPNMAP_ENTRIES)
+				break;
+			map = &qpt->map[qpt->nmaps++];
+			offset = 0;
+		} else if (map < &qpt->map[qpt->nmaps]) {
+			++map;
+			offset = 0;
+		} else {
+			map = &qpt->map[0];
+			offset = 2;
+		}
+		qpn = mk_qpn(qpt, map, offset);
+	}
+	return 0;
+}
+
+static inline void free_qpn(struct ipath_qp_table *qpt, u32 qpn)
+{
+	struct qpn_map *map;
+
+	map = qpt->map + qpn / BITS_PER_PAGE;
+	if (map->page)
+		clear_bit(qpn & BITS_PER_PAGE_MASK, map->page);
+	atomic_inc(&map->n_free);
+}
+
+/*
+ * Allocate the next available QPN and put the QP into the hash table.
+ * The hash table holds a reference to the QP.
+ */
+static int ipath_alloc_qpn(struct ipath_qp_table *qpt, struct ipath_qp *qp,
+			   enum ib_qp_type type)
+{
+	unsigned long flags;
+	u32 qpn;
+
+	if (type == IB_QPT_SMI)
+		qpn = 0;
+	else if (type == IB_QPT_GSI)
+		qpn = 1;
+	else {
+		/* Allocate the next available QPN */
+		qpn = alloc_qpn(qpt);
+		if (qpn == 0) {
+			return -ENOMEM;
+		}
+	}
+	qp->ibqp.qp_num = qpn;
+
+	/* Add the QP to the hash table. */
+	spin_lock_irqsave(&qpt->lock, flags);
+
+	qpn %= qpt->max;
+	qp->next = qpt->table[qpn];
+	qpt->table[qpn] = qp;
+	atomic_inc(&qp->refcount);
+
+	spin_unlock_irqrestore(&qpt->lock, flags);
+	return 0;
+}
+
+/*
+ * Remove the QP from the table so it can't be found asynchronously by
+ * the receive interrupt routine.
+ */
+static void ipath_free_qp(struct ipath_qp_table *qpt, struct ipath_qp *qp)
+{
+	struct ipath_qp *q, **qpp;
+	unsigned long flags;
+	int fnd = 0;
+
+	spin_lock_irqsave(&qpt->lock, flags);
+
+	/* Remove QP from the hash table. */
+	qpp = &qpt->table[qp->ibqp.qp_num % qpt->max];
+	for (; (q = *qpp) != NULL; qpp = &q->next) {
+		if (q == qp) {
+			*qpp = qp->next;
+			qp->next = NULL;
+			atomic_dec(&qp->refcount);
+			fnd = 1;
+			break;
+		}
+	}
+
+	spin_unlock_irqrestore(&qpt->lock, flags);
+
+	if (!fnd)
+		return;
+
+	/* If QPN is not reserved, mark QPN free in the bitmap. */
+	if (qp->ibqp.qp_num > 1)
+		free_qpn(qpt, qp->ibqp.qp_num);
+
+	wait_event(qp->wait, !atomic_read(&qp->refcount));
+}
+
+/*
+ * Remove all QPs from the table.
+ */
+static void ipath_free_all_qps(struct ipath_qp_table *qpt)
+{
+	unsigned long flags;
+	struct ipath_qp *qp, *nqp;
+	u32 n;
+
+	for (n = 0; n < qpt->max; n++) {
+		spin_lock_irqsave(&qpt->lock, flags);
+		qp = qpt->table[n];
+		qpt->table[n] = NULL;
+		spin_unlock_irqrestore(&qpt->lock, flags);
+
+		while (qp) {
+			nqp = qp->next;
+			if (qp->ibqp.qp_num > 1)
+				free_qpn(qpt, qp->ibqp.qp_num);
+			if (!atomic_dec_and_test(&qp->refcount) ||
+			    !ipath_destroy_qp(&qp->ibqp))
+				_VERBS_INFO("QP memory leak!\n");
+			qp = nqp;
+		}
+	}
+
+	for (n = 0; n < ARRAY_SIZE(qpt->map); n++) {
+		if (qpt->map[n].page)
+			free_page((unsigned long)qpt->map[n].page);
+	}
+}
+
+/*
+ * Return the QP with the given QPN.
+ * The caller is responsible for decrementing the QP reference count when done.
+ */
+static struct ipath_qp *ipath_lookup_qpn(struct ipath_qp_table *qpt, u32 qpn)
+{
+	unsigned long flags;
+	struct ipath_qp *qp;
+
+	spin_lock_irqsave(&qpt->lock, flags);
+
+	for (qp = qpt->table[qpn % qpt->max]; qp; qp = qp->next) {
+		if (qp->ibqp.qp_num == qpn) {
+			atomic_inc(&qp->refcount);
+			break;
+		}
+	}
+
+	spin_unlock_irqrestore(&qpt->lock, flags);
+	return qp;
+}
+
+static int ipath_alloc_lkey(struct ipath_lkey_table *rkt,
+			    struct ipath_mregion *mr)
+{
+	unsigned long flags;
+	u32 r;
+	u32 n;
+
+	spin_lock_irqsave(&rkt->lock, flags);
+
+	/* Find the next available LKEY */
+	r = n = rkt->next;
+	for (;;) {
+		if (rkt->table[r] == NULL)
+			break;
+		r = (r + 1) & (rkt->max - 1);
+		if (r == n) {
+			spin_unlock_irqrestore(&rkt->lock, flags);
+			_VERBS_INFO("LKEY table full\n");
+			return 0;
+		}
+	}
+	rkt->next = (r + 1) & (rkt->max - 1);
+	/*
+	 * Make sure lkey is never zero which is reserved to indicate an
+	 * unrestricted LKEY.
+	 */
+	rkt->gen++;
+	mr->lkey = (r << (32 - ib_ipath_lkey_table_size)) |
+	    ((((1 << (24 - ib_ipath_lkey_table_size)) - 1) & rkt->gen) << 8);
+	if (mr->lkey == 0) {
+		mr->lkey |= 1 << 8;
+		rkt->gen++;
+	}
+	rkt->table[r] = mr;
+	spin_unlock_irqrestore(&rkt->lock, flags);
+
+	return 1;
+}
+
+static void ipath_free_lkey(struct ipath_lkey_table *rkt, u32 lkey)
+{
+	unsigned long flags;
+	u32 r;
+
+	if (lkey == 0)
+		return;
+	r = lkey >> (32 - ib_ipath_lkey_table_size);
+	spin_lock_irqsave(&rkt->lock, flags);
+	rkt->table[r] = NULL;
+	spin_unlock_irqrestore(&rkt->lock, flags);
+}
+
+/*
+ * Check the IB SGE for validity and initialize our internal version of it.
+ * Return 1 if OK, else zero.
+ */
+static int ipath_lkey_ok(struct ipath_lkey_table *rkt, struct ipath_sge *isge,
+			 struct ib_sge *sge, int acc)
+{
+	struct ipath_mregion *mr;
+	size_t off;
+
+	/*
+	 * We use LKEY == zero to mean a physical kmalloc() address.
+	 * This is a bit of a hack since we rely on dma_map_single()
+	 * being reversible by calling bus_to_virt().
+	 */
+	if (sge->lkey == 0) {
+		isge->mr = NULL;
+		isge->vaddr = bus_to_virt(sge->addr);
+		isge->length = sge->length;
+		isge->sge_length = sge->length;
+		return 1;
+	}
+	spin_lock(&rkt->lock);
+	mr = rkt->table[(sge->lkey >> (32 - ib_ipath_lkey_table_size))];
+	spin_unlock(&rkt->lock);
+	if (unlikely(mr == NULL || mr->lkey != sge->lkey))
+		return 0;
+
+	off = sge->addr - mr->user_base;
+	if (unlikely(sge->addr < mr->user_base ||
+		     off + sge->length > mr->length ||
+		     (mr->access_flags & acc) != acc))
+		return 0;
+
+	off += mr->offset;
+	isge->mr = mr;
+	isge->m = 0;
+	isge->n = 0;
+	while (off >= mr->map[isge->m]->segs[isge->n].length) {
+		off -= mr->map[isge->m]->segs[isge->n].length;
+		if (++isge->n >= IPATH_SEGSZ) {
+			isge->m++;
+			isge->n = 0;
+		}
+	}
+	isge->vaddr = mr->map[isge->m]->segs[isge->n].vaddr + off;
+	isge->length = mr->map[isge->m]->segs[isge->n].length - off;
+	isge->sge_length = sge->length;
+	return 1;
+}
+
+/*
+ * Initialize the qp->s_sge after a restart.
+ * The QP s_lock should be held.
+ */
+static void ipath_init_restart(struct ipath_qp *qp, struct ipath_swqe *wqe)
+{
+	struct ipath_ibdev *dev;
+	u32 len;
+
+	len = ((qp->s_psn - wqe->psn) & 0xFFFFFF) *
+	    ib_mtu_enum_to_int(qp->path_mtu);
+	qp->s_sge.sge = wqe->sg_list[0];
+	qp->s_sge.sg_list = wqe->sg_list + 1;
+	qp->s_sge.num_sge = wqe->wr.num_sge;
+	skip_sge(&qp->s_sge, len);
+	qp->s_len = wqe->length - len;
+	dev = to_idev(qp->ibqp.device);
+	spin_lock(&dev->pending_lock);
+	if (qp->timerwait.next == LIST_POISON1)
+		list_add_tail(&qp->timerwait,
+			      &dev->pending[dev->pending_index]);
+	spin_unlock(&dev->pending_lock);
+}
+
+/*
+ * Check the IB virtual address, length, and RKEY.
+ * Return 1 if OK, else zero.
+ * The QP r_rq.lock should be held.
+ */
+static int ipath_rkey_ok(struct ipath_ibdev *dev, struct ipath_sge_state *ss,
+			 u32 len, u64 vaddr, u32 rkey, int acc)
+{
+	struct ipath_lkey_table *rkt = &dev->lk_table;
+	struct ipath_sge *sge = &ss->sge;
+	struct ipath_mregion *mr;
+	size_t off;
+
+	spin_lock(&rkt->lock);
+	mr = rkt->table[(rkey >> (32 - ib_ipath_lkey_table_size))];
+	spin_unlock(&rkt->lock);
+	if (unlikely(mr == NULL || mr->lkey != rkey))
+		return 0;
+
+	off = vaddr - mr->iova;
+	if (unlikely(vaddr < mr->iova || off + len > mr->length ||
+		     (mr->access_flags & acc) == 0))
+		return 0;
+
+	off += mr->offset;
+	sge->mr = mr;
+	sge->m = 0;
+	sge->n = 0;
+	while (off >= mr->map[sge->m]->segs[sge->n].length) {
+		off -= mr->map[sge->m]->segs[sge->n].length;
+		if (++sge->n >= IPATH_SEGSZ) {
+			sge->m++;
+			sge->n = 0;
+		}
+	}
+	sge->vaddr = mr->map[sge->m]->segs[sge->n].vaddr + off;
+	sge->length = mr->map[sge->m]->segs[sge->n].length - off;
+	sge->sge_length = len;
+	ss->sg_list = NULL;
+	ss->num_sge = 1;
+	return 1;
+}
+
+/*
+ * Add a new entry to the completion queue.
+ * This may be called with one of the qp->s_lock or qp->r_rq.lock held.
+ */
+static void ipath_cq_enter(struct ipath_cq *cq, struct ib_wc *entry, int sig)
+{
+	unsigned long flags;
+	u32 next;
+
+	spin_lock_irqsave(&cq->lock, flags);
+
+	cq->queue[cq->head] = *entry;
+	next = cq->head + 1;
+	if (next == cq->ibcq.cqe)
+		next = 0;
+	if (likely(next != cq->tail))
+		cq->head = next;
+	else {
+		spin_unlock_irqrestore(&cq->lock, flags);
+		if (cq->ibcq.event_handler) {
+			struct ib_event ev;
+
+			ev.device = cq->ibcq.device;
+			ev.element.cq = &cq->ibcq;
+			ev.event = IB_EVENT_CQ_ERR;
+			cq->ibcq.event_handler(&ev, cq->ibcq.cq_context);
+		}
+		return;
+	}
+
+	if (cq->notify == IB_CQ_NEXT_COMP ||
+	    (cq->notify == IB_CQ_SOLICITED && sig)) {
+		cq->notify = IB_CQ_NONE;
+		cq->triggered++;
+		/*
+		 * This will cause send_complete() to be called in
+		 * another thread.
+		 */
+		tasklet_schedule(&cq->comptask);
+	}
+
+	spin_unlock_irqrestore(&cq->lock, flags);
+
+	if (entry->status != IB_WC_SUCCESS)
+		to_idev(cq->ibcq.device)->n_wqe_errs++;
+}
+
+static void send_complete(unsigned long data)
+{
+	struct ipath_cq *cq = (struct ipath_cq *)data;
+
+	/*
+	 * The completion handler will most likely rearm the notification
+	 * and poll for all pending entries.  If a new completion entry
+	 * is added while we are in this routine, tasklet_schedule()
+	 * won't call us again until we return so we check triggered to
+	 * see if we need to call the handler again.
+	 */
+	for (;;) {
+		u8 triggered = cq->triggered;
+
+		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
+
+		if (cq->triggered == triggered)
+			return;
+	}
+}
+
+/*
+ * This is the QP state transition table.
+ * See ipath_modify_qp() for details.
+ */
+static const struct {
+	int trans;
+	u32 req_param[IB_QPT_RAW_IPV6];
+	u32 opt_param[IB_QPT_RAW_IPV6];
+} qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
+	[IB_QPS_RESET] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_INIT] = {
+			.trans = IPATH_TRANS_RST2INIT,
+			.req_param = {
+				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_ACCESS_FLAGS),
+				[IB_QPT_RC] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_ACCESS_FLAGS),
+			},
+		},
+	},
+	[IB_QPS_INIT] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_INIT] = {
+			.trans = IPATH_TRANS_INIT2INIT,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_ACCESS_FLAGS),
+				[IB_QPT_RC] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_ACCESS_FLAGS),
+			}
+		},
+		[IB_QPS_RTR] = {
+			.trans = IPATH_TRANS_INIT2RTR,
+			.req_param = {
+				[IB_QPT_UC] = (IB_QP_AV |
+					 IB_QP_PATH_MTU |
+					 IB_QP_DEST_QPN |
+					 IB_QP_RQ_PSN),
+				[IB_QPT_RC] = (IB_QP_AV |
+					 IB_QP_PATH_MTU |
+					 IB_QP_DEST_QPN |
+					 IB_QP_RQ_PSN |
+					 IB_QP_MAX_DEST_RD_ATOMIC |
+					 IB_QP_MIN_RNR_TIMER),
+			},
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX),
+				[IB_QPT_RC] = (IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX),
+			}
+		}
+	},
+	[IB_QPS_RTR] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_RTS] = {
+			.trans = IPATH_TRANS_RTR2RTS,
+			.req_param = {
+				[IB_QPT_SMI] = IB_QP_SQ_PSN,
+				[IB_QPT_GSI] = IB_QP_SQ_PSN,
+				[IB_QPT_UD] = IB_QP_SQ_PSN,
+				[IB_QPT_UC] = IB_QP_SQ_PSN,
+				[IB_QPT_RC] = (IB_QP_TIMEOUT |
+					 IB_QP_RETRY_CNT |
+					 IB_QP_RNR_RETRY |
+					 IB_QP_SQ_PSN |
+					 IB_QP_MAX_QP_RD_ATOMIC),
+			},
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX |
+					 IB_QP_PATH_MIG_STATE),
+				[IB_QPT_RC] = (IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX |
+					 IB_QP_MIN_RNR_TIMER |
+					 IB_QP_PATH_MIG_STATE),
+			}
+		}
+	},
+	[IB_QPS_RTS] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_RTS] = {
+			.trans = IPATH_TRANS_RTS2RTS,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_ACCESS_FLAGS |
+					 IB_QP_ALT_PATH |
+					 IB_QP_PATH_MIG_STATE),
+				[IB_QPT_RC] = (IB_QP_ACCESS_FLAGS |
+					 IB_QP_ALT_PATH |
+					 IB_QP_PATH_MIG_STATE |
+					 IB_QP_MIN_RNR_TIMER),
+			}
+		},
+		[IB_QPS_SQD] = {
+			.trans = IPATH_TRANS_RTS2SQD,
+		},
+	},
+	[IB_QPS_SQD] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_RTS] = {
+			.trans = IPATH_TRANS_SQD2RTS,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PATH_MIG_STATE),
+				[IB_QPT_RC] = (IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_MIN_RNR_TIMER |
+					 IB_QP_PATH_MIG_STATE),
+			}
+		},
+		[IB_QPS_SQD] = {
+			.trans = IPATH_TRANS_SQD2SQD,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_PKEY_INDEX | IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_AV |
+					 IB_QP_TIMEOUT |
+					 IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX |
+					 IB_QP_PATH_MIG_STATE),
+				[IB_QPT_RC] = (IB_QP_AV |
+					 IB_QP_TIMEOUT |
+					 IB_QP_RETRY_CNT |
+					 IB_QP_RNR_RETRY |
+					 IB_QP_MAX_QP_RD_ATOMIC |
+					 IB_QP_MAX_DEST_RD_ATOMIC |
+					 IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX |
+					 IB_QP_MIN_RNR_TIMER |
+					 IB_QP_PATH_MIG_STATE),
+			}
+		}
+	},
+	[IB_QPS_SQE] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_RTS] = {
+			.trans = IPATH_TRANS_SQERR2RTS,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UC] = IB_QP_CUR_STATE,
+				[IB_QPT_RC] = (IB_QP_CUR_STATE |
+					 IB_QP_MIN_RNR_TIMER),
+			}
+		}
+	},
+	[IB_QPS_ERR] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR }
+	}
+};
+
+/*
+ * Initialize the QP state to the reset state.
+ */
+static void ipath_reset_qp(struct ipath_qp *qp)
+{
+	qp->remote_qpn = 0;
+	qp->qkey = 0;
+	qp->qp_access_flags = 0;
+	qp->s_hdrwords = 0;
+	qp->s_psn = 0;
+	qp->r_psn = 0;
+	atomic_set(&qp->msn, 0);
+	if (qp->ibqp.qp_type == IB_QPT_RC) {
+		qp->s_state = IB_OPCODE_RC_SEND_LAST;
+		qp->r_state = IB_OPCODE_RC_SEND_LAST;
+	} else {
+		qp->s_state = IB_OPCODE_UC_SEND_LAST;
+		qp->r_state = IB_OPCODE_UC_SEND_LAST;
+	}
+	qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+	qp->s_nak_state = 0;
+	qp->s_rnr_timeout = 0;
+	qp->s_head = 0;
+	qp->s_tail = 0;
+	qp->s_cur = 0;
+	qp->s_last = 0;
+	qp->s_ssn = 1;
+	qp->s_lsn = 0;
+	qp->r_rq.head = 0;
+	qp->r_rq.tail = 0;
+	qp->r_reuse_sge = 0;
+}
+
+/*
+ * Flush send work queue.
+ * The QP s_lock should be held.
+ */
+static void ipath_sqerror_qp(struct ipath_qp *qp, struct ib_wc *wc)
+{
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
+
+	_VERBS_INFO("Send queue error on QP%d/%d: err: %d\n",
+		    qp->ibqp.qp_num, qp->remote_qpn, wc->status);
+
+	spin_lock(&dev->pending_lock);
+	/* XXX What if its already removed by the timeout code? */
+	if (qp->timerwait.next != LIST_POISON1)
+		list_del(&qp->timerwait);
+	if (qp->piowait.next != LIST_POISON1)
+		list_del(&qp->piowait);
+	spin_unlock(&dev->pending_lock);
+
+	ipath_cq_enter(to_icq(qp->ibqp.send_cq), wc, 1);
+	if (++qp->s_last >= qp->s_size)
+		qp->s_last = 0;
+
+	wc->status = IB_WC_WR_FLUSH_ERR;
+
+	while (qp->s_last != qp->s_head) {
+		wc->wr_id = wqe->wr.wr_id;
+		wc->opcode = wc_opcode[wqe->wr.opcode];
+		ipath_cq_enter(to_icq(qp->ibqp.send_cq), wc, 1);
+		if (++qp->s_last >= qp->s_size)
+			qp->s_last = 0;
+		wqe = get_swqe_ptr(qp, qp->s_last);
+	}
+	qp->s_cur = qp->s_tail = qp->s_head;
+	qp->state = IB_QPS_SQE;
+}
+
+/*
+ * Flush both send and receive work queues.
+ * QP r_rq.lock and s_lock should be held.
+ */
+static void ipath_error_qp(struct ipath_qp *qp)
+{
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ib_wc wc;
+
+	_VERBS_INFO("QP%d/%d in error state\n",
+		    qp->ibqp.qp_num, qp->remote_qpn);
+
+	spin_lock(&dev->pending_lock);
+	/* XXX What if its already removed by the timeout code? */
+	if (qp->timerwait.next != LIST_POISON1)
+		list_del(&qp->timerwait);
+	if (qp->piowait.next != LIST_POISON1)
+		list_del(&qp->piowait);
+	spin_unlock(&dev->pending_lock);
+
+	wc.status = IB_WC_WR_FLUSH_ERR;
+	wc.vendor_err = 0;
+	wc.byte_len = 0;
+	wc.imm_data = 0;
+	wc.qp_num = qp->ibqp.qp_num;
+	wc.src_qp = 0;
+	wc.wc_flags = 0;
+	wc.pkey_index = 0;
+	wc.slid = 0;
+	wc.sl = 0;
+	wc.dlid_path_bits = 0;
+	wc.port_num = 0;
+
+	while (qp->s_last != qp->s_head) {
+		struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
+
+		wc.wr_id = wqe->wr.wr_id;
+		wc.opcode = wc_opcode[wqe->wr.opcode];
+		if (++qp->s_last >= qp->s_size)
+			qp->s_last = 0;
+		ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 1);
+	}
+	qp->s_cur = qp->s_tail = qp->s_head;
+	qp->s_hdrwords = 0;
+	qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+
+	wc.opcode = IB_WC_RECV;
+	while (qp->r_rq.tail != qp->r_rq.head) {
+		wc.wr_id = get_rwqe_ptr(&qp->r_rq, qp->r_rq.tail)->wr_id;
+		if (++qp->r_rq.tail >= qp->r_rq.size)
+			qp->r_rq.tail = 0;
+		ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
+	}
+}
+
+static int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+			   int attr_mask)
+{
+	struct ipath_qp *qp = to_iqp(ibqp);
+	enum ib_qp_state cur_state, new_state;
+	u32 req_param, opt_param;
+	unsigned long flags;
+
+	if (attr_mask & IB_QP_CUR_STATE) {
+		cur_state = attr->cur_qp_state;
+		if (cur_state != IB_QPS_RTR &&
+		    cur_state != IB_QPS_RTS &&
+		    cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
+			return -EINVAL;
+		spin_lock_irqsave(&qp->r_rq.lock, flags);
+		spin_lock(&qp->s_lock);
+	} else {
+		spin_lock_irqsave(&qp->r_rq.lock, flags);
+		spin_lock(&qp->s_lock);
+		cur_state = qp->state;
+	}
+
+	if (attr_mask & IB_QP_STATE) {
+		new_state = attr->qp_state;
+		if (new_state < 0 || new_state > IB_QPS_ERR)
+			goto inval;
+	} else
+		new_state = cur_state;
+
+	switch (qp_state_table[cur_state][new_state].trans) {
+	case IPATH_TRANS_INVALID:
+		goto inval;
+
+	case IPATH_TRANS_ANY2RST:
+		ipath_reset_qp(qp);
+		break;
+
+	case IPATH_TRANS_ANY2ERR:
+		ipath_error_qp(qp);
+		break;
+
+	}
+
+	req_param =
+	    qp_state_table[cur_state][new_state].req_param[qp->ibqp.qp_type];
+	opt_param =
+	    qp_state_table[cur_state][new_state].opt_param[qp->ibqp.qp_type];
+
+	if ((req_param & attr_mask) != req_param)
+		goto inval;
+
+	if (attr_mask & ~(req_param | opt_param | IB_QP_STATE))
+		goto inval;
+
+	if (attr_mask & IB_QP_PKEY_INDEX) {
+		struct ipath_ibdev *dev = to_idev(ibqp->device);
+
+		if (attr->pkey_index >= ipath_layer_get_npkeys(dev->ib_unit))
+			goto inval;
+		qp->s_pkey_index = attr->pkey_index;
+	}
+
+	if (attr_mask & IB_QP_DEST_QPN)
+		qp->remote_qpn = attr->dest_qp_num;
+
+	if (attr_mask & IB_QP_SQ_PSN) {
+		qp->s_next_psn = attr->sq_psn;
+		qp->s_last_psn = qp->s_next_psn - 1;
+	}
+
+	if (attr_mask & IB_QP_RQ_PSN)
+		qp->r_psn = attr->rq_psn;
+
+	if (attr_mask & IB_QP_ACCESS_FLAGS)
+		qp->qp_access_flags = attr->qp_access_flags;
+
+	if (attr_mask & IB_QP_AV)
+		qp->remote_ah_attr = attr->ah_attr;
+
+	if (attr_mask & IB_QP_PATH_MTU)
+		qp->path_mtu = attr->path_mtu;
+
+	if (attr_mask & IB_QP_RETRY_CNT)
+		qp->s_retry = qp->s_retry_cnt = attr->retry_cnt;
+
+	if (attr_mask & IB_QP_RNR_RETRY) {
+		qp->s_rnr_retry = attr->rnr_retry;
+		if (qp->s_rnr_retry > 7)
+			qp->s_rnr_retry = 7;
+		qp->s_rnr_retry_cnt = qp->s_rnr_retry;
+	}
+
+	if (attr_mask & IB_QP_MIN_RNR_TIMER)
+		qp->s_min_rnr_timer = attr->min_rnr_timer & 0x1F;
+
+	if (attr_mask & IB_QP_QKEY)
+		qp->qkey = attr->qkey;
+
+	if (attr_mask & IB_QP_PKEY_INDEX)
+		qp->s_pkey_index = attr->pkey_index;
+
+	qp->state = new_state;
+	spin_unlock(&qp->s_lock);
+	spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+
+	/*
+	 * Try to move to ARMED if QP1 changed to the RTS state.
+	 */
+	if (qp->ibqp.qp_num == 1 && new_state == IB_QPS_RTS) {
+		struct ipath_ibdev *dev = to_idev(ibqp->device);
+
+		/*
+		 * Bounce the link even if it was active so the SM will
+		 * reinitialize the SMA's state.
+		 */
+		ipath_kset_linkstate((dev->ib_unit << 16) | IPATH_IB_LINKDOWN);
+		ipath_kset_linkstate((dev->ib_unit << 16) | IPATH_IB_LINKARM);
+	}
+	return 0;
+
+inval:
+	spin_unlock(&qp->s_lock);
+	spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+	return -EINVAL;
+}
+
+/*
+ * Compute the AETH (syndrome + MSN).
+ * The QP s_lock should be held.
+ */
+static u32 ipath_compute_aeth(struct ipath_qp *qp)
+{
+	u32 aeth = atomic_read(&qp->msn) & 0xFFFFFF;
+
+	if (qp->s_nak_state) {
+		aeth |= qp->s_nak_state << 24;
+	} else if (qp->ibqp.srq) {
+		/* Shared receive queues don't generate credits. */
+		aeth |= 0x1F << 24;
+	} else {
+		u32 min, max, x;
+		u32 credits;
+
+		/*
+		 * Compute the number of credits available (RWQEs).
+		 * XXX Not holding the r_rq.lock here so there is a small
+		 * chance that the pair of reads are not atomic.
+		 */
+		credits = qp->r_rq.head - qp->r_rq.tail;
+		if ((int)credits < 0)
+			credits += qp->r_rq.size;
+		/* Binary search the credit table to find the code to use. */
+		min = 0;
+		max = 31;
+		for (;;) {
+			x = (min + max) / 2;
+			if (credit_table[x] == credits)
+				break;
+			if (credit_table[x] > credits)
+				max = x;
+			else if (min == x)
+				break;
+			else
+				min = x;
+		}
+		aeth |= x << 24;
+	}
+	return cpu_to_be32(aeth);
+}
+
+
+static void no_bufs_available(struct ipath_qp *qp, struct ipath_ibdev *dev)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->pending_lock, flags);
+	if (qp->piowait.next == LIST_POISON1)
+		list_add_tail(&qp->piowait, &dev->piowait);
+	spin_unlock_irqrestore(&dev->pending_lock, flags);
+	/*
+	 * Note that as soon as ipath_layer_want_buffer() is called and
+	 * possibly before it returns, ipath_ib_piobufavail()
+	 * could be called.  If we are still in the tasklet function,
+	 * tasklet_schedule() will not call us until the next time
+	 * tasklet_schedule() is called.
+	 * We clear the tasklet flag now since we are committing to return
+	 * from the tasklet function.
+	 */
+	tasklet_unlock(&qp->s_task);
+	ipath_layer_want_buffer(dev->ib_unit);
+	dev->n_piowait++;
+}
+
+/*
+ * Process entries in the send work queue until the queue is exhausted.
+ * Only allow one CPU to send a packet per QP (tasklet).
+ * Otherwise, after we drop the QP lock, two threads could send
+ * packets out of order.
+ * This is similar to do_rc_send() below except we don't have timeouts or
+ * resends.
+ */
+static void do_uc_send(unsigned long data)
+{
+	struct ipath_qp *qp = (struct ipath_qp *)data;
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ipath_swqe *wqe;
+	unsigned long flags;
+	u16 lrh0;
+	u32 hwords;
+	u32 nwords;
+	u32 extra_bytes;
+	u32 bth0;
+	u32 bth2;
+	u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);
+	u32 len;
+	struct ipath_other_headers *ohdr;
+	struct ib_wc wc;
+
+	if (test_and_set_bit(IPATH_S_BUSY, &qp->s_flags))
+		return;
+
+	if (unlikely(qp->remote_ah_attr.dlid ==
+		     ipath_layer_get_lid(dev->ib_unit))) {
+		/* Pass in an uninitialized ib_wc to save stack space. */
+		ipath_ruc_loopback(qp, &wc);
+		clear_bit(IPATH_S_BUSY, &qp->s_flags);
+		return;
+	}
+
+	ohdr = &qp->s_hdr.u.oth;
+	if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
+		ohdr = &qp->s_hdr.u.l.oth;
+
+again:
+	/* Check for a constructed packet to be sent. */
+	if (qp->s_hdrwords != 0) {
+			/*
+			 * If no PIO bufs are available, return.
+			 * An interrupt will call ipath_ib_piobufavail()
+			 * when one is available.
+			 */
+			if (ipath_verbs_send(dev->ib_unit, qp->s_hdrwords,
+					     (uint32_t *) &qp->s_hdr,
+					     qp->s_cur_size, qp->s_cur_sge)) {
+				no_bufs_available(qp, dev);
+				return;
+			}
+			dev->n_unicast_xmit++;
+		/* Record that we sent the packet and s_hdr is empty. */
+		qp->s_hdrwords = 0;
+	}
+
+	lrh0 = IPS_LRH_BTH;
+	/* header size in 32-bit words LRH+BTH = (8+12)/4. */
+	hwords = 5;
+
+	/*
+	 * The lock is needed to synchronize between
+	 * setting qp->s_ack_state and post_send().
+	 */
+	spin_lock_irqsave(&qp->s_lock, flags);
+
+	if (!(state_ops[qp->state] & IPATH_PROCESS_SEND_OK))
+		goto done;
+
+	bth0 = ipath_layer_get_pkey(dev->ib_unit, qp->s_pkey_index);
+
+	/* Send a request. */
+	wqe = get_swqe_ptr(qp, qp->s_last);
+	switch (qp->s_state) {
+	default:
+		/* Signal the completion of the last send (if there is one). */
+		if (qp->s_last != qp->s_tail) {
+			if (++qp->s_last == qp->s_size)
+				qp->s_last = 0;
+			if (!test_bit(IPATH_S_SIGNAL_REQ_WR, &qp->s_flags) ||
+			    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
+				wc.wr_id = wqe->wr.wr_id;
+				wc.status = IB_WC_SUCCESS;
+				wc.opcode = wc_opcode[wqe->wr.opcode];
+				wc.vendor_err = 0;
+				wc.byte_len = wqe->length;
+				wc.qp_num = qp->ibqp.qp_num;
+				wc.src_qp = qp->remote_qpn;
+				wc.pkey_index = 0;
+				wc.slid = qp->remote_ah_attr.dlid;
+				wc.sl = qp->remote_ah_attr.sl;
+				wc.dlid_path_bits = 0;
+				wc.port_num = 0;
+				ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc,
+					       0);
+			}
+			wqe = get_swqe_ptr(qp, qp->s_last);
+		}
+		/* Check if send work queue is empty. */
+		if (qp->s_tail == qp->s_head)
+			goto done;
+		/*
+		 * Start a new request.
+		 */
+		qp->s_psn = wqe->psn = qp->s_next_psn;
+		qp->s_sge.sge = wqe->sg_list[0];
+		qp->s_sge.sg_list = wqe->sg_list + 1;
+		qp->s_sge.num_sge = wqe->wr.num_sge;
+		qp->s_len = len = wqe->length;
+		switch (wqe->wr.opcode) {
+		case IB_WR_SEND:
+		case IB_WR_SEND_WITH_IMM:
+			if (len > pmtu) {
+				qp->s_state = IB_OPCODE_UC_SEND_FIRST;
+				len = pmtu;
+				break;
+			}
+			if (wqe->wr.opcode == IB_WR_SEND) {
+				qp->s_state = IB_OPCODE_UC_SEND_ONLY;
+			} else {
+				qp->s_state =
+				    IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE;
+				/* Immediate data comes after the BTH */
+				ohdr->u.imm_data = wqe->wr.imm_data;
+				hwords += 1;
+			}
+			if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+				bth0 |= 1 << 23;
+			break;
+
+		case IB_WR_RDMA_WRITE:
+		case IB_WR_RDMA_WRITE_WITH_IMM:
+			ohdr->u.rc.reth.vaddr =
+			    cpu_to_be64(wqe->wr.wr.rdma.remote_addr);
+			ohdr->u.rc.reth.rkey =
+			    cpu_to_be32(wqe->wr.wr.rdma.rkey);
+			ohdr->u.rc.reth.length = cpu_to_be32(len);
+			hwords += sizeof(struct ib_reth) / 4;
+			if (len > pmtu) {
+				qp->s_state = IB_OPCODE_UC_RDMA_WRITE_FIRST;
+				len = pmtu;
+				break;
+			}
+			if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
+				qp->s_state = IB_OPCODE_UC_RDMA_WRITE_ONLY;
+			} else {
+				qp->s_state =
+				    IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE;
+				/* Immediate data comes after the RETH */
+				ohdr->u.rc.imm_data = wqe->wr.imm_data;
+				hwords += 1;
+				if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+					bth0 |= 1 << 23;
+			}
+			break;
+
+		default:
+			goto done;
+		}
+		if (++qp->s_tail >= qp->s_size)
+			qp->s_tail = 0;
+		break;
+
+	case IB_OPCODE_UC_SEND_FIRST:
+		qp->s_state = IB_OPCODE_UC_SEND_MIDDLE;
+		/* FALLTHROUGH */
+	case IB_OPCODE_UC_SEND_MIDDLE:
+		len = qp->s_len;
+		if (len > pmtu) {
+			len = pmtu;
+			break;
+		}
+		if (wqe->wr.opcode == IB_WR_SEND)
+			qp->s_state = IB_OPCODE_UC_SEND_LAST;
+		else {
+			qp->s_state = IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE;
+			/* Immediate data comes after the BTH */
+			ohdr->u.imm_data = wqe->wr.imm_data;
+			hwords += 1;
+		}
+		if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+			bth0 |= 1 << 23;
+		break;
+
+	case IB_OPCODE_UC_RDMA_WRITE_FIRST:
+		qp->s_state = IB_OPCODE_UC_RDMA_WRITE_MIDDLE;
+		/* FALLTHROUGH */
+	case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:
+		len = qp->s_len;
+		if (len > pmtu) {
+			len = pmtu;
+			break;
+		}
+		if (wqe->wr.opcode == IB_WR_RDMA_WRITE)
+			qp->s_state = IB_OPCODE_UC_RDMA_WRITE_LAST;
+		else {
+			qp->s_state =
+			    IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE;
+			/* Immediate data comes after the BTH */
+			ohdr->u.imm_data = wqe->wr.imm_data;
+			hwords += 1;
+			if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+				bth0 |= 1 << 23;
+		}
+		break;
+	}
+	bth2 = qp->s_next_psn++ & 0xFFFFFF;
+	qp->s_len -= len;
+	bth0 |= qp->s_state << 24;
+
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+
+	/* Construct the header. */
+	extra_bytes = (4 - len) & 3;
+	nwords = (len + extra_bytes) >> 2;
+	if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
+		/* Header size in 32-bit words. */
+		hwords += 10;
+		lrh0 = IPS_LRH_GRH;
+		qp->s_hdr.u.l.grh.version_tclass_flow =
+		    cpu_to_be32((6 << 28) |
+				(qp->remote_ah_attr.grh.traffic_class << 20) |
+				qp->remote_ah_attr.grh.flow_label);
+		qp->s_hdr.u.l.grh.paylen =
+		    cpu_to_be16(((hwords - 12) + nwords + SIZE_OF_CRC) << 2);
+		qp->s_hdr.u.l.grh.next_hdr = 0x1B;
+		qp->s_hdr.u.l.grh.hop_limit = qp->remote_ah_attr.grh.hop_limit;
+		/* The SGID is 32-bit aligned. */
+		qp->s_hdr.u.l.grh.sgid.global.subnet_prefix = dev->gid_prefix;
+		qp->s_hdr.u.l.grh.sgid.global.interface_id =
+		    ipath_layer_get_guid(dev->ib_unit);
+		qp->s_hdr.u.l.grh.dgid = qp->remote_ah_attr.grh.dgid;
+	}
+	qp->s_hdrwords = hwords;
+	qp->s_cur_sge = &qp->s_sge;
+	qp->s_cur_size = len;
+	lrh0 |= qp->remote_ah_attr.sl << 4;
+	qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
+	/* DEST LID */
+	qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
+	qp->s_hdr.lrh[2] = cpu_to_be16(hwords + nwords + SIZE_OF_CRC);
+	qp->s_hdr.lrh[3] = cpu_to_be16(ipath_layer_get_lid(dev->ib_unit));
+	bth0 |= extra_bytes << 20;
+	ohdr->bth[0] = cpu_to_be32(bth0);
+	ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
+	ohdr->bth[2] = cpu_to_be32(bth2);
+
+	/* Check for more work to do. */
+	goto again;
+
+done:
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+	clear_bit(IPATH_S_BUSY, &qp->s_flags);
+}
+
+/*
+ * Process entries in the send work queue until credit or queue is exhausted.
+ * Only allow one CPU to send a packet per QP (tasklet).
+ * Otherwise, after we drop the QP s_lock, two threads could send
+ * packets out of order.
+ */
+static void do_rc_send(unsigned long data)
+{
+	struct ipath_qp *qp = (struct ipath_qp *)data;
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ipath_swqe *wqe;
+	struct ipath_sge_state *ss;
+	unsigned long flags;
+	u16 lrh0;
+	u32 hwords;
+	u32 nwords;
+	u32 extra_bytes;
+	u32 bth0;
+	u32 bth2;
+	u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);
+	u32 len;
+	struct ipath_other_headers *ohdr;
+	char newreq;
+
+	if (test_and_set_bit(IPATH_S_BUSY, &qp->s_flags))
+		return;
+
+	if (unlikely(qp->remote_ah_attr.dlid ==
+		     ipath_layer_get_lid(dev->ib_unit))) {
+		struct ib_wc wc;
+
+		/*
+		 * Pass in an uninitialized ib_wc to be consistent with
+		 * other places where ipath_ruc_loopback() is called.
+		 */
+		ipath_ruc_loopback(qp, &wc);
+		clear_bit(IPATH_S_BUSY, &qp->s_flags);
+		return;
+	}
+
+	ohdr = &qp->s_hdr.u.oth;
+	if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
+		ohdr = &qp->s_hdr.u.l.oth;
+
+again:
+	/* Check for a constructed packet to be sent. */
+	if (qp->s_hdrwords != 0) {
+			/*
+			 * If no PIO bufs are available, return.
+			 * An interrupt will call ipath_ib_piobufavail()
+			 * when one is available.
+			 */
+			if (ipath_verbs_send(dev->ib_unit, qp->s_hdrwords,
+					     (uint32_t *) &qp->s_hdr,
+					     qp->s_cur_size, qp->s_cur_sge)) {
+				no_bufs_available(qp, dev);
+				return;
+			}
+			dev->n_unicast_xmit++;
+		/* Record that we sent the packet and s_hdr is empty. */
+		qp->s_hdrwords = 0;
+	}
+
+	lrh0 = IPS_LRH_BTH;
+	/* header size in 32-bit words LRH+BTH = (8+12)/4. */
+	hwords = 5;
+
+	/*
+	 * The lock is needed to synchronize between
+	 * setting qp->s_ack_state, resend timer, and post_send().
+	 */
+	spin_lock_irqsave(&qp->s_lock, flags);
+
+	bth0 = ipath_layer_get_pkey(dev->ib_unit, qp->s_pkey_index);
+
+	/* Sending responses has higher priority over sending requests. */
+	if (qp->s_ack_state != IB_OPCODE_RC_ACKNOWLEDGE) {
+		/*
+		 * Send a response.
+		 * Note that we are in the responder's side of the QP context.
+		 */
+		switch (qp->s_ack_state) {
+		case IB_OPCODE_RC_RDMA_READ_REQUEST:
+			ss = &qp->s_rdma_sge;
+			len = qp->s_rdma_len;
+			if (len > pmtu) {
+				len = pmtu;
+				qp->s_ack_state =
+				    IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST;
+			} else {
+				qp->s_ack_state =
+				    IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY;
+			}
+			qp->s_rdma_len -= len;
+			bth0 |= qp->s_ack_state << 24;
+			ohdr->u.aeth = ipath_compute_aeth(qp);
+			hwords++;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST:
+			qp->s_ack_state =
+			    IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE;
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:
+			ss = &qp->s_rdma_sge;
+			len = qp->s_rdma_len;
+			if (len > pmtu) {
+				len = pmtu;
+			} else {
+				ohdr->u.aeth = ipath_compute_aeth(qp);
+				hwords++;
+				qp->s_ack_state =
+				    IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST;
+			}
+			qp->s_rdma_len -= len;
+			bth0 |= qp->s_ack_state << 24;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST:
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY:
+			/*
+			 * We have to prevent new requests from changing
+			 * the r_sge state while a ipath_verbs_send()
+			 * is in progress.
+			 * Changing r_state allows the receiver
+			 * to continue processing new packets.
+			 * We do it here now instead of above so
+			 * that we are sure the packet was sent before
+			 * changing the state.
+			 */
+			qp->r_state = IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST;
+			qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+			goto send_req;
+
+		case IB_OPCODE_RC_COMPARE_SWAP:
+		case IB_OPCODE_RC_FETCH_ADD:
+			ss = NULL;
+			len = 0;
+			qp->r_state = IB_OPCODE_RC_SEND_LAST;
+			qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+			bth0 |= IB_OPCODE_ATOMIC_ACKNOWLEDGE << 24;
+			ohdr->u.at.aeth = ipath_compute_aeth(qp);
+			ohdr->u.at.atomic_ack_eth =
+			    cpu_to_be64(qp->s_ack_atomic);
+			hwords += sizeof(ohdr->u.at) / 4;
+			break;
+
+		default:
+			/* Send a regular ACK. */
+			ss = NULL;
+			len = 0;
+			qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+			bth0 |= qp->s_ack_state << 24;
+			ohdr->u.aeth = ipath_compute_aeth(qp);
+			hwords++;
+		}
+		bth2 = qp->s_ack_psn++ & 0xFFFFFF;
+	} else {
+	      send_req:
+		if (!(state_ops[qp->state] & IPATH_PROCESS_SEND_OK) ||
+		    qp->s_rnr_timeout)
+			goto done;
+
+		/* Send a request. */
+		wqe = get_swqe_ptr(qp, qp->s_cur);
+		switch (qp->s_state) {
+		default:
+			/*
+			 * Resend an old request or start a new one.
+			 *
+			 * We keep track of the current SWQE so that
+			 * we don't reset the "furthest progress" state
+			 * if we need to back up.
+			 */
+			newreq = 0;
+			if (qp->s_cur == qp->s_tail) {
+				/* Check if send work queue is empty. */
+				if (qp->s_tail == qp->s_head)
+					goto done;
+				qp->s_psn = wqe->psn = qp->s_next_psn;
+				newreq = 1;
+			}
+			/*
+			 * Note that we have to be careful not to modify the
+			 * original work request since we may need to resend
+			 * it.
+			 */
+			qp->s_sge.sge = wqe->sg_list[0];
+			qp->s_sge.sg_list = wqe->sg_list + 1;
+			qp->s_sge.num_sge = wqe->wr.num_sge;
+			qp->s_len = len = wqe->length;
+			ss = &qp->s_sge;
+			bth2 = 0;
+			switch (wqe->wr.opcode) {
+			case IB_WR_SEND:
+			case IB_WR_SEND_WITH_IMM:
+				/* If no credit, return. */
+				if (qp->s_lsn != (u32) -1 &&
+				    cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {
+					goto done;
+				}
+				wqe->lpsn = wqe->psn;
+				if (len > pmtu) {
+					wqe->lpsn += (len - 1) / pmtu;
+					qp->s_state = IB_OPCODE_RC_SEND_FIRST;
+					len = pmtu;
+					break;
+				}
+				if (wqe->wr.opcode == IB_WR_SEND) {
+					qp->s_state = IB_OPCODE_RC_SEND_ONLY;
+				} else {
+					qp->s_state =
+					    IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE;
+					/* Immediate data comes after the BTH */
+					ohdr->u.imm_data = wqe->wr.imm_data;
+					hwords += 1;
+				}
+				if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+					bth0 |= 1 << 23;
+				bth2 = 1 << 31;	/* Request ACK. */
+				if (++qp->s_cur == qp->s_size)
+					qp->s_cur = 0;
+				break;
+
+			case IB_WR_RDMA_WRITE:
+				if (newreq)
+					qp->s_lsn++;
+				/* FALLTHROUGH */
+			case IB_WR_RDMA_WRITE_WITH_IMM:
+				/* If no credit, return. */
+				if (qp->s_lsn != (u32) -1 &&
+				    cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {
+					goto done;
+				}
+				ohdr->u.rc.reth.vaddr =
+				    cpu_to_be64(wqe->wr.wr.rdma.remote_addr);
+				ohdr->u.rc.reth.rkey =
+				    cpu_to_be32(wqe->wr.wr.rdma.rkey);
+				ohdr->u.rc.reth.length = cpu_to_be32(len);
+				hwords += sizeof(struct ib_reth) / 4;
+				wqe->lpsn = wqe->psn;
+				if (len > pmtu) {
+					wqe->lpsn += (len - 1) / pmtu;
+					qp->s_state =
+					    IB_OPCODE_RC_RDMA_WRITE_FIRST;
+					len = pmtu;
+					break;
+				}
+				if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
+					qp->s_state =
+					    IB_OPCODE_RC_RDMA_WRITE_ONLY;
+				} else {
+					qp->s_state =
+					    IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE;
+					/* Immediate data comes after RETH */
+					ohdr->u.rc.imm_data = wqe->wr.imm_data;
+					hwords += 1;
+					if (wqe->wr.
+					    send_flags & IB_SEND_SOLICITED)
+						bth0 |= 1 << 23;
+				}
+				bth2 = 1 << 31;	/* Request ACK. */
+				if (++qp->s_cur == qp->s_size)
+					qp->s_cur = 0;
+				break;
+
+			case IB_WR_RDMA_READ:
+				ohdr->u.rc.reth.vaddr =
+				    cpu_to_be64(wqe->wr.wr.rdma.remote_addr);
+				ohdr->u.rc.reth.rkey =
+				    cpu_to_be32(wqe->wr.wr.rdma.rkey);
+				ohdr->u.rc.reth.length = cpu_to_be32(len);
+				qp->s_state = IB_OPCODE_RC_RDMA_READ_REQUEST;
+				hwords += sizeof(ohdr->u.rc.reth) / 4;
+				if (newreq) {
+					qp->s_lsn++;
+					/*
+					 * Adjust s_next_psn to count the
+					 * expected number of responses.
+					 */
+					if (len > pmtu)
+						qp->s_next_psn +=
+						    (len - 1) / pmtu;
+					wqe->lpsn = qp->s_next_psn++;
+				}
+				ss = NULL;
+				len = 0;
+				if (++qp->s_cur == qp->s_size)
+					qp->s_cur = 0;
+				break;
+
+			case IB_WR_ATOMIC_CMP_AND_SWP:
+			case IB_WR_ATOMIC_FETCH_AND_ADD:
+				qp->s_state =
+				    wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ?
+				    IB_OPCODE_RC_COMPARE_SWAP :
+				    IB_OPCODE_RC_FETCH_ADD;
+				ohdr->u.atomic_eth.vaddr =
+				    cpu_to_be64(wqe->wr.wr.atomic.remote_addr);
+				ohdr->u.atomic_eth.rkey =
+				    cpu_to_be32(wqe->wr.wr.atomic.rkey);
+				ohdr->u.atomic_eth.swap_data =
+				    cpu_to_be64(wqe->wr.wr.atomic.swap);
+				ohdr->u.atomic_eth.compare_data =
+				    cpu_to_be64(wqe->wr.wr.atomic.compare_add);
+				hwords += sizeof(struct ib_atomic_eth) / 4;
+				if (newreq) {
+					qp->s_lsn++;
+					wqe->lpsn = wqe->psn;
+				}
+				if (++qp->s_cur == qp->s_size)
+					qp->s_cur = 0;
+				ss = NULL;
+				len = 0;
+				break;
+
+			default:
+				goto done;
+			}
+			if (newreq) {
+				if (++qp->s_tail >= qp->s_size)
+					qp->s_tail = 0;
+			}
+			bth2 |= qp->s_psn++ & 0xFFFFFF;
+			if ((int)(qp->s_psn - qp->s_next_psn) > 0)
+				qp->s_next_psn = qp->s_psn;
+			spin_lock(&dev->pending_lock);
+			if (qp->timerwait.next == LIST_POISON1) {
+				list_add_tail(&qp->timerwait,
+					      &dev->pending[dev->
+							    pending_index]);
+			}
+			spin_unlock(&dev->pending_lock);
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST:
+			/*
+			 * This case can only happen if a send is
+			 * restarted.  See ipath_restart_rc().
+			 */
+			ipath_init_restart(qp, wqe);
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_SEND_FIRST:
+			qp->s_state = IB_OPCODE_RC_SEND_MIDDLE;
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_SEND_MIDDLE:
+			bth2 = qp->s_psn++ & 0xFFFFFF;
+			if ((int)(qp->s_psn - qp->s_next_psn) > 0)
+				qp->s_next_psn = qp->s_psn;
+			ss = &qp->s_sge;
+			len = qp->s_len;
+			if (len > pmtu) {
+				/*
+				 * Request an ACK every 1/2 MB to avoid
+				 * retransmit timeouts.
+				 */
+				if (((wqe->length - len) % (512 * 1024)) == 0)
+					bth2 |= 1 << 31;
+				len = pmtu;
+				break;
+			}
+			if (wqe->wr.opcode == IB_WR_SEND)
+				qp->s_state = IB_OPCODE_RC_SEND_LAST;
+			else {
+				qp->s_state =
+				    IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE;
+				/* Immediate data comes after the BTH */
+				ohdr->u.imm_data = wqe->wr.imm_data;
+				hwords += 1;
+			}
+			if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+				bth0 |= 1 << 23;
+			bth2 |= 1 << 31;	/* Request ACK. */
+			if (++qp->s_cur >= qp->s_size)
+				qp->s_cur = 0;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST:
+			/*
+			 * This case can only happen if a RDMA write is
+			 * restarted.  See ipath_restart_rc().
+			 */
+			ipath_init_restart(qp, wqe);
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_RDMA_WRITE_FIRST:
+			qp->s_state = IB_OPCODE_RC_RDMA_WRITE_MIDDLE;
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:
+			bth2 = qp->s_psn++ & 0xFFFFFF;
+			if ((int)(qp->s_psn - qp->s_next_psn) > 0)
+				qp->s_next_psn = qp->s_psn;
+			ss = &qp->s_sge;
+			len = qp->s_len;
+			if (len > pmtu) {
+				/*
+				 * Request an ACK every 1/2 MB to avoid
+				 * retransmit timeouts.
+				 */
+				if (((wqe->length - len) % (512 * 1024)) == 0)
+					bth2 |= 1 << 31;
+				len = pmtu;
+				break;
+			}
+			if (wqe->wr.opcode == IB_WR_RDMA_WRITE)
+				qp->s_state = IB_OPCODE_RC_RDMA_WRITE_LAST;
+			else {
+				qp->s_state =
+				    IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE;
+				/* Immediate data comes after the BTH */
+				ohdr->u.imm_data = wqe->wr.imm_data;
+				hwords += 1;
+				if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+					bth0 |= 1 << 23;
+			}
+			bth2 |= 1 << 31;	/* Request ACK. */
+			if (++qp->s_cur >= qp->s_size)
+				qp->s_cur = 0;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:
+			/*
+			 * This case can only happen if a RDMA read is
+			 * restarted.  See ipath_restart_rc().
+			 */
+			ipath_init_restart(qp, wqe);
+			len = ((qp->s_psn - wqe->psn) & 0xFFFFFF) * pmtu;
+			ohdr->u.rc.reth.vaddr =
+			    cpu_to_be64(wqe->wr.wr.rdma.remote_addr + len);
+			ohdr->u.rc.reth.rkey =
+			    cpu_to_be32(wqe->wr.wr.rdma.rkey);
+			ohdr->u.rc.reth.length = cpu_to_be32(qp->s_len);
+			qp->s_state = IB_OPCODE_RC_RDMA_READ_REQUEST;
+			hwords += sizeof(ohdr->u.rc.reth) / 4;
+			bth2 = qp->s_psn++ & 0xFFFFFF;
+			if ((int)(qp->s_psn - qp->s_next_psn) > 0)
+				qp->s_next_psn = qp->s_psn;
+			ss = NULL;
+			len = 0;
+			if (++qp->s_cur == qp->s_size)
+				qp->s_cur = 0;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_REQUEST:
+		case IB_OPCODE_RC_COMPARE_SWAP:
+		case IB_OPCODE_RC_FETCH_ADD:
+			/*
+			 * We shouldn't start anything new until this request
+			 * is finished.  The ACK will handle rescheduling us.
+			 * XXX The number of outstanding ones is negotiated
+			 * at connection setup time (see pg. 258,289)?
+			 * XXX Also, if we support multiple outstanding
+			 * requests, we need to check the WQE IB_SEND_FENCE
+			 * flag and not send a new request if a RDMA read or
+			 * atomic is pending.
+			 */
+			goto done;
+		}
+		qp->s_len -= len;
+		bth0 |= qp->s_state << 24;
+		/* XXX queue resend timeout. */
+	}
+	/* Make sure it is non-zero before dropping the lock. */
+	qp->s_hdrwords = hwords;
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+
+	/* Construct the header. */
+	extra_bytes = (4 - len) & 3;
+	nwords = (len + extra_bytes) >> 2;
+	if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
+		/* Header size in 32-bit words. */
+		hwords += 10;
+		lrh0 = IPS_LRH_GRH;
+		qp->s_hdr.u.l.grh.version_tclass_flow =
+		    cpu_to_be32((6 << 28) |
+				(qp->remote_ah_attr.grh.traffic_class << 20) |
+				qp->remote_ah_attr.grh.flow_label);
+		qp->s_hdr.u.l.grh.paylen =
+		    cpu_to_be16(((hwords - 12) + nwords + SIZE_OF_CRC) << 2);
+		qp->s_hdr.u.l.grh.next_hdr = 0x1B;
+		qp->s_hdr.u.l.grh.hop_limit = qp->remote_ah_attr.grh.hop_limit;
+		/* The SGID is 32-bit aligned. */
+		qp->s_hdr.u.l.grh.sgid.global.subnet_prefix = dev->gid_prefix;
+		qp->s_hdr.u.l.grh.sgid.global.interface_id =
+		    ipath_layer_get_guid(dev->ib_unit);
+		qp->s_hdr.u.l.grh.dgid = qp->remote_ah_attr.grh.dgid;
+		qp->s_hdrwords = hwords;
+	}
+	qp->s_cur_sge = ss;
+	qp->s_cur_size = len;
+	lrh0 |= qp->remote_ah_attr.sl << 4;
+	qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
+	/* DEST LID */
+	qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
+	qp->s_hdr.lrh[2] = cpu_to_be16(hwords + nwords + SIZE_OF_CRC);
+	qp->s_hdr.lrh[3] = cpu_to_be16(ipath_layer_get_lid(dev->ib_unit));
+	bth0 |= extra_bytes << 20;
+	ohdr->bth[0] = cpu_to_be32(bth0);
+	ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
+	ohdr->bth[2] = cpu_to_be32(bth2);
+
+	/* Check for more work to do. */
+	goto again;
+
+done:
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+	clear_bit(IPATH_S_BUSY, &qp->s_flags);
+}

diff -r 801287704e40 -r b792638cc4bc include/asm-alpha/io.h
@@ -504,6 +504,8 @@
 extern void memcpy_toio(volatile void __iomem *, const void *, long);
 extern void _memset_c_io(volatile void __iomem *, unsigned long, long);
 
+#define memcpy_toio32 __memcpy_toio32
+
 static inline void memset_io(volatile void __iomem *addr, u8 c, long len)
 {
 	_memset_c_io(addr, 0x0101010101010101UL * c, len);

diff -r 801287704e40 -r b792638cc4bc include/asm-arm/io.h
@@ -184,6 +184,8 @@
 #define memset_io(c,v,l)	_memset_io(__mem_pci(c),(v),(l))
 #define memcpy_fromio(a,c,l)	_memcpy_fromio((a),__mem_pci(c),(l))
 #define memcpy_toio(c,a,l)	_memcpy_toio(__mem_pci(c),(a),(l))
+
+#define memcpy_toio32 __memcpy_toio32
 
 #define eth_io_copy_and_sum(s,c,l,b) \
 				eth_copy_and_sum((s),__mem_pci(c),(l),(b))

diff -r 801287704e40 -r b792638cc4bc include/asm-cris/io.h
@@ -121,6 +121,8 @@
 #define memcpy_fromio(a,b,c)	memcpy((a),(void *)(b),(c))
 #define memcpy_toio(a,b,c)	memcpy((void *)(a),(b),(c))
 
+#define memcpy_toio32 __memcpy_toio32
+
 /*
  * Again, CRIS does not require mem IO specific function.
  */

diff -r 801287704e40 -r b792638cc4bc include/asm-frv/io.h
@@ -124,6 +124,8 @@
 	memcpy((void __force *) dst, src, count);
 }
 
+#define memcpy_toio32 __memcpy_toio32
+
 static inline uint8_t inb(unsigned long addr)
 {
 	return __builtin_read8((void *)addr);

diff -r 801287704e40 -r b792638cc4bc include/asm-h8300/io.h
@@ -209,6 +209,8 @@
 #define memcpy_fromio(a,b,c)	memcpy((a),(void *)(b),(c))
 #define memcpy_toio(a,b,c)	memcpy((void *)(a),(b),(c))
 
+#define memcpy_toio32 __memcpy_toio32
+
 #define mmiowb()
 
 #define inb(addr)    ((h8300_buswidth(addr))?readw((addr) & ~1) & 0xff:readb(addr))

diff -r 801287704e40 -r b792638cc4bc include/asm-i386/io.h
@@ -203,6 +203,9 @@
 {
 	__memcpy((void __force *) dst, src, count);
 }
+
+#define memcpy_toio32 __memcpy_toio32
+
 
 /*
  * ISA space is 'always mapped' on a typical x86 system, no need to

diff -r 801287704e40 -r b792638cc4bc include/asm-ia64/io.h
@@ -443,6 +443,8 @@
 extern void memcpy_toio(volatile void __iomem *dst, const void *src, long n);
 extern void memset_io(volatile void __iomem *s, int c, long n);
 
+#define memcpy_toio32 __memcpy_toio32
+
 #define dma_cache_inv(_start,_size)             do { } while (0)
 #define dma_cache_wback(_start,_size)           do { } while (0)
 #define dma_cache_wback_inv(_start,_size)       do { } while (0)

diff -r 801287704e40 -r b792638cc4bc include/asm-m32r/io.h
@@ -216,6 +216,8 @@
 	memcpy((void __force *) dst, src, count);
 }
 
+#define memcpy_toio32 __memcpy_toio32
+
 /*
  * Convert a physical pointer to a virtual kernel pointer for /dev/mem
  * access

diff -r 801287704e40 -r b792638cc4bc include/asm-m68knommu/io.h
@@ -113,6 +113,8 @@
 #define memcpy_fromio(a,b,c)	memcpy((a),(void *)(b),(c))
 #define memcpy_toio(a,b,c)	memcpy((void *)(a),(b),(c))
 
+#define memcpy_toio32 __memcpy_toio32
+
 #define inb(addr)    readb(addr)
 #define inw(addr)    readw(addr)
 #define inl(addr)    readl(addr)

diff -r 801287704e40 -r b792638cc4bc include/asm-mips/io.h
@@ -534,6 +534,8 @@
 	memcpy((void __force *) dst, src, count);
 }
 
+#define memcpy_toio32 __memcpy_toio32
+
 /*
  * Memory Mapped I/O
  */

diff -r 801287704e40 -r b792638cc4bc include/asm-parisc/io.h
@@ -294,6 +294,8 @@
 void memcpy_fromio(void *dst, const volatile void __iomem *src, int count);
 void memcpy_toio(volatile void __iomem *dst, const void *src, int count);
 
+#define memcpy_toio32 __memcpy_toio32
+
 /* Support old drivers which don't ioremap.
  * NB this interface is scheduled to disappear in 2.5
  */

diff -r 801287704e40 -r b792638cc4bc include/asm-powerpc/io.h
@@ -63,6 +63,8 @@
 #define memcpy_fromio(a,b,c)	iSeries_memcpy_fromio((a), (b), (c))
 #define memcpy_toio(a,b,c)	iSeries_memcpy_toio((a), (b), (c))
 
+#define memcpy_toio32 __memcpy_toio32
+
 #define inb(addr)		readb(((void __iomem *)(long)(addr)))
 #define inw(addr)		readw(((void __iomem *)(long)(addr)))
 #define inl(addr)		readl(((void __iomem *)(long)(addr)))

diff -r 801287704e40 -r b792638cc4bc include/asm-ppc/io.h
@@ -367,6 +367,8 @@
 }
 #endif
 
+#define memcpy_toio32 __memcpy_toio32
+
 #define eth_io_copy_and_sum(a,b,c,d)		eth_copy_and_sum((a),(void __force *)(void __iomem *)(b),(c),(d))
 
 /*

diff -r 801287704e40 -r b792638cc4bc include/asm-s390/io.h
@@ -99,6 +99,8 @@
 #define memcpy_fromio(a,b,c)    memcpy((a),__io_virt(b),(c))
 #define memcpy_toio(a,b,c)      memcpy(__io_virt(a),(b),(c))
 
+#define memcpy_toio32 __memcpy_toio32
+
 #define inb_p(addr) readb(addr)
 #define inb(addr) readb(addr)
 

diff -r 801287704e40 -r b792638cc4bc include/asm-sh/io.h
@@ -177,6 +177,8 @@
 extern void memcpy_toio(unsigned long, const void *, unsigned long);
 extern void memset_io(unsigned long, int, unsigned long);
 
+#define memcpy_toio32 __memcpy_toio32
+
 /* SuperH on-chip I/O functions */
 static __inline__ unsigned char ctrl_inb(unsigned long addr)
 {

diff -r 801287704e40 -r b792638cc4bc include/asm-sh64/io.h
@@ -125,6 +125,8 @@
 
 void memcpy_toio(void __iomem *to, const void *from, long count);
 void memcpy_fromio(void *to, void __iomem *from, long count);
+
+#define memcpy_toio32 __memcpy_toio32
 
 #define mmiowb()
 

diff -r 801287704e40 -r b792638cc4bc include/asm-sparc/io.h
@@ -239,6 +239,8 @@
 
 #define memcpy_toio(d,s,sz)	_memcpy_toio(d,s,sz)
 
+#define memcpy_toio32 __memcpy_toio32
+
 #ifdef __KERNEL__
 
 /*

diff -r 801287704e40 -r b792638cc4bc include/asm-sparc64/io.h
@@ -440,6 +440,8 @@
 
 #define memcpy_toio(d,s,sz)	_memcpy_toio(d,s,sz)
 
+#define memcpy_toio32 __memcpy_toio32
+
 static inline int check_signature(void __iomem *io_addr,
 				  const unsigned char *signature,
 				  int length)

diff -r 801287704e40 -r b792638cc4bc include/asm-v850/io.h
@@ -130,6 +130,8 @@
 #define memcpy_fromio(dst, src, len) memcpy (dst, (void *)src, len)
 #define memcpy_toio(dst, src, len) memcpy ((void *)dst, src, len)
 
+#define memcpy_toio32 __memcpy_toio32
+
 /*
  * Convert a physical pointer to a virtual kernel pointer for /dev/mem
  * access

diff -r 801287704e40 -r b792638cc4bc include/asm-x86_64/io.h
@@ -252,6 +252,13 @@
 	__memcpy_toio((unsigned long)to,from,len);
 }
 
+#include <asm/string.h>
+
+static inline void memcpy_toio32(void __iomem *dst, const void *src, size_t count)
+{
+	memcpy32((void __force *) dst, src, count);
+}
+
 void memset_io(volatile void __iomem *a, int b, size_t c);
 
 /*

diff -r 801287704e40 -r b792638cc4bc include/asm-xtensa/io.h
@@ -159,6 +159,8 @@
 #define memcpy_fromio(a,b,c)   memcpy((a),(void *)(b),(c))
 #define memcpy_toio(a,b,c)      memcpy((void *)(a),(b),(c))
 
+#define memcpy_toio32 __memcpy_toio32
+
 /* At this point the Xtensa doesn't provide byte swap instructions */
 
 #ifdef __XTENSA_EB__

diff -r a3a00f637da6 -r 2d9a3f27a10c drivers/infiniband/hw/ipath/ipath_common.h
@@ -0,0 +1,704 @@
+/*
+ * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Patent licenses, if any, provided herein do not apply to
+ * combinations of this program with other software, or any other
+ * product whatsoever.
+ */
+
+#ifndef _IPATH_COMMON_H
+#define _IPATH_COMMON_H
+
+/*
+ * This file contains defines, structures, etc. that are used
+ * to communicate between kernel and user code.
+ */
+
+#define IPATH_IDSTR "PathScale 1.1\n"
+
+typedef uint8_t ipath_type;
+
+/* This is the IEEE-assigned OUI for PathScale, Inc. */
+#define IPATH_SRC_OUI_1 0x00
+#define IPATH_SRC_OUI_2 0x11
+#define IPATH_SRC_OUI_3 0x75
+
+/* version of protocol header (known to chip also). In the long run,
+ * we should be able to generate and accept a range of version numbers;
+ * for now we only accept one, and it's compiled in.
+ */
+#define IPS_PROTO_VERSION 2
+
+/*
+ * These are compile time constants that you may want to enable or disable
+ * if you are trying to debug problems with code or performance.
+ * IPATH_VERBOSE_TRACING define as 1 if you want additional tracing in
+ * fastpath code
+ * IPATH_TRACE_REGWRITES define as 1 if you want register writes to be
+ * traced in faspath code
+ * _IPATH_TRACING define as 0 if you want to remove all tracing in a
+ * compilation unit
+ * _IPATH_DEBUGGING define as 0 if you want to remove debug prints
+ */
+
+
+/*
+ * These tell the driver which ioctl's belong to the diags interface.
+ * As above, don't use them elsewhere.
+ */
+#define _IPATH_DIAG_IOCTL_LOW 100
+#define _IPATH_DIAG_IOCTL_HIGH 109
+
+struct ipath_eeprom_req {
+	long long addr;
+	uint16_t len;
+	uint16_t offset;
+};
+
+/*
+ * NOTE:   We use compatible ioctls, the same ioctl code for both 32 and 64
+ * bit user mode.  For that reason, all structures, etc. used in these
+ * ioctls must have the same size and offsets, in both 32 and 64 bit mode.
+ * We normally use uint64_t to  hold pointers for this reason, doing appropriate
+ * casts on both sides before using the data value.
+ */
+
+/* init;  user params to kernel  */
+#define IPATH_USERINIT      _IOW('s', 16, struct ipath_user_info)
+/* init;  kernel/chip params to user */
+#define IPATH_BASEINFO      _IOR('s', 17, struct ipath_base_info)
+/* send a packet */
+#define IPATH_SENDPKT       _IOW('s', 18, struct ipath_sendpkt)
+/*
+ * if arg is 0, disable port, used when flushing after a hdrq overflow.
+ * If arg ia 1, re-enable, and return new value of head register
+ */
+#define IPATH_RCVCTRL       _IOR('s', 19, uint32_t)
+#define IPATH_READ_EEPROM  _IOWR('s', 20, struct ipath_eeprom_req)
+/* set an accepted partition key; up to 4 pkeys can be active at once */
+#define IPATH_SET_PKEY      _IOW('s', 21, uint16_t)
+#define IPATH_WRITE_EEPROM _IOWR('s', 22, struct ipath_eeprom_req)
+/* set LID for interface (SMA) */
+#define IPATH_SET_LID       _IOW('s', 23, uint32_t)
+/* set IB MTU for interface (SMA) */
+#define IPATH_SET_MTU       _IOW('s', 24, uint32_t)
+/* set IB link state for interface (SMA) */
+#define IPATH_SET_LINKSTATE _IOW('s', 25, uint32_t)
+/* send an SMA packet, sps_flags contains "normal" SMA unit and minor number. */
+#define IPATH_SEND_SMA_PKT  _IOW('s', 26, struct ipath_sendpkt)
+/* receive an SMA packet */
+#define IPATH_RCV_SMA_PKT   _IOW('s', 27, struct ipath_sendpkt)
+/* get the portinfo data (SMA)
+ * takes array of 13, returns port info fields.  Data is in host order,
+ * not network order; SMA-only fields are not filled in
+ */
+#define IPATH_GET_PORTINFO  _IOWR('s', 28, uint32_t *)
+/*
+ * get the nodeinfo data (SMA)
+ * takes an array of 10, returns nodeinfo fields in host order
+ */
+#define IPATH_GET_NODEINFO  _IOWR('s', 29, uint32_t *)
+/* set GUID on interface (SMA; GUID given in network order) */
+#define IPATH_SET_GUID      _IOW('s', 30,  struct ipath_setguid)
+/* set MLID for interface (SMA) */
+#define IPATH_SET_MLID      _IOW('s', 31, uint32_t)
+#define IPATH_GET_MLID      _IOWR('s', 32, uint32_t *)	/* get the MLID (SMA) */
+/* update expected TID entries */
+#define IPATH_UPDM_TID      _IOWR('s', 33, struct _tidupd)
+/* free expected TID entries */
+#define IPATH_FREE_TID      _IOW('s', 34, struct _tidupd)
+/* return assigned unit:port */
+#define IPATH_GETPORT       _IOR('s', 35, uint32_t)
+/* wait for rcv pkt or pioavail */
+#define IPATH_WAIT          _IOW('s', 36, uint32_t)
+/* return LID for passed in unit */
+#define IPATH_GETLID        _IOR('s', 37, uint16_t)
+/* return # of units supported by driver */
+#define IPATH_GETUNITS      _IO('s',  38)
+/* get the device status */
+#define IPATH_GET_DEVSTATUS _IOWR('s', 39, uint64_t *)
+
+/* available for reuse ('s', 48) */
+
+/* diagnostic read */
+#define IPATH_DIAGREAD      _IOR('s', 100, struct ipath_diag_info)
+/* diagnostic write */
+#define IPATH_DIAGWRITE     _IOW('s', 101, struct ipath_diag_info)
+/* HT Config read */
+#define IPATH_DIAG_HTREAD   _IOR('s', 102, struct ipath_diag_info)
+/* HT config write */
+#define IPATH_DIAG_HTWRITE  _IOW('s', 103, struct ipath_diag_info)
+#define IPATH_DIAGENTER     _IO('s', 104)	/* Enter diagnostic mode */
+#define IPATH_DIAGLEAVE     _IO('s', 105)	/* Leave diagnostic mode */
+/* send a packet, sps_flags contains unit and minor number. */
+#define IPATH_SEND_DIAG_PKT _IOW('s', 106, struct ipath_sendpkt)
+/*
+ * read I2C FLASH
+ * NOTE: To read the I2C device, the _uaddress field should contain
+ * a pointer to struct ipath_eeprom_req, and _unit must be valid
+ */
+#define IPATH_DIAG_RD_I2C   _IOW('s', 107, struct ipath_diag_info)
+
+/*
+ * Monitoring ioctls.  All of these work with the main device
+ * (/dev/ipath), if you don't mind using a port (e.g. you already have
+ * the device open.)  IPATH_GETSTATS and IPATH_GETUNITCOUNTERS also
+ * work with the control device (/dev/ipath_ctrl), if you don't want to
+ * use a port.
+ */
+
+/* return chip counters for current unit. */
+#define IPATH_GETCOUNTERS     _IOR('s', 40, struct infinipath_counters)
+/* return chip stats */
+#define IPATH_GETSTATS        _IOR('s', 41, struct infinipath_stats)
+/* return chip counters for a particular unit. */
+#define IPATH_GETUNITCOUNTERS _IOR('s', 43, struct infinipath_getunitcounters)
+
+/*
+ * unit is incoming unit number.
+ * data is a pointer to the infinipath_counters structure.
+ */
+struct infinipath_getunitcounters {
+	uint16_t unit;
+	uint16_t fill[3]; /* required for same size struct 32/64 bit */
+	uint64_t data;
+};
+
+/*
+ * The value in the BTH QP field that InfiniPath uses to differentiate
+ * an infinipath protocol IB packet vs standard IB transport
+ */
+#define IPATH_KD_QP 0x656b79
+
+/*
+ * valid states passed to ipath_set_linkstate() user call
+ * (IPATH_SET_LINKSTATE ioctl)
+ */
+#define IPATH_IB_LINKDOWN		0
+#define IPATH_IB_LINKARM		1
+#define IPATH_IB_LINKACTIVE		2
+#define IPATH_IB_LINKINIT		3
+#define IPATH_IB_LINKDOWN_POLL		4
+#define IPATH_IB_LINKDOWN_DISABLE	5
+
+/*
+ * stats maintained by the driver.  For now, at least, this is global
+ * to all minor devices.
+ */
+struct infinipath_stats {
+	uint64_t sps_ints;	/* number of interrupts taken */
+	uint64_t sps_errints;	/* number of interrupts for errors */
+	/* number of errors from chip (not including packet errors or CRC) */
+	uint64_t sps_errs;
+	/* number of packet errors from chip other than CRC */
+	uint64_t sps_pkterrs;
+	/* number of packets with CRC errors (ICRC and VCRC) */
+	uint64_t sps_crcerrs;
+	/* number of hardware errors reported (parity, etc.) */
+	uint64_t sps_hwerrs;
+	/* number of times IB link changed state unexpectedly */
+	uint64_t sps_iblink;
+	uint64_t sps_unused3;	/* no longer used; left for compatibility */
+	uint64_t sps_port0pkts; /* number of kernel (port0) packets received */
+	/* number of "ethernet" packets sent by driver */
+	uint64_t sps_ether_spkts;
+	/* number of "ethernet" packets received by driver */
+	uint64_t sps_ether_rpkts;
+	uint64_t sps_sma_spkts;	/* number of SMA packets sent by driver */
+	uint64_t sps_sma_rpkts;	/* number of SMA packets received by driver */
+	/* number of times all ports rcvhdrq was full and packet dropped */
+	uint64_t sps_hdrqfull;
+	/* number of times all ports egrtid was full and packet dropped */
+	uint64_t sps_etidfull;
+	/*
+	 * number of times we tried to send from driver, but no pio
+	 * buffers avail
+	 */
+	uint64_t sps_nopiobufs;
+	uint64_t sps_ports;	/* number of ports currently open */
+	/* list of pkeys (other than default) accepted (0 means not set) */
+	uint16_t sps_pkeys[4];
+	/* lids for up to 4 infinipaths, indexed by infinipath # */
+	uint16_t sps_lid[4];
+	/* number of user ports per chip (not IB ports) */
+	uint32_t sps_nports;
+	uint32_t sps_nullintr;	/* not our interrupt, or already handled */
+	uint32_t sps_maxpkts_call;  /* max number of packets handled per receive call */
+	uint32_t sps_avgpkts_call;  /* avg number of packets handled per receive call */
+	uint64_t sps_pagelocks;	/* total number of pages locked */
+	uint64_t sps_pageunlocks; /* total number of pages unlocked */
+	/*
+	 * Number of packets dropped in kernel other than errors
+	 * (ether packets if ipath not configured, sma/mad, etc.)
+	 */
+	uint64_t sps_krdrops;
+	/* mlids for up to 4 infinipaths, indexed by infinipath # */
+	uint16_t sps_mlid[4];
+	uint64_t __sps_pad[45];	/* pad for future growth */
+};
+
+/*
+ * These are the status bits returned (in ascii form, 64bit value)
+ * by the IPATH_GETSTATS ioctl.
+ */
+#define IPATH_STATUS_INITTED       0x1	/* basic driver initialization done */
+#define IPATH_STATUS_DISABLED      0x2  /* hardware disabled */
+#define IPATH_STATUS_UNUSED        0x4	/* available */
+#define IPATH_STATUS_OIB_SMA       0x8	/* ipath_mad kernel SMA running */
+#define IPATH_STATUS_SMA          0x10	/* user SMA running */
+/* Chip has been found and initted */
+#define IPATH_STATUS_CHIP_PRESENT 0x20
+#define IPATH_STATUS_IB_READY     0x40	/* IB link is at ACTIVE, has LID,
+										 * usable for all VL's */
+/* after link up, LID,MTU,etc. has been configured */
+#define IPATH_STATUS_IB_CONF      0x80
+/* no link established, probably no cable */
+#define IPATH_STATUS_IB_NOCABLE  0x100
+/* A Fatal hardware error has occurred. */
+#define IPATH_STATUS_HWERROR     0x200
+
+/* The list of usermode accessible registers.  Also see Reg_* later in file */
+typedef enum _ipath_ureg {
+	ur_rcvhdrtail = 0,	/* (RO)  DMA RcvHdr to be used next. */
+	/* (RW)  RcvHdr entry to be processed next by host. */
+	ur_rcvhdrhead = 1,
+	ur_rcvegrindextail = 2,	/* (RO)  Index of next Eager index to use. */
+	ur_rcvegrindexhead = 3,	/* (RW)  Eager TID to be processed next */
+	/* For internal use only; max register number. */
+	_IPATH_UregMax
+} ipath_ureg;
+
+/* SMA minor# no portinfo, one for all instances */
+#define IPATH_SMA 128
+
+/* Control minor# no portinfo, one for all instances */
+#define IPATH_CTRL 130
+
+/*
+ * This structure is returned by ipath_userinit() immediately after open
+ * to get implementation-specific info, and info specific to this
+ * instance.
+ */
+struct ipath_base_info {
+	/* version of hardware, for feature checking. */
+	uint32_t spi_hw_version;
+	/* version of software, for feature checking. */
+	uint32_t spi_sw_version;
+	/* InfiniPath port assigned, goes into sent packets */
+	uint32_t spi_port;
+	/*
+	 * IB MTU, packets IB data must be less than this.
+	 * The MTU is in bytes, and will be a multiple of 4 bytes.
+	 */
+	uint32_t spi_mtu;
+	/*
+	 * size of a PIO buffer.  Any given packet's total
+	 * size must be less than this (in words).  Included is the
+	 * starting control word, so if 513 is returned, then total
+	 * pkt size is 512 words or less.
+	 */
+	uint32_t spi_piosize;
+	/* size of the TID cache in infinipath, in entries */
+	uint32_t spi_tidcnt;
+	/* size of the TID Eager list in infinipath, in entries */
+	uint32_t spi_tidegrcnt;
+	/* size of a single receive header queue entry. */
+	uint32_t spi_rcvhdrent_size;
+	/* Count of receive header queue entries allocated.
+	 * This may be less than the spu_rcvhdrcnt passed in!.
+	 */
+	uint32_t spi_rcvhdr_cnt;
+
+	uint32_t __32_bit_compatibility_pad;	/* DO NOT  MOVE OR REMOVE */
+
+	/* address where receive buffer queue is mapped into */
+	uint64_t spi_rcvhdr_base;
+
+	/* user program. */
+
+	/* base address of eager TID receive buffers. */
+	uint64_t spi_rcv_egrbufs;
+
+	/* Allocated by initialization code, not by protocol. */
+
+	/* size of each TID buffer in host memory,
+	 * starting at spi_rcv_egrbufs.  It includes spu_egrskip, and is
+	 * at least spi_mtu bytes, and the buffers are virtually contiguous
+	 */
+	uint32_t spi_rcv_egrbufsize;
+	/*
+	 * The special QP (queue pair) value that identifies an infinipath
+	 * protocol packet from standard IB packets.  More, probably much
+	 * more, to be added.
+	 */
+	uint32_t spi_qpair;
+
+	/*
+	 * user register base for init code, not to be used directly by
+	 * protocol or applications
+	 */
+	uint64_t __spi_uregbase;
+	/*
+	 * maximum buffer size in bytes that can be used in a
+	 * single TID entry (assuming the buffer is aligned to this boundary).
+	 * This is the minimum of what the hardware and software support
+	 * Guaranteed to be a power of 2.
+	 */
+	uint32_t spi_tid_maxsize;
+	/*
+	 * alignment of each pio send buffer (byte count
+	 * to add to spi_piobufbase to get to second buffer)
+	 */
+	uint32_t spi_pioalign;
+	/*
+	 * the index of the first pio buffer available
+	 * to this process; needed to do lookup in spi_pioavailaddr; not added
+	 * to spi_piobufbase
+	 */
+	uint32_t spi_pioindex;
+	uint32_t spi_piocnt;	/* number of buffers mapped for this process */
+
+	/*
+	 * base address of writeonly pio buffers for this process.
+	 * Each buffer has spi_piosize words, and is aligned on spi_pioalign
+	 * boundaries.  spi_piocnt buffers are mapped from this address
+	 */
+	uint64_t spi_piobufbase;
+
+	/*
+	 * base address of readonly memory copy of the pioavail registers.
+	 * There are 2 bits for each buffer.
+	 */
+	uint64_t spi_pioavailaddr;
+
+	/*
+	 * Address where driver updates a copy
+	 * of the interface and driver status (IPATH_STATUS_*) as a 64 bit value
+	 * It's followed by a string indicating hardware error, if there was one
+	 */
+	uint64_t spi_status;
+
+	/* number of chip ports available to user processes */
+	uint32_t spi_nports;
+	uint32_t spi_unit;	/* unit number of chip we are using */
+	uint32_t spi_rcv_egrperchunk;	/* num bufs in each contiguous set */
+	/* size in bytes of each contiguous set */
+	uint32_t spi_rcv_egrchunksize;
+	/* total size of mmap to cover full rcvegrbuffers */
+	uint32_t spi_rcv_egrbuftotlen;
+	/*
+	 * ioctl cmd includes struct size, so pad out, and adjust down as
+	 * new fields are added to keep size constant
+	 */
+	uint32_t __spi_pad[19];
+} __attribute__ ((aligned(8)));
+
+#define IPATH_WAIT_RCV   0x1	/* IPATH_WAIT, receive */
+#define IPATH_WAIT_PIO   0x2	/* IPATH_WAIT, PIO */
+
+/*
+ * This version number is given to the driver by the user code during
+ * initialization in the spu_userversion field of ipath_user_info, so
+ * the driver can check for compatibility with user code.
+ *
+ * The major version changes when data structures
+ * change in an incompatible way.  The driver must be the same or higher
+ * for initialization to succeed.  In some cases, a higher version
+ * driver will not interoperate with older software, and initialization
+ * will return an error.
+ */
+#define IPATH_USER_SWMAJOR 1
+
+/*
+ * Minor version differences are always compatible
+ * a within a major version, however if if user software is larger
+ * than driver software, some new features and/or structure fields
+ * may not be implemented; the user code must deal with this if it
+ * cares, or it must abort after initialization reports the difference
+ */
+#define IPATH_USER_SWMINOR 2
+
+#define IPATH_USER_SWVERSION ((IPATH_USER_SWMAJOR<<16) | IPATH_USER_SWMINOR)
+
+#define IPATH_KERN_TYPE 0
+
+/* Similarly, this is the kernel version going back to the user.  It's slightly
+ * different, in that we want to tell if the driver was built as part of a
+ * PathScale release, or from the driver from OpenIB, kernel.org, or a
+ * standard distribution, for support reasons.  The high bit is 0 for
+ * non-PathScale, and 1 for PathScale-built/supplied.
+ *
+ * It's returned by the driver to the user code during initialization
+ * in the spi_sw_version field of ipath_base_info, so the user code can
+ * in turn check for compatibility with the kernel.
+*/
+#define IPATH_KERN_SWVERSION ((IPATH_KERN_TYPE<<31) | IPATH_USER_SWVERSION)
+
+/*
+ * This structure is passed to ipath_userinit() to tell the driver where
+ * user code buffers are, sizes, etc.
+ */
+struct ipath_user_info {
+	/*
+	 * version of user software, to detect compatibility issues.
+	 * Should be set to IPATH_USER_SWVERSION.
+	 */
+	uint32_t spu_userversion;
+
+	/* desired number of receive header queue entries */
+	uint32_t spu_rcvhdrcnt;
+
+	/*
+	 * Leave this much unused space at the start of
+	 * each eager buffer for software use.  Similar in effect to
+	 * setting K_Offset to this value.  needs to be 'small', on the
+	 * order of one or two cachelines
+	 */
+	uint32_t spu_egrskip;
+
+	/*
+	 * number of words in KD protocol header
+	 * This tells InfiniPath how many words to copy to rcvhdrq.  If 0,
+	 * kernel uses a default.  Once set, attempts to set any other value
+	 * are an error (EAGAIN) until driver is reloaded.
+	 */
+	uint32_t spu_rcvhdrsize;
+
+	/*
+	 * cache line aligned (64 byte) user address to
+	 * which the rcvhdrtail register will be written by infinipath
+	 * whenever it changes, so that no chip registers are read in
+	 * the performance path.
+	 */
+	uint64_t spu_rcvhdraddr;
+
+	/*
+	 * ioctl cmd includes struct size, so pad out,
+	 * and adjust down as new fields are added to keep size constant
+	 */
+	uint32_t __spu_pad[6];
+} __attribute__ ((aligned(8)));
+
+struct ipath_iovec {
+	/* Pointer to data, but same size 32 and 64 bit */
+	uint64_t iov_base;
+
+	/*
+	 * Length of data; don't need 64 bits, but want
+	 * ipath_sendpkt to remain same size as before 32 bit changes, so...
+	 */
+	uint64_t iov_len;
+};
+
+/*
+ * Describes a single packet for send.  Each packet can have one or more
+ * buffers, but the total length (exclusive of IB headers) must be less
+ * than the MTU, and if using the PIO method, entire packet length,
+ * including IB headers, must be less than the ipath_piosize value (words).
+ * Use of this necessitates including sys/uio.h
+ */
+struct ipath_sendpkt {
+	uint32_t sps_flags;	/* flags for packet (TBD) */
+	uint32_t sps_cnt;	/* number of entries to use in sps_iov */
+	/* array of iov's describing packet. TEMPORARY */
+	struct ipath_iovec sps_iov[4];
+};
+
+struct _tidupd {		/* used only in inlined function for ioctl. */
+	uint32_t tidcnt;
+	uint32_t tid__unused;	/* make structure same size in 32 and 64 bit */
+	uint64_t tidvaddr;	/* virtual address of first page in transfer */
+	/* pointer (same size 32/64 bit) to uint16_t tid array */
+	uint64_t tidlist;
+
+	/*
+	 * pointer (same size 32/64 bit) to bitmap of TIDs used
+	 * for this call; checked for being large enough at open
+	 */
+	uint64_t tidmap;
+};
+
+struct ipath_setguid {		/* set GUID for interface */
+	uint64_t sguid;		/* in network order */
+	uint64_t sunit;		/* unit number of interface */
+};
+
+/*
+ * Structure used to send data to and receive data from a diags ioctl.
+ *
+ * NOTE: For HT reads and writes, we only support byte, word (16bits) and
+ * dword (32bits).  All other sizes for HT are invalid.
+ */
+struct ipath_diag_info {
+	uint64_t _base_offset;	/* register to start reading from */
+	uint64_t _num_bytes;	/* number of bytes to read or write */
+	/*
+	 * address in user space.
+	 * for reads, this is the address to store the read result(s).
+	 * for writes, it the address to get the write data from.
+	 * This memory better be valid in user space!
+	 */
+	uint64_t _uaddress;
+	uint64_t _unit;		/* Unit ID of chip we are accessing. */
+	uint64_t _pad[15];
+};
+
+/*
+ * Data layout in I2C flash (for GUID, etc.)
+ * All fields are little-endian binary unless otherwise stated
+ */
+#define IPATH_FLASH_VERSION 1
+struct ipath_flash {
+	uint8_t if_fversion;	/* flash layout version (IPATH_FLASH_VERSION) */
+	uint8_t if_csum;	/* checksum protecting if_length bytes */
+	/*
+	 * valid length (in use, protected by if_csum), including if_fversion
+	 * and if_sum themselves)
+	 */
+	uint8_t if_length;
+	uint8_t if_guid[8];	/* the GUID, in network order */
+	/* number of GUIDs to use, starting from if_guid */
+	uint8_t if_numguid;
+	char    if_serial[12];	/* the board serial number, in ASCII */
+	char    if_mfgdate[8];	/* board mfg date (YYYYMMDD ASCII) */
+	/* last board rework/test date (YYYYMMDD ASCII) */
+	char    if_testdate[8];
+	uint8_t if_errcntp[4];	/* logging of error counts, TBD */
+	/* powered on hours, updated at driver unload */
+	uint8_t if_powerhour[2];
+	char    if_comment[32];	/* ASCII free-form comment field */
+	uint8_t if_future[50];	/* 78 bytes used, min flash size is 128 bytes */
+};
+
+uint8_t ipath_flash_csum(struct ipath_flash *, int);
+
+/*
+ * These are the counters implemented in the chip, and are listed in order.
+ * They are returned in this order by the IPATH_GETCOUNTERS ioctl
+ */
+struct infinipath_counters {
+	unsigned long long LBIntCnt;
+	unsigned long long LBFlowStallCnt;
+	unsigned long long Reserved1;
+	unsigned long long TxUnsupVLErrCnt;
+	unsigned long long TxDataPktCnt;
+	unsigned long long TxFlowPktCnt;
+	unsigned long long TxDwordCnt;
+	unsigned long long TxLenErrCnt;
+	unsigned long long TxMaxMinLenErrCnt;
+	unsigned long long TxUnderrunCnt;
+	unsigned long long TxFlowStallCnt;
+	unsigned long long TxDroppedPktCnt;
+	unsigned long long RxDroppedPktCnt;
+	unsigned long long RxDataPktCnt;
+	unsigned long long RxFlowPktCnt;
+	unsigned long long RxDwordCnt;
+	unsigned long long RxLenErrCnt;
+	unsigned long long RxMaxMinLenErrCnt;
+	unsigned long long RxICRCErrCnt;
+	unsigned long long RxVCRCErrCnt;
+	unsigned long long RxFlowCtrlErrCnt;
+	unsigned long long RxBadFormatCnt;
+	unsigned long long RxLinkProblemCnt;
+	unsigned long long RxEBPCnt;
+	unsigned long long RxLPCRCErrCnt;
+	unsigned long long RxBufOvflCnt;
+	unsigned long long RxTIDFullErrCnt;
+	unsigned long long RxTIDValidErrCnt;
+	unsigned long long RxPKeyMismatchCnt;
+	unsigned long long RxP0HdrEgrOvflCnt;
+	unsigned long long RxP1HdrEgrOvflCnt;
+	unsigned long long RxP2HdrEgrOvflCnt;
+	unsigned long long RxP3HdrEgrOvflCnt;
+	unsigned long long RxP4HdrEgrOvflCnt;
+	unsigned long long RxP5HdrEgrOvflCnt;
+	unsigned long long RxP6HdrEgrOvflCnt;
+	unsigned long long RxP7HdrEgrOvflCnt;
+	unsigned long long RxP8HdrEgrOvflCnt;
+	unsigned long long Reserved6;
+	unsigned long long Reserved7;
+	unsigned long long IBStatusChangeCnt;
+	unsigned long long IBLinkErrRecoveryCnt;
+	unsigned long long IBLinkDownedCnt;
+	unsigned long long IBSymbolErrCnt;
+};
+
+/*
+ * The next set of defines are for packet headers, and chip register
+ * and memory bits that are visible to and/or used by user-mode software
+ * The other bits that are used only by the driver or diags are in
+ * ipath_registers.h
+ */
+
+/* RcvHdrFlags bits */
+#define INFINIPATH_RHF_LENGTH_MASK 0x7FF
+#define INFINIPATH_RHF_LENGTH_SHIFT 0
+#define INFINIPATH_RHF_RCVTYPE_MASK 0x7
+#define INFINIPATH_RHF_RCVTYPE_SHIFT 11
+#define INFINIPATH_RHF_EGRINDEX_MASK 0x7FF
+#define INFINIPATH_RHF_EGRINDEX_SHIFT 16
+#define INFINIPATH_RHF_H_ICRCERR   0x80000000
+#define INFINIPATH_RHF_H_VCRCERR   0x40000000
+#define INFINIPATH_RHF_H_PARITYERR 0x20000000
+#define INFINIPATH_RHF_H_LENERR    0x10000000
+#define INFINIPATH_RHF_H_MTUERR    0x08000000
+#define INFINIPATH_RHF_H_IHDRERR   0x04000000
+#define INFINIPATH_RHF_H_TIDERR    0x02000000
+#define INFINIPATH_RHF_H_MKERR     0x01000000
+#define INFINIPATH_RHF_H_IBERR     0x00800000
+#define INFINIPATH_RHF_L_SWA       0x00008000
+#define INFINIPATH_RHF_L_SWB       0x00004000
+
+/* infinipath header fields */
+#define INFINIPATH_I_VERS_MASK 0xF
+#define INFINIPATH_I_VERS_SHIFT 28
+#define INFINIPATH_I_PORT_MASK 0xF
+#define INFINIPATH_I_PORT_SHIFT 24
+#define INFINIPATH_I_TID_MASK 0x7FF
+#define INFINIPATH_I_TID_SHIFT 13
+#define INFINIPATH_I_OFFSET_MASK 0x1FFF
+#define INFINIPATH_I_OFFSET_SHIFT 0
+
+/* K_PktFlags bits */
+#define INFINIPATH_KPF_INTR 0x1
+
+/* SendPIO per-buffer control */
+#define INFINIPATH_SP_LENGTHP1_MASK 0x3FF
+#define INFINIPATH_SP_LENGTHP1_SHIFT 0
+#define INFINIPATH_SP_INTR    0x80000000
+#define INFINIPATH_SP_TEST    0x40000000
+#define INFINIPATH_SP_TESTEBP 0x20000000
+
+/* SendPIOAvail bits */
+#define INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT 1
+#define INFINIPATH_SENDPIOAVAIL_CHECK_SHIFT 0
+
+#endif				/* _IPATH_COMMON_H */