Skip to contentSkip to conversation

ipath debug header

RFC PATCH NAK Roland Dreier 15 participants 63 messages Dec 16, 2005 – Dec 26, 2005 1 message not in the archive InfiniBand Page 1 of 2

lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

#1 lore raw

cover IB: PathScale InfiniPath driver

Here is an initial submission from PathScale of a driver for InfiniPath InfiniBand HCAs. The driver is fairly big -- some single files are more than the 100 KB limit for lkml posts -- so I've split it up into a patch series so it can be reviewed inline. The split-up doesn't make sense functionally but I want to make review as easy as possible; any final import will merge the driver as a single git patch.

I've also put the current splitup patchset into my git tree at

git://git.kernel.org/pub/scm/linux/kernel/git/roland/infiniband.git

in the ipath branch.

There are some things I noticed that could maybe be cleaned up, like having sysctls that set values also settable through module parameters under /sys/module, code inside #ifndef __KERNEL__ so include files can be shared with other PathScale code, code in ipath_i2c.c that might be simplified by using drivers/i2c, etc. I'd like to try to get a sense of whether I'm being too picky or whether PathScale really does need to fix these up before the driver is merged.

Basically I'm trying to feel my way as a maintainer so I can find the right balance between wanting kernel code to be absolutely perfect and not wanting to put arbitrary hurdles in front of a vendor who has done a lot of work on contributing an open driver for their hardware.

I am especially interested in feedback about the mergability of this driver from the broader kernel community, although of course feedback from the InfiniBand/RDMA community on IB-specific aspects of the driver is very much appreciated as well.

Thanks,
Roland

#2 lore raw

RFC PATCH 1/13 ipath basic headers

Basic headers for the ipath driver

---

 drivers/infiniband/hw/ipath/ipath_common.h    |  798 +++++++++++++++++++++++++
 drivers/infiniband/hw/ipath/ipath_kernel.h    |  776 ++++++++++++++++++++++++
 drivers/infiniband/hw/ipath/ipath_layer.h     |  131 ++++
 drivers/infiniband/hw/ipath/ipath_registers.h |  359 +++++++++++
 drivers/infiniband/hw/ipath/ips_common.h      |  221 +++++++
 5 files changed, 2285 insertions(+), 0 deletions(-)
 create mode 100644 drivers/infiniband/hw/ipath/ipath_common.h
 create mode 100644 drivers/infiniband/hw/ipath/ipath_kernel.h
 create mode 100644 drivers/infiniband/hw/ipath/ipath_layer.h
 create mode 100644 drivers/infiniband/hw/ipath/ipath_registers.h
 create mode 100644 drivers/infiniband/hw/ipath/ips_common.h

200aa6cff25b6ab39be1f9d8949c2b3b4258ee1d

drivers/infiniband/hw/ipath/ipath_common.h new file +798 -0
new file mode 100644
index 0000000..ac33458
@@ -0,0 +1,798 @@
+/*
+ * Copyright (c) 2003, 2004, 2005. PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Patent licenses, if any, provided herein do not apply to
+ * combinations of this program with other software, or any other
+ * product whatsoever.
+ *
+ * $Id: ipath_common.h 4491 2005-12-15 22:20:31Z rjwalsh $
+ */
+
+#ifndef _IPATH_COMMON_H
+#define _IPATH_COMMON_H
+
+/*
+ * This file contains defines, structures, etc. that are used
+ * to communicate between kernel and user code.
+ */
+
+#ifdef __KERNEL__
+#include <linux/ioctl.h>
+#include <linux/uio.h>
+#include <asm/atomic.h>
+#else				/* !__KERNEL__; user mode */
+#include <sys/ioctl.h>
+#include <sys/uio.h>
+#include <sys/types.h>
+#include <stdint.h>
+
+/* these aren't implemented for user mode, which is OK until we multi-thread */
+typedef struct _atomic {
+	uint32_t counter;
+} atomic_t;			/* no atomic_t type in user-land */
+#define atomic_set(a,v) ((a)->counter = (v))
+#define atomic_inc_return(a)  (++(a)->counter)
+#define likely(x) (x)
+#define unlikely(x) (x)
+
+#define yield() sched_yield()
+
+/*
+ * too horrible to try and use the kernel get_cycles() or equivalent,
+ * so define and inline it here
+ */
+
+#if !defined(rdtscll)
+#if defined(__x86_64) || defined(__i386)
+#define rdtscll(v) do {uint32_t a,d;asm volatile("rdtsc" : "=a" (a), "=d" (d)); \
+         (v) = ((uint64_t)a) | (((uint64_t)d)<<32); \
+} while(0)
+#else
+#error "No cycle counter routine implemented yet for this platform"
+#endif
+#endif				/* !defined(rdtscll) */
+
+#endif				/* ! __KERNEL__ */
+
+typedef uint8_t ipath_type;
+
+/* This is the IEEE-assigned OUI for PathScale, Inc. */
+#define IPATH_SRC_OUI_1 0x00
+#define IPATH_SRC_OUI_2 0x11
+#define IPATH_SRC_OUI_3 0x75
+
+/* version of protocol header (known to chip also). In the long run,
+ * we should be able to generate and accept a range of version numbers;
+ * for now we only accept one, and it's compiled in.
+ */
+#define IPS_PROTO_VERSION 2
+
+#ifndef _BITS_PER_BYTE
+#define _BITS_PER_BYTE 8
+#endif
+
+static __inline__ void ipath_shortcopy(void *dest, void *src, uint32_t cnt)
+    __attribute__ ((always_inline));
+
+/*
+ * this is used for very short copies, usually 1 - 8 bytes,
+ * *NEVER* to the PIO buffers!!!!!!!  use ipath_dwordcpy for longer
+ * copies, or any copy to the PIO buffers.  Works for 32 and 64 bit
+ * gcc and pathcc
+ */
+static __inline__ void ipath_shortcopy(void *dest, void *src, uint32_t cnt)
+{
+	void *ssv, *dsv;
+	uint32_t csv;
+	__asm__ __volatile__("cld\n\trep\n\tmovsb":"=&c"(csv), "=&D"(dsv),
+			     "=&S"(ssv)
+			     :"0"(cnt), "1"(dest), "2"(src)
+			     :"memory");
+}
+
+/*
+ * optimized word copy; good for rev C and later opterons.  Among the best for
+ * short copies, and does as well or slightly better than the optimizization
+ * guide copies 6 and 8 at 2KB.
+ */
+void ipath_dwordcpy(uint32_t * dest, uint32_t * src, uint32_t ndwords);
+
+/*
+ * These are compile time constants that you may want to enable or disable
+ * if you are trying to debug problems with code or performance.
+ * IPATH_VERBOSE_TRACING define as 1 if you want additional tracing in
+ * fastpath code
+ * IPATH_TRACE_REGWRITES define as 1 if you want register writes to be
+ * traced in faspath code
+ * _IPATH_TRACING define as 0 if you want to remove all tracing in a
+ * compilation unit
+ * _IPATH_DEBUGGING define as 0 if you want to remove debug prints
+ */
+
+#define round_up(v,sz) (((v) + (sz)-1) & ~((sz)-1))
+
+/* These are used in the driver, don't use them elsewhere */
+#define _IPATH_SIMFUNC_IOCTL_LOW 1
+#define _IPATH_SIMFUNC_IOCTL_HIGH 7
+
+/*
+ * These tell the driver which ioctl's belong to the diags interface.
+ * As above, don't use them elsewhere.
+ */
+#define _IPATH_DIAG_IOCTL_LOW 100
+#define _IPATH_DIAG_IOCTL_HIGH 109
+
+/* for IPATHSETREGBASE the length is the length covered by addr, in bytes */
+struct ipath_setregbase {
+	void *addr;
+	size_t length;
+};
+/*
+ * IPATHINTERRUPT ioctl passes this as of rev 1.6 of the simulator;
+ * used to be an int
+ */
+struct ipath_int_vec {
+	int long long addr;
+	uint32_t info;
+};
+struct ipath_eeprom_req {
+	long long addr;
+	uint16_t len;
+	uint16_t offset;
+};
+
+/* simulated chip space */
+#define IPATHSETREGBASE     _IOW('s', 1, struct ipath_setregbase)
+/* arg is currently unused */
+#define IPATHINTERRUPT      _IOW('s', 2, struct ipath_int_vec)
+/*
+ * arg is low 32 bits of the simulator sync register, and means that
+ * the simulator has processed up to and including that write
+ */
+#define IPATHSYNC           _IOW('s', 3, int)
+
+/*
+ * simulator has initialized the memory from IPATHSETREGBASE, and driver
+ * can initialize based on the contents
+ */
+#define IPATHREADY          _IOW('s', 4, int)
+/* user mode userreg write, so we can notify simulators */
+#define IPATH_USERREG       _IOW('s', 5, __ipath_rdummy)
+
+/* init;  user params to kernel  */
+#define IPATH_USERINIT      _IOW('s', 16, struct ipath_user_info)
+/* init;  kernel/chip params to user */
+#define IPATH_BASEINFO      _IOR('s', 17, struct ipath_base_info)
+/* send a packet */
+#define IPATH_SENDPKT       _IOW('s', 18, struct ipath_sendpkt)
+/*
+ * if arg is 0, disable port, used when flushing after a hdrq overflow.
+ * If arg ia 1, re-enable, and return new value of head register
+ */
+#define IPATH_RCVCTRL       _IOR('s', 19, uint32_t)
+/* only to make iow macro happy, w/o a struct */
+static uint64_t __ipath_rdummy[2] __attribute__ ((unused));
+#define IPATH_READ_EEPROM  _IOWR('s', 20, struct ipath_eeprom_req)
+/* set an accepted partition key; up to 4 pkeys can be active at once */
+#define IPATH_SET_PKEY      _IOW('s', 21, uint16_t)
+#define IPATH_WRITE_EEPROM _IOWR('s', 22, struct ipath_eeprom_req)
+/* set LID for interface (SMA) */
+#define IPATH_SET_LID       _IOW('s', 23, uint32_t)
+/* set IB MTU for interface (SMA) */
+#define IPATH_SET_MTU       _IOW('s', 24, uint32_t)
+/* set IB link state for interface (SMA) */
+#define IPATH_SET_LINKSTATE _IOW('s', 25, uint32_t)
+/* send an SMA packet, sps_flags contains "normal" SMA unit and minor number. */
+#define IPATH_SEND_SMA_PKT  _IOW('s', 26, struct ipath_sendpkt)
+/* receive an SMA packet */
+#define IPATH_RCV_SMA_PKT   _IOW('s', 27, struct ipath_sendpkt)
+/* get the portinfo data (SMA)
+ * takes array of 13, returns port info fields.  Data is in host order,
+ * not network order; SMA-only fields are not filled in
+ */
+#define IPATH_GET_PORTINFO  _IOWR('s', 28, uint32_t *)
+/*
+ * get the nodeinfo data (SMA)
+ * takes an array of 10, returns nodeinfo fields in host order
+ */
+#define IPATH_GET_NODEINFO  _IOWR('s', 29, uint32_t *)
+/* set GUID on interface (SMA; GUID given in network order) */
+#define IPATH_SET_GUID      _IOW('s', 30,  struct ipath_setguid)
+/* set MLID for interface (SMA) */
+#define IPATH_SET_MLID      _IOW('s', 31, uint32_t)
+#define IPATH_GET_MLID      _IOWR('s', 32, uint32_t *)	/* get the MLID (SMA) */
+/* update expected TID entries */
+#define IPATH_UPDM_TID      _IOWR('s', 33, struct _tidupd)
+/* free expected TID entries */
+#define IPATH_FREE_TID      _IOW('s', 34, struct _tidupd)
+/* return assigned unit:port */
+#define IPATH_GETPORT       _IOR('s', 35, uint32_t)
+/* wait for rcv pkt or pioavail */
+#define IPATH_WAIT          _IOW('s', 36, uint32_t)
+/* return LID for passed in unit */
+#define IPATH_GETLID        _IOR('s', 37, uint16_t)
+/* return # of units supported by driver */
+#define IPATH_GETUNITS      _IO('s',  38)
+/* get the device status */
+#define IPATH_GET_DEVSTATUS _IOWR('s', 39, uint64_t *)
+
+/* available for reuse ('s', 48) */
+
+/* diagnostic read */
+#define IPATH_DIAGREAD      _IOR('s', 100, struct ipath_diag_info)
+/* diagnostic write */
+#define IPATH_DIAGWRITE     _IOW('s', 101, struct ipath_diag_info)
+/* HT Config read */
+#define IPATH_DIAG_HTREAD   _IOR('s', 102, struct ipath_diag_info)
+/* HT config write */
+#define IPATH_DIAG_HTWRITE  _IOW('s', 103, struct ipath_diag_info)
+#define IPATH_DIAGENTER     _IO('s', 104)	/* Enter diagnostic mode */
+#define IPATH_DIAGLEAVE     _IO('s', 105)	/* Leave diagnostic mode */
+/* send a packet, sps_flags contains unit and minor number. */
+#define IPATH_SEND_DIAG_PKT _IOW('s', 106, struct ipath_sendpkt)
+/*
+ * read I2C FLASH
+ * NOTE: To read the I2C device, the _uaddress field should contain 
+ * a pointer to struct ipath_eeprom_req, and _unit must be valid
+ */
+#define IPATH_DIAG_RD_I2C   _IOW('s', 107, struct ipath_diag_info)
+
+/*
+ * Monitoring ioctls.  All of these work with the main device
+ * (/dev/ipath), if you don't mind using a port (e.g. you already have
+ * the device open.)  IPATH_GETSTATS and IPATH_GETUNITCOUNTERS also
+ * work with the control device (/dev/ipath_ctrl), if you don't want to
+ * use a port.
+ */
+
+/* return chip counters for current unit. */
+#define IPATH_GETCOUNTERS     _IOR('s', 40, struct infinipath_counters)
+/* return chip stats */
+#define IPATH_GETSTATS        _IOR('s', 41, struct infinipath_stats)
+/* return chip counters for a particular unit. */
+#define IPATH_GETUNITCOUNTERS _IOR('s', 42, struct infinipath_getunitcounters)
+
+/*
+ * unit is incoming unit number.
+ * data is a pointer to the infinipath_counters structure.
+ */
+struct infinipath_getunitcounters {
+	uint16_t unit;
+	uint64_t data;
+};
+
+/*
+ * The value in the BTH QP field that InfiniPath uses to differentiate
+ * an infinipath protocol IB packet vs standard IB transport
+ */
+#define IPATH_KD_QP 0x656b79
+
+/*
+ * valid states passed to ipath_set_linkstate() user call
+ * (IPATH_SET_LINKSTATE ioctl)
+ */
+#define IPATH_IB_LINKDOWN    0
+#define IPATH_IB_LINKARM     1
+#define IPATH_IB_LINKACTIVE  2
+
+/*
+ * stats maintained by the driver.  For now, at least, this is global
+ * to all minor devices.
+ */
+struct infinipath_stats {
+	uint64_t sps_ints;	/* number of interrupts taken */
+	uint64_t sps_errints;	/* number of interrupts for errors */
+	/* number of errors from chip (not including packet errors or CRC) */
+	uint64_t sps_errs;
+	/* number of packet errors from chip other than CRC */
+	uint64_t sps_pkterrs;
+	/* number of packets with CRC errors (ICRC and VCRC) */
+	uint64_t sps_crcerrs;
+	/* number of hardware errors reported (parity, etc.) */
+	uint64_t sps_hwerrs;
+	/* number of times IB link changed state unexpectedly */
+	uint64_t sps_iblink;
+	uint64_t sps_unused3;	/* no longer used; left for compatibility */
+	uint64_t sps_port0pkts; /* number of kernel (port0) packets received */
+	/* number of "ethernet" packets sent by driver */
+	uint64_t sps_ether_spkts;
+	/* number of "ethernet" packets received by driver */
+	uint64_t sps_ether_rpkts;
+	uint64_t sps_sma_spkts;	/* number of SMA packets sent by driver */
+	uint64_t sps_sma_rpkts;	/* number of SMA packets received by driver */
+	/* number of times all ports rcvhdrq was full and packet dropped */
+	uint64_t sps_hdrqfull;
+	/* number of times all ports egrtid was full and packet dropped */
+	uint64_t sps_etidfull;
+	/*
+	 * number of times we tried to send from driver, but no pio
+	 * buffers avail
+	 */
+	uint64_t sps_nopiobufs;
+	uint64_t sps_ports;	/* number of ports currently open */
+	/* list of pkeys (other than default) accepted (0 means not set) */
+	uint16_t sps_pkeys[4];
+	/* lids for up to 4 infinipaths, indexed by infinipath # */
+	uint16_t sps_lid[4];
+	/* number of user ports per chip (not IB ports) */
+	uint32_t sps_nports;
+	uint32_t sps_nullintr;	/* not our interrupt, or already handled */
+	uint32_t sps_maxpkts_call;  /* max number of packets handled per receive call */
+	uint32_t sps_avgpkts_call;  /* avg number of packets handled per receive call */
+	uint64_t sps_pagelocks;	/* total number of pages ipath_mlock()'ed */
+	/* total number of pages ipath_munlock()'ed */
+	uint64_t sps_pageunlocks;
+	/*
+	 * Number of packets dropped in kernel other than errors
+	 * (ether packets if ipath not configured, sma/mad, etc.)
+	 */
+	uint64_t sps_krdrops;
+	/* mlids for up to 4 infinipaths, indexed by infinipath # */
+	uint16_t sps_mlid[4];
+	uint64_t __sps_pad[45];	/* pad for future growth */
+};
+
+/*
+ * These are the status bits returned (in ascii form, 64bit value)
+ * by the IPATH_GETSTATS ioctl.
+ */
+#define IPATH_STATUS_INITTED       0x1	/* basic driver initialization done */
+#define IPATH_STATUS_DISABLED      0x2  /* hardware disabled */
+#define IPATH_STATUS_UNUSED        0x4	/* available */
+#define IPATH_STATUS_OIB_SMA       0x8	/* ipath_mad kernel SMA running */
+#define IPATH_STATUS_SMA          0x10	/* user SMA running */
+/* Chip (simulator) has been found and initted */
+#define IPATH_STATUS_CHIP_PRESENT 0x20
+#define IPATH_STATUS_IB_READY     0x40	/* IB link is at ACTIVE, has LID,
+										 * usable for all VL's */
+/* after link up, LID,MTU,etc. has been configured */
+#define IPATH_STATUS_IB_CONF      0x80
+/* no link established, probably no cable */
+#define IPATH_STATUS_IB_NOCABLE  0x100
+/* A Fatal hardware error has occurred. */
+#define IPATH_STATUS_HWERROR     0x200
+
+/* The list of usermode accessible registers.  Also see Reg_* later in file */
+typedef enum _ipath_ureg {
+	ur_rcvhdrtail = 0,	/* (RO)  DMA RcvHdr to be used next. */
+	/* (RW)  RcvHdr entry to be processed next by host. */
+	ur_rcvhdrhead = 1,
+	ur_rcvegrindextail = 2,	/* (RO)  Index of next Eager index to use. */
+	ur_rcvegrindexhead = 3,	/* (RW)  Eager TID to be processed next */
+	/* For internal use only; max register number. */
+	_IPATH_UregMax
+} ipath_ureg;
+
+/* SMA minor# no portinfo, one for all instances */
+#define IPATH_SMA 128
+
+/* Control minor# no portinfo, one for all instances */
+#define IPATH_CTRL 130
+
+/*
+ * This structure is returned by ipath_userinit() immediately after open
+ * to get implementation-specific info, and info specific to this
+ * instance.
+ */
+struct ipath_base_info {
+	/* version of hardware, for feature checking. */
+	uint32_t spi_hw_version;
+	/* version of software, for feature checking. */
+	uint32_t spi_sw_version;
+	/* InfiniPath port assigned, goes into sent packets */
+	uint32_t spi_port;
+	/*
+	 * IB MTU, packets IB data must be less than this.
+	 * The MTU is in bytes, and will be a multiple of 4 bytes.
+	 */
+	uint32_t spi_mtu;
+	/*
+	 * size of a PIO buffer.  Any given packet's total
+	 * size must be less than this (in words).  Included is the
+	 * starting control word, so if 513 is returned, then total
+	 * pkt size is 512 words or less.
+	 */
+	uint32_t spi_piosize;
+	/* size of the TID cache in infinipath, in entries */
+	uint32_t spi_tidcnt;
+	/* size of the TID Eager list in infinipath, in entries */
+	uint32_t spi_tidegrcnt;
+	/* size of a single receive header queue entry. */
+	uint32_t spi_rcvhdrent_size;
+	/* Count of receive header queue entries allocated.
+	 * This may be less than the spu_rcvhdrcnt passed in!.
+	 */
+	uint32_t spi_rcvhdr_cnt;
+
+	uint32_t __32_bit_compatibility_pad;	/* DO NOT  MOVE OR REMOVE */
+
+	/* address where receive buffer queue is mapped into */
+	uint64_t spi_rcvhdr_base;
+
+	/* user program. */
+
+	/* base address of eager TID receive buffers. */
+	uint64_t spi_rcv_egrbufs;
+
+	/* Allocated by initialization code, not by protocol. */
+
+	/* size of each TID buffer in host memory,
+	 * starting at spi_rcv_egrbufs.  It includes spu_egrskip, and is
+	 * at least spi_mtu bytes, and the buffers are virtually contiguous
+	 */
+	uint32_t spi_rcv_egrbufsize;
+	/*
+	 * The special QP (queue pair) value that identifies an infinipath
+	 * protocol packet from standard IB packets.  More, probably much
+	 * more, to be added.
+	 */
+	uint32_t spi_qpair;
+
+	/*
+	 * user register base for init code, not to be used directly by
+	 * protocol or applications
+	 */
+	uint64_t __spi_uregbase;
+	/*
+	 * maximum buffer size in bytes that can be used in a
+	 * single TID entry (assuming the buffer is aligned to this boundary).
+	 * This is the minimum of what the hardware and software support
+	 * Guaranteed to be a power of 2.
+	 */
+	uint32_t spi_tid_maxsize;
+	/*
+	 * alignment of each pio send buffer (byte count
+	 * to add to spi_piobufbase to get to second buffer)
+	 */
+	uint32_t spi_pioalign;
+	/*
+	 * the index of the first pio buffer available
+	 * to this process; needed to do lookup in spi_pioavailaddr; not added 
+	 * to spi_piobufbase
+	 */
+	uint32_t spi_pioindex;
+	uint32_t spi_piocnt;	/* number of buffers mapped for this process */
+
+	/*
+	 * base address of writeonly pio buffers for this process.
+	 * Each buffer has spi_piosize words, and is aligned on spi_pioalign
+	 * boundaries.  spi_piocnt buffers are mapped from this address
+	 */
+	uint64_t spi_piobufbase;
+
+	/*
+	 * base address of readonly memory copy of the pioavail registers.
+	 * There are 2 bits for each buffer.
+	 */
+	uint64_t spi_pioavailaddr;
+
+	/*
+	 * Address where driver updates a copy
+	 * of the interface and driver status (IPATH_STATUS_*) as a 64 bit value
+	 * It's followed by a string indicating hardware error, if there was one
+	 */
+	uint64_t spi_status;
+
+	/* number of chip ports available to user processes */
+	uint32_t spi_nports;
+	uint32_t spi_unit;	/* unit number of chip we are using */
+	uint32_t spi_rcv_egrperchunk;	/* num bufs in each contiguous set */
+	/* size in bytes of each contiguous set */
+	uint32_t spi_rcv_egrchunksize;
+	/* total size of mmap to cover full rcvegrbuffers */
+	uint32_t spi_rcv_egrbuftotlen;
+	/*
+	 * ioctl cmd includes struct size, so pad out, and adjust down as
+	 * new fields are added to keep size constant
+	 */
+	uint32_t __spi_pad[19];
+} __attribute__ ((aligned(8)));
+
+#define IPATH_WAIT_RCV   0x1	/* IPATH_WAIT, receive */
+#define IPATH_WAIT_PIO   0x2	/* IPATH_WAIT, PIO */
+
+/*
+ * This version number is given to the driver by the user code during
+ * initialization in the spu_userversion field of ipath_user_info, so
+ * the driver can check for compatibility with user code.
+ *
+ * The major version changes when data structures
+ * change in an incompatible way.  The driver must be the same or higher
+ * for initialization to succeed.  In some cases, a higher version
+ * driver will not interoperate with older software, and initialization
+ * will return an error.
+ */
+#define IPATH_USER_SWMAJOR 1
+
+/*
+ * Minor version differences are always compatible
+ * a within a major version, however if if user software is larger
+ * than driver software, some new features and/or structure fields
+ * may not be implemented; the user code must deal with this if it
+ * cares, or it must abort after initialization reports the difference
+ */
+#define IPATH_USER_SWMINOR 2
+
+#define IPATH_USER_SWVERSION ((IPATH_USER_SWMAJOR<<16) | IPATH_USER_SWMINOR)
+
+/* Similarly, this is the kernel version going back to the user.  It's slightly
+ * different, in that we want to tell if the driver was built as part of a
+ * PathScale release, or from the driver from the OpenIB, kernel.org, or a
+ * standard distribution, for support reasons.  The high bit is 0 for
+ * non-PathScale, and 1 for PathScale-built/supplied.  That bit is defined
+ * in Makefiles, rather than this file.
+ *
+ * It's returned by the driver to the user code during initialization
+ * in the spi_sw_version field of ipath_base_info, so the user code can
+ * in turn check for compatibility with the kernel.
+*/
+#define IPATH_KERN_SWVERSION ((IPATH_KERN_TYPE<<31) | IPATH_USER_SWVERSION)
+
+/*
+ * This structure is passed to ipath_userinit() to tell the driver where
+ * user code buffers are, sizes, etc.
+ */
+struct ipath_user_info {
+	/*
+	 * version of user software, to detect compatibility issues.
+	 * Should be set to IPATH_USER_SWVERSION.
+	 */
+	uint32_t spu_userversion;
+
+	/* desired number of receive header queue entries */
+	uint32_t spu_rcvhdrcnt;
+
+	/*
+	 * Leave this much unused space at the start of
+	 * each eager buffer for software use.  Similar in effect to
+	 * setting K_Offset to this value.  needs to be 'small', on the
+	 * order of one or two cachelines
+	 */
+	uint32_t spu_egrskip;
+
+	/*
+	 * number of words in KD protocol header
+	 * This tells InfiniPath how many words to copy to rcvhdrq.  If 0,
+	 * kernel uses a default.  Once set, attempts to set any other value
+	 * are an error (EAGAIN) until driver is reloaded.
+	 */
+	uint32_t spu_rcvhdrsize;
+
+	/*
+	 * cache line aligned (64 byte) user address to
+	 * which the rcvhdrtail register will be written by infinipath
+	 * whenever it changes, so that no chip registers are read in
+	 * the performance path.
+	 */
+	uint64_t spu_rcvhdraddr;
+
+	/*
+	 * ioctl cmd includes struct size, so pad out,
+	 * and adjust down as new fields are added to keep size constant
+	 */
+	uint32_t __spu_pad[6];
+} __attribute__ ((aligned(8)));
+
+struct ipath_iovec {
+	/* Pointer to data, but same size 32 and 64 bit */
+	uint64_t iov_base;
+
+	/*
+	 * Length of data; don't need 64 bits, but want
+	 * ipath_sendpkt to remain same size as before 32 bit changes, so...
+	 */
+	uint64_t iov_len;
+};
+
+/*
+ * Describes a single packet for send.  Each packet can have one or more
+ * buffers, but the total length (exclusive of IB headers) must be less
+ * than the MTU, and if using the PIO method, entire packet length,
+ * including IB headers, must be less than the ipath_piosize value (words).
+ * Use of this necessitates including sys/uio.h
+ */
+struct ipath_sendpkt {
+	uint32_t sps_flags;	/* flags for packet (TBD) */
+	uint32_t sps_cnt;	/* number of entries to use in sps_iov */
+	/* array of iov's describing packet. TEMPORARY */
+	struct ipath_iovec sps_iov[4];
+};
+
+struct _tidupd {		/* used only in inlined function for ioctl. */
+	uint32_t tidcnt;
+	uint32_t tid__unused;	/* make structure same size in 32 and 64 bit */
+	uint64_t tidvaddr;	/* virtual address of first page in transfer */
+	/* pointer (same size 32/64 bit) to uint16_t tid array */
+	uint64_t tidlist;
+
+	/*
+	 * pointer (same size 32/64 bit) to bitmap of TIDs used
+	 * for this call; checked for being large enough at open
+	 */
+	uint64_t tidmap;
+};
+
+struct ipath_setguid {		/* set GUID for interface */
+	uint64_t sguid;		/* in network order */
+	uint64_t sunit;		/* unit number of interface */
+};
+
+/*
+ * Structure used to send data to and receive data from a diags ioctl.
+ *
+ * NOTE: For HT reads and writes, we only support byte, word (16bits) and
+ * dword (32bits).  All other sizes for HT are invalid.
+ */
+struct ipath_diag_info {
+	uint64_t _base_offset;	/* register to start reading from */
+	uint64_t _num_bytes;	/* number of bytes to read or write */
+	/*
+	 * address in user space.
+	 * for reads, this is the address to store the read result(s).
+	 * for writes, it the address to get the write data from.
+	 * This memory better be valid in user space!
+	 */
+	uint64_t _uaddress;
+	uint64_t _unit;		/* Unit ID of chip we are accessing. */
+	uint64_t _pad[15];
+};
+
+/*
+ * Data layout in I2C flash (for GUID, etc.)
+ * All fields are little-endian binary unless otherwise stated
+ */
+#define IPATH_FLASH_VERSION 1
+struct ipath_flash {
+	uint8_t if_fversion;	/* flash layout version (IPATH_FLASH_VERSION) */
+	uint8_t if_csum;	/* checksum protecting if_length bytes */
+	/*
+	 * valid length (in use, protected by if_csum), including if_fversion
+	 * and if_sum themselves)
+	 */
+	uint8_t if_length;
+	uint8_t if_guid[8];	/* the GUID, in network order */
+	/* number of GUIDs to use, starting from if_guid */
+	uint8_t if_numguid;
+	uint8_t if_serial[12];	/* the board serial number, in ASCII */
+	uint8_t if_mfgdate[8];	/* board mfg date (YYYYMMDD ASCII) */
+	/* last board rework/test date (YYYYMMDD ASCII) */
+	uint8_t if_testdate[8];
+	uint8_t if_errcntp[4];	/* logging of error counts, TBD */
+	/* powered on hours, updated at driver unload */
+	uint8_t if_powerhour[2];
+	uint8_t if_comment[32];	/* ASCII free-form comment field */
+	uint8_t if_future[50];	/* 78 bytes used, min flash size is 128 bytes */
+};
+
+uint8_t ipath_flash_csum(struct ipath_flash *, int);
+
+/*
+ * These are the counters implemented in the chip, and are listed in order.
+ * They are returned in this order by the IPATH_GETCOUNTERS ioctl
+ */
+struct infinipath_counters {
+	unsigned long long LBIntCnt;
+	unsigned long long LBFlowStallCnt;
+	unsigned long long Reserved1;
+	unsigned long long TxUnsupVLErrCnt;
+	unsigned long long TxDataPktCnt;
+	unsigned long long TxFlowPktCnt;
+	unsigned long long TxDwordCnt;
+	unsigned long long TxLenErrCnt;
+	unsigned long long TxMaxMinLenErrCnt;
+	unsigned long long TxUnderrunCnt;
+	unsigned long long TxFlowStallCnt;
+	unsigned long long TxDroppedPktCnt;
+	unsigned long long RxDroppedPktCnt;
+	unsigned long long RxDataPktCnt;
+	unsigned long long RxFlowPktCnt;
+	unsigned long long RxDwordCnt;
+	unsigned long long RxLenErrCnt;
+	unsigned long long RxMaxMinLenErrCnt;
+	unsigned long long RxICRCErrCnt;
+	unsigned long long RxVCRCErrCnt;
+	unsigned long long RxFlowCtrlErrCnt;
+	unsigned long long RxBadFormatCnt;
+	unsigned long long RxLinkProblemCnt;
+	unsigned long long RxEBPCnt;
+	unsigned long long RxLPCRCErrCnt;
+	unsigned long long RxBufOvflCnt;
+	unsigned long long RxTIDFullErrCnt;
+	unsigned long long RxTIDValidErrCnt;
+	unsigned long long RxPKeyMismatchCnt;
+	unsigned long long RxP0HdrEgrOvflCnt;
+	unsigned long long RxP1HdrEgrOvflCnt;
+	unsigned long long RxP2HdrEgrOvflCnt;
+	unsigned long long RxP3HdrEgrOvflCnt;
+	unsigned long long RxP4HdrEgrOvflCnt;
+	unsigned long long RxP5HdrEgrOvflCnt;
+	unsigned long long RxP6HdrEgrOvflCnt;
+	unsigned long long RxP7HdrEgrOvflCnt;
+	unsigned long long RxP8HdrEgrOvflCnt;
+	unsigned long long Reserved6;
+	unsigned long long Reserved7;
+	unsigned long long IBStatusChangeCnt;
+	unsigned long long IBLinkErrRecoveryCnt;
+	unsigned long long IBLinkDownedCnt;
+	unsigned long long IBSymbolErrCnt;
+};
+
+/*
+ * The next set of defines are for packet headers, and chip register
+ * and memory bits that are visible to and/or used by user-mode software
+ * The other bits that are used only by the driver or diags are in 
+ * ipath_registers.h
+ */
+
+/* RcvHdrFlags bits */
+#define INFINIPATH_RHF_LENGTH_MASK 0x7FF
+#define INFINIPATH_RHF_LENGTH_SHIFT 0
+#define INFINIPATH_RHF_RCVTYPE_MASK 0x7
+#define INFINIPATH_RHF_RCVTYPE_SHIFT 11
+#define INFINIPATH_RHF_EGRINDEX_MASK 0x7FF
+#define INFINIPATH_RHF_EGRINDEX_SHIFT 16
+#define INFINIPATH_RHF_H_ICRCERR   0x80000000
+#define INFINIPATH_RHF_H_VCRCERR   0x40000000
+#define INFINIPATH_RHF_H_PARITYERR 0x20000000
+#define INFINIPATH_RHF_H_LENERR    0x10000000
+#define INFINIPATH_RHF_H_MTUERR    0x08000000
+#define INFINIPATH_RHF_H_IHDRERR   0x04000000
+#define INFINIPATH_RHF_H_TIDERR    0x02000000
+#define INFINIPATH_RHF_H_MKERR     0x01000000
+#define INFINIPATH_RHF_H_IBERR     0x00800000
+#define INFINIPATH_RHF_L_SWA       0x00008000
+#define INFINIPATH_RHF_L_SWB       0x00004000
+
+/* infinipath header fields */
+#define INFINIPATH_I_VERS_MASK 0xF
+#define INFINIPATH_I_VERS_SHIFT 28
+#define INFINIPATH_I_PORT_MASK 0xF
+#define INFINIPATH_I_PORT_SHIFT 24
+#define INFINIPATH_I_TID_MASK 0x7FF
+#define INFINIPATH_I_TID_SHIFT 13
+#define INFINIPATH_I_OFFSET_MASK 0x1FFF
+#define INFINIPATH_I_OFFSET_SHIFT 0
+
+/* K_PktFlags bits */
+#define INFINIPATH_KPF_INTR 0x1
+
+/* SendPIO per-buffer control */
+#define INFINIPATH_SP_LENGTHP1_MASK 0x3FF
+#define INFINIPATH_SP_LENGTHP1_SHIFT 0
+#define INFINIPATH_SP_INTR    0x80000000
+#define INFINIPATH_SP_TEST    0x40000000
+#define INFINIPATH_SP_TESTEBP 0x20000000
+
+/* SendPIOAvail bits */
+#define INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT 1
+#define INFINIPATH_SENDPIOAVAIL_CHECK_SHIFT 0
+
+#endif				/* _IPATH_COMMON_H */
drivers/infiniband/hw/ipath/ipath_kernel.h new file +776 -0

776 lines not on this page · raw message

drivers/infiniband/hw/ipath/ipath_layer.h new file +131 -0

131 lines not on this page · raw message

drivers/infiniband/hw/ipath/ipath_registers.h new file +359 -0

359 lines not on this page · raw message

drivers/infiniband/hw/ipath/ips_common.h new file +221 -0

221 lines not on this page · raw message

Signature · 1 line
0.99.9n
#3 lore raw

RFC PATCH 2/13 ipath debug header

Debugging macros for ipath driver

---

 drivers/infiniband/hw/ipath/ipath_debug.h |  211 +++++++++++++++++++++++++++++
 1 files changed, 211 insertions(+), 0 deletions(-)
 create mode 100644 drivers/infiniband/hw/ipath/ipath_debug.h

8f834f72344a3c7e9c5eafdf59b9fc96b4e08e5f

drivers/infiniband/hw/ipath/ipath_debug.h new file +211 -0
new file mode 100644
index 0000000..c8b7374
@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2003, 2004, 2005. PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Patent licenses, if any, provided herein do not apply to
+ * combinations of this program with other software, or any other
+ * product whatsoever.
+ *
+ * $Id: ipath_debug.h 4504 2005-12-16 06:15:47Z rjwalsh $
+ */
+
+#ifndef _IPATH_DEBUG_H
+#define _IPATH_DEBUG_H
+
+/*
+ * This file contains tracing code that is lightweight, and can be
+ * called from both user and kernel mode
+ * _IPATH_DBG should have the same calling conventions and semantics
+ * for both user and kernel.
+ */
+#ifndef _IPATH_DEBUGGING	/* tracing enabled or not */
+#define _IPATH_DEBUGGING 1
+#endif
+
+/* This macros should only be used in the trace library code. */
+#define _IPATH_DEBUG_VARS_DECL unsigned infinipath_debug;
+extern unsigned infinipath_debug;
+extern const char *ipath_get_unit_name(int unit);
+
+/*
+ * These are always defined, because _IPATH_ERROR is always defined,
+ * unlike the other debugging calls.  It might make sense to change
+ * to using "fprintf(stderr", for the usermode version, but not now.
+ */
+#ifdef __KERNEL__
+#define __IPPRT printk
+#define __IPATH_UNIT_ERRID(unit) ipath_get_unit_name(unit)
+#define __IPATH_ERRID "infinipath"
+#else
+#define __IPPRT printf
+extern char *__progname;
+#define __IPATH_UNIT_ERRID(unit) __progname
+#define __IPATH_ERRID __progname
+#define KERN_ERR
+#endif
+
+#if _IPATH_DEBUGGING
+
+/*
+ * Mask values for debugging.  The scheme allows us to compile out any of
+ * the debug tracing stuff, and if compiled in, to enable or disable dynamically
+ * This can be set at modprobe time also:
+ *      modprobe infinipath.ko infinipath_debug=7
+ */
+#define __IPATH_INFO        0x1	/* generic low verbosity stuff */
+#define __IPATH_DBG         0x2	/* generic debug */
+#define __IPATH_TRSAMPLE    0x8	/* generate trace buffer sample entries */
+/* leave some low verbosity spots open */
+#define __IPATH_VERBDBG     0x40	/* very verbose debug */
+#define __IPATH_PKTDBG      0x80	/* print packet data */
+/* print process startup (init)/exit messages */
+#define __IPATH_PROCDBG     0x100
+/* print mmap/nopage stuff, not using VDBG any more */
+#define __IPATH_MMDBG       0x200
+#define __IPATH_USER_SEND   0x1000	/* use user mode send */
+#define __IPATH_KERNEL_SEND 0x2000	/* use kernel mode send */
+#define __IPATH_EPKTDBG     0x4000	/* print ethernet packet data */
+#define __IPATH_SMADBG      0x8000	/* sma packet debug */
+#define __IPATH_IPATHDBG    0x10000	/* Ethernet (IPATH) general debug on */
+#define __IPATH_IPATHWARN   0x20000	/* Ethernet (IPATH) warnings on */
+#define __IPATH_IPATHERR    0x40000	/* Ethernet (IPATH) errors on */
+#define __IPATH_IPATHPD     0x80000	/* Ethernet (IPATH) packet dump on */
+#define __IPATH_IPATHTABLE  0x100000	/* Ethernet (IPATH) table dump on */
+
+#ifdef __KERNEL__
+#define __IPIDENT
+#define __IPIDENT_ARG
+#define __IP_INFO_TAG __IPATH_ERRID
+#define _Pragma_unlikely
+#else
+#define KERN_INFO
+#define KERN_DEBUG
+#define __IPIDENT "%s"
+#define __IPIDENT_ARG __ipath_mylabel,
+#define __IP_INFO_TAG __func__
+extern char *__ipath_mylabel;
+extern void ipath_set_mylabel(char *);
+#define _Pragma_unlikely _Pragma("mips_frequency_hint never")
+#endif
+
+#define _IPATH_UNIT_ERROR(unit,fmt,...) do { \
+        _Pragma_unlikely \
+        __IPPRT (KERN_ERR  __IPIDENT "%s: " fmt, __IPIDENT_ARG __IPATH_UNIT_ERRID(unit), \
+                ##__VA_ARGS__); \
+        } while(0)
+
+#define _IPATH_ERROR(fmt,...) do { \
+        _Pragma_unlikely \
+        __IPPRT (KERN_ERR  __IPIDENT "%s: " fmt, __IPIDENT_ARG __IPATH_ERRID, \
+                ##__VA_ARGS__); \
+        } while(0)
+
+#define _IPATH_INFO(fmt,...) do { \
+        _Pragma_unlikely \
+        if(unlikely(infinipath_debug&__IPATH_INFO))  \
+            __IPPRT (KERN_INFO __IPIDENT "%s: " fmt,\
+              __IPIDENT_ARG __IP_INFO_TAG,##__VA_ARGS__); \
+        } while(0)
+
+#define  __IPATH_USER_MODE_SEND unlikely(infinipath_debug & __IPATH_USER_SEND)
+#define  __IPATH_KERNEL_MODE_SEND unlikely(infinipath_debug & __IPATH_KERNEL_SEND)
+#define  __IPATH_PKTDBG_ON unlikely(infinipath_debug & __IPATH_PKTDBG)
+
+#define  __IPATH_DBG_WHICH(which,fmt,...) do { \
+        _Pragma_unlikely \
+        if(unlikely(infinipath_debug&(which)))  __IPPRT (KERN_DEBUG __IPIDENT "%s: " fmt,\
+            __IPIDENT_ARG __func__,##__VA_ARGS__); \
+        } while(0)
+#define  _IPATH_DBG(fmt,...) __IPATH_DBG_WHICH(__IPATH_DBG,fmt,##__VA_ARGS__)
+#define  _IPATH_VDBG(fmt,...) __IPATH_DBG_WHICH(__IPATH_VERBDBG,fmt,##__VA_ARGS__)
+#define  _IPATH_PDBG(fmt,...) __IPATH_DBG_WHICH(__IPATH_PKTDBG,fmt,##__VA_ARGS__)
+#define  _IPATH_EPDBG(fmt,...) __IPATH_DBG_WHICH(__IPATH_EPKTDBG,fmt,##__VA_ARGS__)
+#define  _IPATH_PRDBG(fmt,...) __IPATH_DBG_WHICH(__IPATH_PROCDBG,fmt,##__VA_ARGS__)
+#define  _IPATH_MMDBG(fmt,...) __IPATH_DBG_WHICH(__IPATH_MMDBG,fmt,##__VA_ARGS__)
+#define  _IPATH_SMADBG(fmt,...) __IPATH_DBG_WHICH(__IPATH_SMADBG,fmt,##__VA_ARGS__)
+#define  _IPATH_IPATHDBG(fmt,...) __IPATH_DBG_WHICH(__IPATH_IPATHDBG,fmt,##__VA_ARGS__)
+#define  _IPATH_IPATHWARN(fmt,...) __IPATH_DBG_WHICH(__IPATH_IPATHWARN,fmt,##__VA_ARGS__)
+#define  _IPATH_IPATHERR(fmt,...) __IPATH_DBG_WHICH(__IPATH_IPATHERR ,fmt,##__VA_ARGS__)
+#define  _IPATH_IPATHPD(fmt,...) __IPATH_DBG_WHICH(__IPATH_IPATHPD  ,fmt,##__VA_ARGS__)
+#define  _IPATH_IPATHTABLE(fmt,...) __IPATH_DBG_WHICH(__IPATH_IPATHTABLE  ,fmt,##__VA_ARGS__)
+
+#else				/* ! _IPATH_DEBUGGING */
+
+#define _IPATH_UNIT_ERROR(unit,fmt,...) do { \
+        __IPPRT (KERN_ERR  "%s" fmt, "",##__VA_ARGS__); \
+        } while(0)
+
+#define _IPATH_ERROR(fmt,...) do { \
+        __IPPRT (KERN_ERR  "%s" fmt, "",##__VA_ARGS__); \
+        } while(0)
+
+#define _IPATH_INFO(fmt,...)
+#define _IPATH_DBG(fmt,...)
+#define _IPATH_PDBG(fmt,...)
+#define _IPATH_EPDBG(fmt,...)
+#define _IPATH_PRDBG(fmt,...)
+#define _IPATH_VDBG(fmt,...)
+#define _IPATH_MMDBG(fmt,...)
+#define _IPATH_SMADBG(fmt,...)
+#define _IPATH_IPATHDBG(fmt,...)
+#define _IPATH_IPATHWARN(fmt,...)
+#define _IPATH_IPATHERR(fmt,...)
+#define _IPATH_IPATHPD(fmt,...)
+#define _IPATH_IPATHTABLE(fmt,...)
+
+/*
+ * define all of these even with debugging off, for the few places that do
+ * if(infinipath_debug&_IPATH_xyzzy), but in a way that will make the
+ * compiler eliminate the code
+ */
+#define __IPATH_INFO      0x0	/* generic low verbosity stuff */
+#define __IPATH_DBG       0x0	/* generic debug */
+#define __IPATH_CALL      0x0	/* function call entrance/exit */
+#define __IPATH_TRSAMPLE  0x0	/* generate trace buffer sample entries */
+#define __IPATH_VCALL     0x0	/* function call entrance/exit */
+#define __IPATH_VERBDBG   0x0	/* very verbose debug */
+#define __IPATH_PKTDBG    0x0	/* print packet data */
+#define __IPATH_PROCDBG   0x0	/* print process startup (init)/exit messages */
+/* print mmap/nopage stuff, not using VDBG any more */
+#define __IPATH_MMDBG     0x0
+#define __IPATH_EPKTDBG   0x0	/* print ethernet packet data */
+#define __IPATH_SMADBG    0x0	/* print process startup (init)/exit messages */
+#define __IPATH_IPATHDBG  0x0	/* Ethernet (IPATH) table dump on */
+#define __IPATH_IPATHWARN 0x0	/* Ethernet (IPATH) warnings on   */
+#define __IPATH_IPATHERR  0x0	/* Ethernet (IPATH) errors on   */
+#define __IPATH_IPATHPD   0x0	/* Ethernet (IPATH) packet dump on   */
+#define __IPATH_IPATHTABLE 0x0	/* Ethernet (IPATH) packet dump on   */
+#define  __IPATH_USER_MODE_SEND 0
+#define  __IPATH_KERNEL_MODE_SEND 0
+#define  __IPATH_PKTDBG_ON 0
+
+#endif				/* _IPATH_DEBUGGING */
+
+#endif				/* _IPATH_DEBUG_H */
Signature · 1 line
0.99.9n
#4 lore raw

RFC PATCH 3/13 ipath copy routines

Copy routines for ipath driver

---

 drivers/infiniband/hw/ipath/ipath_copy.c     |  666 ++++++++++++++++++++++++++
 drivers/infiniband/hw/ipath/ipath_dwordcpy.S |   62 ++
 2 files changed, 728 insertions(+), 0 deletions(-)
 create mode 100644 drivers/infiniband/hw/ipath/ipath_copy.c
 create mode 100644 drivers/infiniband/hw/ipath/ipath_dwordcpy.S

99f636a78e0d759ab663a7abb29e6a71b32a552d

drivers/infiniband/hw/ipath/ipath_copy.c new file +666 -0
new file mode 100644
index 0000000..26211ad
@@ -0,0 +1,666 @@
+/*
+ * Copyright (c) 2003, 2004, 2005. PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Patent licenses, if any, provided herein do not apply to
+ * combinations of this program with other software, or any other
+ * product whatsoever.
+ *
+ * $Id: ipath_copy.c 4365 2005-12-10 00:04:16Z rjwalsh $
+ */
+
+/*
+ * This file provides support for doing sk_buff buffer swapping between
+ * the low level driver eager buffers, and the network layer.  It's part
+ * of the core driver, rather than the ether driver, because it relies
+ * on variables and functions in the core driver.  It exports a single
+ * entry point for use in the ipath_ether module.
+ */
+
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <asm/io.h>
+#include <asm/byteorder.h>
+#include <asm/bitops.h>
+#include <linux/skbuff.h>
+#include <linux/netdevice.h>
+
+#include <linux/crc32.h>	/* we can generate our own crc's for testing */
+
+#include "ipath_kernel.h"
+#include "ips_common.h"
+#include "ipath_layer.h"
+
+#define TRUE 1
+#define FALSE 0
+
+/*
+ * Allocate a PIO send buffer, initialize the header and copy it out.
+ */
+static int layer_send_getpiobuf(struct copy_data_s *cdp)
+{
+	int whichpb;
+	uint32_t device = cdp->device;
+	uint32_t extra_bytes;
+	uint32_t len, nwords;
+	uint32_t *piobuf;
+
+	whichpb = ipath_getpiobuf(device);
+	if (whichpb < 0) {
+		cdp->error = whichpb;
+		return whichpb;
+	}
+
+	/*
+	 * Compute the max amount of data that can fit into a PIO buffer.
+	 * buffer size - header size - trigger qword length & flags - CRC
+	 */
+	len = devdata[device].ipath_ibmaxlen -
+	    sizeof(ether_header_typ) - 8 - (SIZE_OF_CRC << 2);
+	if (len > (cdp->len + cdp->extra))
+		len = (cdp->len + cdp->extra);
+	/* Compute word aligment (i.e., (len & 3) ? 4 - (len & 3) : 0) */
+	extra_bytes = (4 - len) & 3;
+	nwords = (sizeof(ether_header_typ) + len + extra_bytes) >> 2;
+	cdp->hdr->lrh[2] = htons(nwords + SIZE_OF_CRC);
+	cdp->hdr->bth[0] = htonl((OPCODE_ITH4X << 24) + (extra_bytes << 20) +
+				 IPS_DEFAULT_P_KEY);
+	cdp->hdr->sub_opcode = OPCODE_ENCAP;
+
+	cdp->hdr->bth[2] = 0;
+	/* Generate an interrupt on the receive side for the last fragment. */
+	cdp->hdr->iph.pkt_flags = ((cdp->len+cdp->extra) == len) ? INFINIPATH_KPF_INTR : 0;
+	cdp->hdr->iph.chksum =
+	    (uint16_t) IPS_LRH_BTH +
+	    (uint16_t) (nwords + SIZE_OF_CRC) -
+	    (uint16_t) ((cdp->hdr->iph.ver_port_tid_offset >> 16) & 0xFFFF) -
+	    (uint16_t) (cdp->hdr->iph.ver_port_tid_offset & 0xFFFF) -
+	    (uint16_t) cdp->hdr->iph.pkt_flags;
+
+	piobuf = (uint32_t *) (((char *)(devdata[device].ipath_kregbase)) +
+			       devdata[device].ipath_piobufbase +
+			       whichpb * devdata[device].ipath_palign);
+	_IPATH_VDBG("send %d (%x %x %x %x %x %x %x)\n", nwords,
+		    cdp->hdr->lrh[0],
+		    cdp->hdr->lrh[1],
+		    cdp->hdr->lrh[2],
+		    cdp->hdr->lrh[3],
+		    cdp->hdr->bth[0], cdp->hdr->bth[1], cdp->hdr->bth[2]);
+	/*
+	 * Write len to control qword, no flags.
+	 * +1 is for the qword padding of pbc.
+	 */
+	*((uint64_t *) piobuf) = (uint64_t) (nwords + 1);
+	piobuf += 2;
+	ipath_dwordcpy(piobuf, (uint32_t *) cdp->hdr,
+		       sizeof(ether_header_typ) >> 2);
+	cdp->csum_pio = &((ether_header_typ *) piobuf)->csum;
+	cdp->to = piobuf + (sizeof(ether_header_typ) >> 2);
+	cdp->flen = nwords - (sizeof(ether_header_typ) >> 2);
+	cdp->hdr->frag_num++;
+	return 0;
+}
+
+/*
+ * Copy data out of one or a chain of sk_buffs, into the PIO buffer.
+ * Fragment an sk_buff into multiple IB packets if the amount of data is
+ * more than a single eager send.
+ * Offset and len are in bytes.
+ * Note that this function is recursive!
+ */
+static void copy_bits(const struct sk_buff *skb, unsigned int offset,
+		      unsigned int len, struct copy_data_s *cdp)
+{
+	unsigned int start = skb_headlen(skb);
+	unsigned int i, copy;
+	uint32_t n;
+	u8 *p;
+
+	/* Copy header. */
+	if ((int)(copy = start - offset) > 0) {
+		if (copy > len)
+			copy = len;
+		p = skb->data + offset;
+		offset += copy;
+		len -= copy;
+		/*
+		 * If the alignment buffer is not empty, fill it and write
+		 * it out.
+		 */
+		if (cdp->extra) {
+			if (cdp->extra == 4)
+				goto extra_copy_bits_done;
+
+			while (copy != 0) {
+				cdp->u.buf[cdp->extra] = *p++;
+				copy--;
+				cdp->offset++;
+				cdp->len--;
+
+				if (++cdp->extra == 4) {
+extra_copy_bits_done:
+					if (cdp->flen == 0
+					    && layer_send_getpiobuf(cdp) < 0)
+						return;
+					*cdp->to++ = cdp->u.w;
+					cdp->extra = 0;
+					cdp->flen -= 1;
+					break;
+				}
+			}
+		}
+		while (copy >= 4) {
+			if (cdp->flen == 0 && layer_send_getpiobuf(cdp) < 0)
+				return;
+			n = copy >> 2;
+			if (n > cdp->flen)
+				n = cdp->flen;
+			ipath_dwordcpy(cdp->to, (uint32_t *) p, n);
+			cdp->to += n;
+			cdp->flen -= n;
+			n <<= 2;
+			p += n;
+			cdp->offset += n;
+			cdp->len -= n;
+			copy -= n;
+		}
+		/*
+		 * Either cdp->extra is zero or copy is zero which means that
+		 * the loop here can't cause the alignment buffer to fill up.
+		 */
+		while (copy != 0) {
+			cdp->u.buf[cdp->extra++] = *p++;
+			copy--;
+			cdp->offset++;
+			cdp->len--;
+
+		}
+		if (len == 0)
+			return;
+	}
+
+	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+		unsigned int end;
+
+		end = start + frag->size;
+		if ((int)(copy = end - offset) > 0) {
+			u8 *vaddr;
+
+			if (copy > len)
+				copy = len;
+			vaddr = kmap_skb_frag(frag);
+			p = vaddr + frag->page_offset + offset - start;
+			offset += copy;
+			len -= copy;
+			/*
+			 * If the alignment buffer is not empty, fill
+			 * it and write it out.
+			 */
+			if (cdp->extra) {
+				if (cdp->extra == 4)
+					goto extra1_copy_bits_done;
+
+				while (copy != 0) {
+					cdp->u.buf[cdp->extra] = *p++;
+					copy--;
+					cdp->offset++;
+					cdp->len--;
+
+					if (++cdp->extra == 4) {
+extra1_copy_bits_done:
+						if (cdp->flen == 0
+						    && layer_send_getpiobuf(cdp)
+						    < 0)
+							return;
+						*cdp->to++ = cdp->u.w;
+						cdp->extra = 0;
+						cdp->flen -= 1;
+						break;
+					}
+				}
+			}
+			while (copy >= 4) {
+				if (cdp->flen == 0
+				    && layer_send_getpiobuf(cdp) < 0)
+					return;
+				n = copy >> 2;
+				if (n > cdp->flen)
+					n = cdp->flen;
+				ipath_dwordcpy(cdp->to, (uint32_t *) p, n);
+				cdp->to += n;
+				cdp->flen -= n;
+				n <<= 2;
+				p += n;
+				cdp->offset += n;
+				cdp->len -= n;
+				copy -= n;
+			}
+			/*
+			 * Either cdp->extra is zero or copy is zero
+			 * which means that the loop here can't cause
+			 * the alignment buffer to fill up.
+			 */
+			while (copy != 0) {
+				cdp->u.buf[cdp->extra++] = *p++;
+				copy--;
+				cdp->offset++;
+				cdp->len--;
+			}
+			kunmap_skb_frag(vaddr);
+
+			if (len == 0)
+				return;
+		}
+		start = end;
+	}
+
+	if (skb_shinfo(skb)->frag_list) {
+		struct sk_buff *list = skb_shinfo(skb)->frag_list;
+
+		for (; list; list = list->next) {
+			unsigned int end;
+
+			end = start + list->len;
+			if ((int)(copy = end - offset) > 0) {
+				if (copy > len)
+					copy = len;
+				copy_bits(list, offset - start, copy, cdp);
+				if (cdp->error || (len -= copy) == 0)
+					return;
+			}
+			start = end;
+		}
+	}
+	if (len)
+		cdp->error = -EFAULT;
+}
+
+/*
+ * Copy data out of one or a chain of sk_buffs, into the PIO buffer, generating
+ * the checksum as we go.
+ * Fragment an sk_buff into multiple IB packets if the amount of data is
+ * more than a single eager send.
+ * Offset and len are in bytes.
+ * Note that this function is recursive!
+ */
+static void copy_and_csum_bits(const struct sk_buff *skb, unsigned int offset,
+			       unsigned int len, struct copy_data_s *cdp)
+{
+	unsigned int start = skb_headlen(skb);
+	unsigned int i, copy;
+	unsigned int csum2;
+	uint32_t n;
+	u8 *p;
+
+	/* Copy header. */
+	if ((int)(copy = start - offset) > 0) {
+		if (copy > len)
+			copy = len;
+		p = skb->data + offset;
+		offset += copy;
+		len -= copy;
+		if (!cdp->checksum_calc) {
+			cdp->checksum_calc = TRUE;
+
+			csum2 = csum_partial(p, copy, 0);
+			cdp->csum = csum_block_add(cdp->csum, csum2, cdp->pos);
+			cdp->pos += copy;
+		}
+		/*
+		 * If the alignment buffer is not empty, fill it and
+		 * write it out.
+		 */
+		if (cdp->extra) {
+			if (cdp->extra == 4)
+				goto extra_copy_and_csum_bits_done;
+
+			while (copy != 0) {
+				cdp->u.buf[cdp->extra] = *p++;
+				copy--;
+				cdp->offset++;
+				cdp->len--;
+				if (++cdp->extra == 4) {
+extra_copy_and_csum_bits_done:
+					if (cdp->flen == 0
+					    && layer_send_getpiobuf(cdp) < 0)
+						return;
+					/*
+					 * write the checksum before
+					 * the last PIO write.
+					 */
+					if (cdp->flen == 1) {
+						*cdp->csum_pio =
+						    csum_fold(cdp->csum);
+						mb();
+					}
+					*cdp->to++ = cdp->u.w;
+					cdp->extra = 0;
+					cdp->flen -= 1;
+					break;
+				}
+			}
+		}
+
+		while (copy >= 4) {
+			if (cdp->flen == 0 && layer_send_getpiobuf(cdp) < 0)
+				return;
+
+			n = copy >> 2;
+			if (n > cdp->flen)
+				n = cdp->flen;
+			/* write the checksum before the last PIO write. */
+			if (cdp->flen == n) {
+				*cdp->csum_pio = csum_fold(cdp->csum);
+				mb();
+			}
+			ipath_dwordcpy(cdp->to, (uint32_t *) p, n);
+			cdp->to += n;
+			cdp->flen -= n;
+			n <<= 2;
+			p += n;
+			cdp->offset += n;
+			cdp->len -= n;
+			copy -= n;
+		}
+		/*
+		 * Either cdp->extra is zero or copy is zero which means that
+		 * the loop here can't cause the alignment buffer to fill up.
+		 */
+		while (copy != 0) {
+			cdp->u.buf[cdp->extra++] = *p++;
+			copy--;
+			cdp->offset++;
+			cdp->len--;
+		}
+
+		cdp->checksum_calc = FALSE;
+
+		if (len == 0)
+			return;
+	}
+
+	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+		unsigned int end;
+
+		end = start + frag->size;
+		if ((int)(copy = end - offset) > 0) {
+			u8 *vaddr;
+
+			if (copy > len)
+				copy = len;
+			vaddr = kmap_skb_frag(frag);
+			p = vaddr + frag->page_offset + offset - start;
+			offset += copy;
+			len -= copy;
+
+			if (!cdp->checksum_calc) {
+				cdp->checksum_calc = TRUE;
+
+				csum2 = csum_partial(p, copy, 0);
+				cdp->csum = csum_block_add(cdp->csum, csum2,
+							   cdp->pos);
+				cdp->pos += copy;
+			}
+			/*
+			 * If the alignment buffer is not empty, fill
+			 * it and write it out.
+			 */
+			if (cdp->extra) {
+				if (cdp->extra == 4)
+					goto extra1_copy_and_csum_bits_done;
+				while (copy != 0) {
+					cdp->u.buf[cdp->extra] = *p++;
+					copy--;
+					cdp->offset++;
+					cdp->len--;
+
+					if (++cdp->extra == 4) {
+extra1_copy_and_csum_bits_done:
+						if (cdp->flen == 0
+						    && layer_send_getpiobuf(cdp)
+						    < 0) {
+							kunmap_skb_frag(vaddr);
+							return;
+						}
+						/*
+						 * write the checksum
+						 * before the last PIO
+						 * write.
+						 */
+						if (cdp->flen == 1) {
+							*cdp->csum_pio =
+							    csum_fold(cdp->
+								      csum);
+							mb();
+						}
+						*cdp->to++ = cdp->u.w;
+						cdp->extra = 0;
+						cdp->flen -= 1;
+						break;
+					}
+				}
+			}
+			while (copy >= 4) {
+				if (cdp->flen == 0
+				    && layer_send_getpiobuf(cdp) < 0) {
+					kunmap_skb_frag(vaddr);
+					return;
+				}
+				n = copy >> 2;
+				if (n > cdp->flen)
+					n = cdp->flen;
+				/*
+				 * write the checksum before the last
+				 * PIO write.
+				 */
+				if (cdp->flen == n) {
+					*cdp->csum_pio = csum_fold(cdp->csum);
+					mb();
+				}
+				ipath_dwordcpy(cdp->to, (uint32_t *) p, n);
+				cdp->to += n;
+				cdp->flen -= n;
+				n <<= 2;
+				p += n;
+				cdp->offset += n;
+				cdp->len -= n;
+				copy -= n;
+			}
+			/*
+			 * Either cdp->extra is zero or copy is zero
+			 * which means that the loop here can't cause
+			 * the alignment buffer to fill up.
+			 */
+			while (copy != 0) {
+				cdp->u.buf[cdp->extra++] = *p++;
+				copy--;
+				cdp->offset++;
+				cdp->len--;
+			}
+			kunmap_skb_frag(vaddr);
+
+			cdp->checksum_calc = FALSE;
+
+			if (len == 0)
+				return;
+		}
+		start = end;
+	}
+
+	if (skb_shinfo(skb)->frag_list) {
+		struct sk_buff *list = skb_shinfo(skb)->frag_list;
+
+		for (; list; list = list->next) {
+			unsigned int end;
+
+			end = start + list->len;
+			if ((int)(copy = end - offset) > 0) {
+				if (copy > len)
+					copy = len;
+				copy_and_csum_bits(list, offset - start, copy, cdp);
+				if (cdp->error || (len -= copy) == 0)
+					return;
+				offset += copy;
+			}
+			start = end;
+		}
+	}
+	if (len)
+		cdp->error = -EFAULT;
+}
+
+/*
+ * Note that the header should have the unchanging parts
+ * initialized but the rest of the header is computed as needed in
+ * order to break up skb data buffers larger than the hardware MTU.
+ * In other words, the Linux network stack MTU can be larger than the
+ * hardware MTU.
+ */
+int ipath_layer_send_skb(struct copy_data_s *cdata)
+{
+	int ret = 0;
+	uint16_t vlsllnh;
+	int device = cdata->device;
+
+	if (device >= infinipath_max) {
+		_IPATH_INFO("Invalid unit %u, failing\n", device);
+		return -EINVAL;
+	}
+	if (!(devdata[device].ipath_flags & IPATH_RCVHDRSZ_SET)) {
+		_IPATH_INFO("send while not open\n");
+		ret = -EINVAL;
+	} else
+	    if ((devdata[device].ipath_flags & (IPATH_LINKUNK | IPATH_LINKDOWN))
+		|| devdata[device].ipath_lid == 0) {
+		/* lid check is for when sma hasn't yet configured */
+		ret = -ENETDOWN;
+		_IPATH_VDBG("send while not ready, mylid=%u, flags=0x%x\n",
+			    devdata[device].ipath_lid,
+			    devdata[device].ipath_flags);
+	}
+	vlsllnh = *((uint16_t *) cdata->hdr);
+	if (vlsllnh != htons(IPS_LRH_BTH)) {
+		_IPATH_DBG("Warning: lrh[0] wrong (%x, not %x); not sending\n",
+			   vlsllnh, htons(IPS_LRH_BTH));
+		ret = -EINVAL;
+	}
+	if (ret)
+		goto done;
+
+	cdata->error = 0;	/* clear last calls error  */
+
+	if (cdata->skb->ip_summed == CHECKSUM_HW) {
+		unsigned int csstart = cdata->skb->h.raw - cdata->skb->data;
+
+		/*
+		 * Computing the checksum is a bit tricky since if we fragment
+		 * the packet, the fragment that should contain the checksum
+		 * will have already been sent.  The solution is to
+		 * store the checksum in the header of the last fragment
+		 * just before we write the last data word which triggers
+		 * the last fragment to be sent.  The receiver will
+		 * check the header "tag" field, see that there is a
+		 * checksum, and store the checksum back into the packet.
+		 *
+		 * Save the offset of the two byte checksum.
+		 * Note that we have to add 2 to account for the two
+		 * bytes of the ethernet address we stripped from the
+		 * packet and put in the header.
+		 */
+		cdata->hdr->csum_offset = csstart + cdata->skb->csum + 2;
+
+		if (cdata->offset < csstart)
+			copy_bits(cdata->skb, cdata->offset,
+				  csstart - cdata->offset, cdata);
+
+		if (cdata->error) {
+			return (cdata->error);
+
+		}
+
+		if (cdata->offset < cdata->skb->len)
+			copy_and_csum_bits(cdata->skb, cdata->offset,
+					   cdata->skb->len - cdata->offset,
+					   cdata);
+
+		if (cdata->error) {
+			return (cdata->error);
+		}
+
+		if (cdata->extra) {
+			while (cdata->extra < 4)
+				cdata->u.buf[cdata->extra++] = 0;
+			if (cdata->flen != 0
+			    || layer_send_getpiobuf(cdata) >= 0) {
+				/*
+				 * write the checksum before the last
+				 * PIO write.
+				 */
+				*cdata->csum_pio = csum_fold(cdata->csum);
+				mb();
+				*cdata->to = cdata->u.w;
+			}
+		}
+	} else {
+		copy_bits(cdata->skb, cdata->offset,
+			  cdata->skb->len - cdata->offset, cdata);
+
+		if (cdata->error) {
+			return (cdata->error);
+		}
+
+		if (cdata->extra) {
+			while (cdata->extra < 4)
+				cdata->u.buf[cdata->extra++] = 0;
+			if (cdata->flen != 0
+			    || layer_send_getpiobuf(cdata) >= 0)
+				*cdata->to = cdata->u.w;
+		}
+	}
+
+	if (cdata->error) {
+		ret = cdata->error;
+		if (cdata->error != -EBUSY)
+			/* just means no PIO buffers available */
+			_IPATH_UNIT_ERROR(device,
+					  "layer_send copy_bits failed with error %d\n",
+					  -ret);
+	}
+
+	ipath_stats.sps_ether_spkts++;	/* another ether packet sent */
+
+done:
+	return ret;
+}
+
+EXPORT_SYMBOL(ipath_layer_send_skb);
drivers/infiniband/hw/ipath/ipath_dwordcpy.S new file +62 -0

62 lines not on this page · raw message

Signature · 1 line
0.99.9n
#5 lore raw

RFC PATCH 4/13 ipath LLD core, part 1

First part of core driver

---

 drivers/infiniband/hw/ipath/ipath_driver.c | 2589 ++++++++++++++++++++++++++++
 1 files changed, 2589 insertions(+), 0 deletions(-)
 create mode 100644 drivers/infiniband/hw/ipath/ipath_driver.c

04a2c405bc3b7f074758c4329933e9499681ccf0

drivers/infiniband/hw/ipath/ipath_driver.c new file +2589 -0
new file mode 100644
index 0000000..df650d6
@@ -0,0 +1,2589 @@
+/*
+ * Copyright (c) 2003, 2004, 2005. PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Patent licenses, if any, provided herein do not apply to
+ * combinations of this program with other software, or any other
+ * product whatsoever.
+ *
+ * $Id: ipath_driver.c 4500 2005-12-16 01:34:22Z rjwalsh $
+ */
+
+#include <linux/version.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <linux/swap.h>
+#include <linux/sysctl.h>
+#include <asm/mtrr.h>
+#include <linux/netdevice.h>
+
+#include <linux/crc32.h>	/* we can generate our own crc's for testing */
+
+#include "ipath_kernel.h"
+#include "ips_common.h"
+#include "ipath_layer.h"
+
+/*
+ * Our LSB-assigned major number, so scripts can figure
+ * out how to make entry in /dev.
+ */
+
+static int ipath_major = 233;
+
+/*
+ * number of buffers reserved for driver (layered drivers and SMA send),
+ * settable via sysctl, although it may not take effect if user
+ * processes have the port open.  Reserved at end of buffer list
+ */
+
+static uint infinipath_kpiobufs = 32;
+
+/*
+ * number of ports we are configured to use (to allow for more pio
+ * buffers per port, etc.)  Zero means use chip value
+ */
+
+static uint infinipath_cfgports;
+
+/*
+ * number of units we are configured to use (to allow for bringup on
+ * multi-chip systems)  Zero means use only one for now, but eventually
+ * will mean to use infinipath_max
+ */
+
+static uint infinipath_cfgunits;
+
+uint64_t ipath_dummy_val_for_testing;
+
+static __kernel_pid_t ipath_sma_alive;	/* PID of SMA, if it's running */
+static spinlock_t ipath_sma_lock;	/* SMA receive */
+
+/* max SM received packets we'll queue; we keep the most recent packets. */
+
+#define IPATH_NUM_SMAPKTS  16
+
+#define IPATH_SMA_HDRSZ (8+12+8)	/* LRH+BTH+DETH */
+
+static struct _ipath_sma_rpkt {
+	/* length of received packet; non-zero if queued */
+	uint32_t len;
+	/* unit number of interface packet was received from */
+	uint32_t unit;
+	uint8_t *buf;
+} ipath_sma_data[IPATH_NUM_SMAPKTS];
+
+static unsigned ipath_sma_first; /* oldest sma packet index */
+static unsigned ipath_sma_next;	/* next sma packet index to use */
+
+/*
+ * ipath_sma_data_bufs has one extra, pointed to by ipath_sma_data_spare,
+ * so we can exchange buffers to do copy_to_user, and not hold the lock
+ * across the copy_to_user().
+ */
+
+#define SMA_MAX_PKTSZ (IPATH_SMA_HDRSZ+256)	/* max len of an SMA packet */
+
+static uint8_t ipath_sma_data_bufs[IPATH_NUM_SMAPKTS + 1][SMA_MAX_PKTSZ];
+static uint8_t *ipath_sma_data_spare;
+/* sma waits globally on all units */
+static wait_queue_head_t ipath_sma_wait;
+static wait_queue_head_t ipath_sma_state_wait;
+
+struct infinipath_stats ipath_stats;
+
+static __inline__ uint64_t ipath_kget_sreg(const ipath_type, ipath_sreg)
+    __attribute__ ((always_inline));
+
+/*
+ * this will only be used for diags, now that we have enabled the DMA
+ * of the sendpioavail regs to system memory.
+ */
+
+static __inline__ uint64_t ipath_kget_sreg(const ipath_type stype,
+					   ipath_sreg regno)
+{
+	uint64_t val;
+	uint64_t *sbase;
+
+	sbase = (uint64_t *) (devdata[stype].ipath_sregbase
+			      + (char *)devdata[stype].ipath_kregbase);
+	val = sbase ? sbase[regno] : 0ULL;
+	return val;
+}
+
+/*
+ * make infinipath_debug, etc. changeable on the fly via sysctl.
+ */
+
+static int ipath_sysctl(ctl_table *, int, struct file *, void __user *,
+			size_t *, loff_t *);
+
+static int ipath_do_user_init(ipath_portdata *, struct ipath_user_info *);
+static int ipath_get_baseinfo(ipath_portdata *, struct ipath_base_info *);
+static int ipath_get_units(void);
+static int ipath_wr_eeprom(ipath_portdata *, struct ipath_eeprom_req *);
+static int ipath_wait_intr(ipath_portdata *, uint32_t);
+static int ipath_tid_update(ipath_portdata *, struct _tidupd *);
+static int ipath_tid_free(ipath_portdata *, struct _tidupd *);
+static int ipath_get_counters(ipath_type, struct infinipath_counters *);
+static int ipath_get_unit_counters(struct infinipath_getunitcounters *a);
+static int ipath_get_stats(struct infinipath_stats *);
+static int ipath_set_partkey(ipath_portdata *, uint16_t);
+static int ipath_manage_rcvq(ipath_portdata *, uint16_t);
+static void ipath_clean_partkey(ipath_portdata *, ipath_devdata *);
+static void ipath_disarm_piobufs(const ipath_type, unsigned, unsigned);
+static int ipath_create_user_egr(ipath_portdata *);
+static int ipath_create_port0_egr(ipath_portdata *);
+static int ipath_create_rcvhdrq(ipath_portdata *);
+static void ipath_handle_errors(const ipath_type, uint64_t);
+static void ipath_update_pio_bufs(const ipath_type);
+static __inline__ void *ipath_get_egrbuf(const ipath_type, uint32_t, int);
+static int ipath_shutdown_link(const ipath_type);
+static int ipath_bringup_link(const ipath_type);
+int ipath_bringup_serdes(const ipath_type);
+static void ipath_get_faststats(unsigned long);
+static int ipath_setup_htconfig(struct pci_dev *, uint64_t *, const ipath_type);
+static struct page *ipath_nopage(struct vm_area_struct *, unsigned long, int *);
+static irqreturn_t ipath_intr(int irq, void *devid, struct pt_regs *regs);
+static void ipath_decode_err(char *, size_t, uint64_t);
+void ipath_free_pddata(ipath_devdata *, uint32_t, int);
+static void ipath_clear_tids(const ipath_type, unsigned);
+static void ipath_get_guid(const ipath_type);
+static int ipath_sma_ioctl(struct file *, unsigned int, unsigned long);
+static int ipath_rcvsma_pkt(struct ipath_sendpkt *);
+static int ipath_kset_lid(uint32_t);
+static int ipath_kset_mlid(uint32_t);
+static int ipath_get_mlid(uint32_t *);
+static int ipath_get_devstatus(uint64_t *);
+static int ipath_kset_guid(struct ipath_setguid *);
+static int ipath_get_portinfo(uint32_t *);
+static int ipath_get_nodeinfo(uint32_t *);
+#ifdef _IPATH_EXTRA_DEBUG
+static void ipath_dump_allregs(char *, ipath_type);
+#endif
+
+static const char ipath_sma_name[] = "infinipath_SMA";
+
+/*
+ * is diags mode enabled?  if it is, then things like auto bringup of
+ * links is disabled
+ */
+
+int ipath_diags_enabled = 0;
+
+void ipath_chip_done(void)
+{
+}
+
+void ipath_chip_cleanup(ipath_devdata * dd)
+{
+}
+
+/*
+ * cache aligned location
+ *
+ * where port 0 rcvhdrtail register is written back; also want
+ * nothing else sharing the cache line, so make it a cache line in size
+ * used for all units
+ *
+ * This is volatile as it's the target of a DMA from the chip.
+ */
+
+static volatile uint64_t ipath_port0_rcvhdrtail[512]
+    __attribute__ ((aligned(4096)));
+
+#define MODNAME "ipath_core"
+#define DRIVER_LOAD_MSG "PathScale " MODNAME " loaded: "
+#define PFX MODNAME ": "
+
+/*
+ * min buffers we want to have per port, after driver
+ */
+
+#define IPATH_MIN_USER_PORT_BUFCNT 8
+
+/* The size has to be longer than this string, so we can
+ * append board/chip information to it in the init code.
+ */
+static char ipath_core_version[192] = _IPATH_IDSTR "\n";
+static char *chip_driver_version;
+static int chip_driver_size;
+
+/* mylid and lidbase are to deal with LIDs in "fabric", until SM is working */
+
+module_param(infinipath_debug, uint, 0644);
+module_param(infinipath_kpiobufs, uint, 0644);
+module_param(infinipath_cfgports, uint, 0644);
+module_param(infinipath_cfgunits, uint, 0644);
+
+MODULE_PARM_DESC(infinipath_debug, "mask for debug prints");
+MODULE_PARM_DESC(infinipath_cfgports, "Set max number of ports to use");
+MODULE_PARM_DESC(infinipath_cfgunits, "Set max number of devices to use");
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("PathScale <[email protected]>");
+MODULE_DESCRIPTION("Pathscale InfiniPath driver");
+
+#ifdef IPATH_DIAG
+static __kernel_pid_t ipath_diag_alive;	/* PID of diags, if running */
+extern int ipath_diags_ioctl(struct file *, unsigned, unsigned long);
+static int ipath_opendiag(struct inode *, struct file *);
+#endif
+
+#if __IPATH_INFO || __IPATH_DBG
+static const char *ipath_ibcstatus_str[] = {
+	"Disabled",
+	"LinkUp",
+	"PollActive",
+	"PollQuiet",
+	"SleepDelay",
+	"SleepQuiet",
+	"LState6",		/* unused */
+	"LState7",		/* unused */
+	"CfgDebounce",
+	"CfgRcvfCfg",
+	"CfgWaitRmt",
+	"CfgIdle",
+	"RecovRetrain",
+	"LState0xD",		/* unused */
+	"RecovWaitRmt",
+	"RecovIdle",
+};
+#endif
+
+static ssize_t show_version(struct device_driver *dev, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%s", ipath_core_version);
+}
+
+static ssize_t show_status(struct device *dev,
+			   struct device_attribute *attr,
+			   char *buf)
+{
+	ipath_devdata *dd = dev_get_drvdata(dev);
+
+	if(!dd)
+		return -EINVAL;
+
+	if(!dd->ipath_statusp)
+		return -EINVAL;
+
+	return snprintf(buf, PAGE_SIZE, "%llx\n", *(dd->ipath_statusp));
+}
+
+static const char *ipath_status_str[] = {
+	"Initted",
+	"Disabled",
+	"4",                    /* unused */
+	"OIB_SMA",
+	"SMA",
+	"Present",
+	"IB_link_up",
+	"IB_configured",
+	"NoIBcable",
+	"Fatal_Hardware_Error",
+	NULL,
+};
+
+static ssize_t show_status_str(struct device *dev,
+			   struct device_attribute *attr,
+			   char *buf)
+{
+	ipath_devdata *dd = dev_get_drvdata(dev);
+	int i, any;
+	uint64_t s;
+
+	if(!dd)
+		return -EINVAL;
+
+	if(!dd->ipath_statusp)
+		return -EINVAL;
+
+	s = *(dd->ipath_statusp);
+	*buf = '\0';
+	for (any = i = 0; s && ipath_status_str[i]; i++) {
+		if (s & 1) {
+			if (any && strlcat(buf, " ", PAGE_SIZE) >= PAGE_SIZE)
+				/* overflow */
+				break;
+			if (strlcat(buf, ipath_status_str[i],
+				    PAGE_SIZE) >= PAGE_SIZE)
+				break;
+			any = 1;
+		}
+		s >>= 1;
+	}
+	if(any)
+		strlcat(buf, "\n", PAGE_SIZE);
+
+	return strlen(buf);
+}
+
+static ssize_t show_lid(struct device *dev,
+			   struct device_attribute *attr,
+			   char *buf)
+{
+	ipath_devdata *dd = dev_get_drvdata(dev);
+
+	if(!dd)
+		return -EINVAL;
+
+	return snprintf(buf, PAGE_SIZE, "%x\n", dd->ipath_lid);
+}
+
+static ssize_t show_mlid(struct device *dev,
+			   struct device_attribute *attr,
+			   char *buf)
+{
+	ipath_devdata *dd = dev_get_drvdata(dev);
+
+	if(!dd)
+		return -EINVAL;
+
+	return snprintf(buf, PAGE_SIZE, "%x\n", dd->ipath_mlid);
+}
+
+static ssize_t show_guid(struct device *dev,
+			   struct device_attribute *attr,
+			   char *buf)
+{
+	ipath_devdata *dd = dev_get_drvdata(dev);
+	uint8_t *guid;
+
+	if(!dd)
+		return -EINVAL;
+
+	guid = (uint8_t *)&(dd->ipath_guid);
+
+	return snprintf(buf, PAGE_SIZE, "%x:%x:%x:%x:%x:%x:%x:%x\n",
+			guid[0], guid[1], guid[2], guid[3], guid[4], guid[5],
+			guid[6], guid[7]);
+}
+
+static ssize_t show_nguid(struct device *dev,
+			   struct device_attribute *attr,
+			   char *buf)
+{
+	ipath_devdata *dd = dev_get_drvdata(dev);
+
+	if(!dd)
+		return -EINVAL;
+
+	return snprintf(buf, PAGE_SIZE, "%u\n", dd->ipath_nguid);
+}
+
+static ssize_t show_serial(struct device *dev,
+			   struct device_attribute *attr,
+			   char *buf)
+{
+	ipath_devdata *dd = dev_get_drvdata(dev);
+
+	if(!dd)
+		return -EINVAL;
+
+	buf[sizeof dd->ipath_serial] = '\0';
+	memcpy(buf, dd->ipath_serial, sizeof dd->ipath_serial);
+	strcat(buf, "\n");
+	return strlen(buf);
+}
+
+static ssize_t show_unit(struct device *dev,
+			   struct device_attribute *attr,
+			   char *buf)
+{
+	ipath_devdata *dd = dev_get_drvdata(dev);
+
+	if(!dd)
+		return -EINVAL;
+
+	snprintf(buf, PAGE_SIZE, "%u\n", dd->ipath_unit);
+	return strlen(buf);
+}
+
+static DRIVER_ATTR(version, S_IRUGO, show_version, NULL);
+static DEVICE_ATTR(status, S_IRUGO, show_status, NULL);
+static DEVICE_ATTR(status_str, S_IRUGO, show_status_str, NULL);
+static DEVICE_ATTR(lid, S_IRUGO, show_lid, NULL);
+static DEVICE_ATTR(mlid, S_IRUGO, show_mlid, NULL);
+static DEVICE_ATTR(guid, S_IRUGO, show_guid, NULL);
+static DEVICE_ATTR(nguid, S_IRUGO, show_nguid, NULL);
+static DEVICE_ATTR(serial, S_IRUGO, show_serial, NULL);
+static DEVICE_ATTR(unit, S_IRUGO, show_unit, NULL);
+
+/*
+ * Much like proc_dointvec_minmax, but only one int, and show as 0x on read
+ * Apparently between 2.6.3 and 2.6.10, convenience functions were added
+ * that I should probably convert to using.  For now, do the minimum possible
+ * change, using the new ppos parameter, instead of f_pos
+ */
+
+static int ipath_sysctl(ctl_table * ct, int wr, struct file *f, void __user * b,
+			size_t * l, loff_t * ppos)
+{
+	char t[20];
+	int len, ret = 0;
+
+	if (*ppos && !wr)
+		*l = 0;
+	if (!*l)
+		goto done;
+	if (!access_ok(wr ? VERIFY_READ : VERIFY_WRITE, b, *l)) {
+		ret = -EFAULT;
+		goto done;
+	}
+	len = min_t(int, sizeof t, *l);
+
+	if (!wr) {
+		/* All of our changeable sysctl stuff is unsigned's for now */
+		*l = snprintf(t, len, "%x\n", *(unsigned *)ct->data);
+		if (*l < 0)
+			*l = 0;
+		else
+			copy_to_user(b, t, *l);
+	} else {
+		int i;
+		char *e;
+		if (copy_from_user(t, b, len)) {
+			ret = -EFAULT;
+			goto done;
+		}
+		t[len < (sizeof t - 1) ? len : (sizeof t - 1)] = '\0';
+		i = simple_strtoul(t, &e, 0);
+		if (e > t) {
+			/*
+			 * All of our changeable sysctl stuff is
+			 * unsigned's for now
+			 */
+			if (ct->ctl_name == CTL_INFINIPATH_LAYERBUF) {
+				/* we don't need locking for this,
+				 * because we still do the normal avail
+				 * checks, it's just a question of what
+				 * range we check within; at best
+				 * during the update, we miss checking
+				 * some buffers we could have used,
+				 * for a short period
+				 */
+
+				int d;
+
+				if (i < 1) {
+					_IPATH_ERROR
+					    ("Must have at least one kernel PIO buffer\n");
+					ret = -EINVAL;
+					goto done;
+				}
+				for (d = 0; d < infinipath_max; d++) {
+					if (devdata[d].ipath_kregbase) {
+						if (i >
+						    (devdata[d].ipath_piobcnt -
+						     (devdata[d].
+						      ipath_cfgports *
+						      IPATH_MIN_USER_PORT_BUFCNT)))
+						{
+							_IPATH_UNIT_ERROR(d,
+									  "Allocating %d PIO bufs for kernel leaves too few for %d user ports (%d each)\n",
+									  i,
+									  devdata
+									  [d].
+									  ipath_cfgports
+									  - 1,
+									  IPATH_MIN_USER_PORT_BUFCNT);
+							ret = -EINVAL;
+							goto done;
+						}
+						devdata[d].ipath_lastport_piobuf =
+						    devdata[d].ipath_piobcnt - i;
+						devdata[d].ipath_lastpioindex =
+							devdata[d].ipath_lastport_piobuf;
+					}
+				}
+			}
+			*(unsigned *)ct->data = i;
+		} else {
+			ret = -EINVAL;
+			goto done;
+		}
+		*l = len;
+	}
+done:
+
+	*ppos += *l;
+	return ret;
+}
+
+/*
+ * make infinipath_debug changeable on the fly via sysctl.
+ */
+
+static struct ctl_table_header *ipath_ctl_header;
+
+static ctl_table ipath_ctl_debug[] = {
+	{
+	 .ctl_name = CTL_INFINIPATH_DEBUG,
+	 .procname = "debug",
+	 .data = &infinipath_debug,
+	 .maxlen = sizeof(infinipath_debug),
+	 .mode = 0644,
+	 .proc_handler = ipath_sysctl,
+	 }
+	,
+	{
+	 .ctl_name = CTL_INFINIPATH_LAYERBUF,
+	 .procname = "kern_piobufs",
+	 .data = &infinipath_kpiobufs,
+	 .maxlen = sizeof(infinipath_kpiobufs),
+	 .mode = 0644,
+	 .proc_handler = ipath_sysctl,
+	 }
+	,
+	{.ctl_name = 0}
+};
+
+static ctl_table ipath_ctl[] = {
+	{
+	 .ctl_name = CTL_INFINIPATH,
+	 .procname = "infinipath",
+	 .mode = 0555,
+	 .child = ipath_ctl_debug,
+	 },
+	{.ctl_name = 0}
+};
+
+/*
+ * called from add_timer and user counter read calls, to deal with
+ * counters that wrap in "human time".  The words sent and received, and
+ * the packets sent and received are all that we worry about.  For now,
+ * at least, we don't worry about error counters, because if they wrap
+ * that quickly, we probably don't care.  We may eventually just make this
+ * handle all the counters.  word counters can wrap in about 20 seconds
+ * of full bandwidth traffic, packet counters in a few hours.
+ */
+
+uint64_t ipath_snap_cntr(const ipath_type t, ipath_creg creg)
+{
+	uint32_t val;
+	uint64_t val64, t0, t1;
+	ipath_devdata *dd = &devdata[t];
+	static uint64_t one_sec_in_cycles;
+	extern uint32_t _ipath_pico_per_cycle;
+
+	if (!one_sec_in_cycles && _ipath_pico_per_cycle)
+		one_sec_in_cycles = 1000000000000UL / _ipath_pico_per_cycle;
+
+	t0 = get_cycles();
+	val = ipath_kget_creg32(t, creg);
+	t1 = get_cycles();
+	if ((t1 - t0) > one_sec_in_cycles && val == ~0) {
+		/*
+		 * This is just a way to detect things that are quite broken.
+		 * Normally this should take just a few cycles (the check is
+		 * for long enough that we don't care if we get pre-empted.)
+		 * An Opteron HT O read timeout is 4 seconds with normal
+		 * NB values
+		 */
+
+		_IPATH_UNIT_ERROR(t, "Error!  Reading counter 0x%x timed out\n",
+				  creg);
+		return 0ULL;
+	}
+
+	if (creg == cr_wordsendcnt) {
+		if (val != dd->ipath_lastsword) {
+			dd->ipath_sword += val - dd->ipath_lastsword;
+			dd->ipath_lastsword = val;
+		}
+		val64 = dd->ipath_sword;
+	} else if (creg == cr_wordrcvcnt) {
+		if (val != dd->ipath_lastrword) {
+			dd->ipath_rword += val - dd->ipath_lastrword;
+			dd->ipath_lastrword = val;
+		}
+		val64 = dd->ipath_rword;
+	} else if (creg == cr_pktsendcnt) {
+		if (val != dd->ipath_lastspkts) {
+			dd->ipath_spkts += val - dd->ipath_lastspkts;
+			dd->ipath_lastspkts = val;
+		}
+		val64 = dd->ipath_spkts;
+	} else if (creg == cr_pktrcvcnt) {
+		if (val != dd->ipath_lastrpkts) {
+			dd->ipath_rpkts += val - dd->ipath_lastrpkts;
+			dd->ipath_lastrpkts = val;
+		}
+		val64 = dd->ipath_rpkts;
+	} else
+		val64 = (uint64_t) val;
+
+	return val64;
+}
+
+/*
+ * print the delta of egrfull/hdrqfull errors for kernel ports no more
+ * than every 5 seconds.  User processes are printed at close, but kernel
+ * doesn't close, so...  Separate routine so may call from other places
+ * someday, and so function name when printed by _IPATH_INFO is meaningfull
+ */
+
+static void ipath_qcheck(const ipath_type t)
+{
+	static uint64_t last_tot_hdrqfull;
+	size_t blen = 0;
+	ipath_devdata *dd = &devdata[t];
+	char buf[128];
+
+	*buf = 0;
+	if (dd->ipath_pd[0]->port_hdrqfull != dd->ipath_p0_hdrqfull) {
+		blen = snprintf(buf, sizeof buf, "port 0 hdrqfull %u",
+				dd->ipath_pd[0]->port_hdrqfull -
+				dd->ipath_p0_hdrqfull);
+		dd->ipath_p0_hdrqfull = dd->ipath_pd[0]->port_hdrqfull;
+	}
+	if (ipath_stats.sps_etidfull != dd->ipath_last_tidfull) {
+		blen +=
+		    snprintf(buf + blen, sizeof buf - blen, "%srcvegrfull %llu",
+			     blen ? ", " : "",
+			     ipath_stats.sps_etidfull - dd->ipath_last_tidfull);
+		dd->ipath_last_tidfull = ipath_stats.sps_etidfull;
+	}
+
+	/*
+	 * this is actually the number of hdrq full interrupts, not actual
+	 * events, but at the moment that's mostly what I'm interested in.
+	 * Actual count, etc. is in the counters, if needed.  For production
+	 * users this won't ordinarily be printed.
+	 */
+
+	if ((infinipath_debug & (__IPATH_PKTDBG | __IPATH_DBG)) &&
+	    ipath_stats.sps_hdrqfull != last_tot_hdrqfull) {
+		blen +=
+		    snprintf(buf + blen, sizeof buf - blen,
+			     "%shdrqfull %llu (all ports)", blen ? ", " : "",
+			     ipath_stats.sps_hdrqfull - last_tot_hdrqfull);
+		last_tot_hdrqfull = ipath_stats.sps_hdrqfull;
+	}
+	if (blen)
+		_IPATH_DBG("%s\n", buf);
+
+	if(*dd->ipath_hdrqtailptr != dd->ipath_port0head) {
+		if(dd->ipath_lastport0rcv_cnt == ipath_stats.sps_port0pkts) {
+			_IPATH_PDBG("missing rcv interrupts? port0 hd=%llx tl=%x; port0pkts %llx\n",
+				*dd->ipath_hdrqtailptr, dd->ipath_port0head,ipath_stats.sps_port0pkts);
+			ipath_kreceive(t);
+		}
+		dd->ipath_lastport0rcv_cnt = ipath_stats.sps_port0pkts;
+	}
+}
+
+/*
+ * called from add_timer to get word counters from chip before they
+ * can overflow
+ */
+
+static void ipath_get_faststats(unsigned long t)
+{
+	uint32_t val;
+	ipath_devdata *dd = &devdata[t];
+	static unsigned cnt;
+
+	/*
+	 * don't access the chip while running diags, or memory diags
+	 * can fail
+	 */
+	if (!dd->ipath_kregbase || !(dd->ipath_flags & IPATH_PRESENT) ||
+	    ipath_diags_enabled) {
+		/* but re-arm the timer, for diags case; won't hurt other */
+		goto done;
+	}
+
+	ipath_snap_cntr((ipath_type) t, cr_wordsendcnt);
+	ipath_snap_cntr((ipath_type) t, cr_wordrcvcnt);
+	ipath_snap_cntr((ipath_type) t, cr_pktsendcnt);
+	ipath_snap_cntr((ipath_type) t, cr_pktrcvcnt);
+
+	ipath_qcheck(t);
+
+	/*
+	 * deal with repeat error suppression.  Doesn't really matter if
+	 * last error was almost a full interval ago, or just a few usecs
+	 * ago; still won't get more than 2 per interval.  We may want
+	 * longer intervals for this eventually, could do with mod, counter
+	 * or separate timer.  Also see code in ipath_handle_errors() and
+	 * ipath_handle_hwerrors().
+	 */
+
+	if (dd->ipath_lasterror)
+		dd->ipath_lasterror = 0;
+	if (dd->ipath_lasthwerror)
+		dd->ipath_lasthwerror = 0;
+	if ((devdata[t].ipath_maskederrs & ~devdata[t].ipath_ignorederrs)
+	    && get_cycles() > devdata[t].ipath_unmasktime) {
+		char ebuf[256];
+		ipath_decode_err(ebuf, sizeof ebuf,
+				 (devdata[t].ipath_maskederrs & ~devdata[t].
+				  ipath_ignorederrs));
+		if ((devdata[t].ipath_maskederrs & ~devdata[t].
+		     ipath_ignorederrs)
+		    & ~(INFINIPATH_E_RRCVEGRFULL | INFINIPATH_E_RRCVHDRFULL)) {
+			_IPATH_UNIT_ERROR(t, "Re-enabling masked errors (%s)\n",
+					  ebuf);
+		} else {
+			/*
+			 * rcvegrfull and rcvhdrqfull are "normal",
+			 * for some types of processes (mostly benchmarks)
+			 * that send huge numbers of messages, while
+			 * not processing them.  So only complain about
+			 * these at debug level.
+			 */
+			_IPATH_DBG
+			    ("Disabling frequent queue full errors (%s)\n",
+			     ebuf);
+		}
+		devdata[t].ipath_maskederrs = devdata[t].ipath_ignorederrs;
+		ipath_kput_kreg(t, kr_errormask, ~devdata[t].ipath_maskederrs);
+	}
+
+	if (dd->ipath_flags & IPATH_LINK_SLEEPING) {
+		uint64_t ibc;
+		_IPATH_VDBG("linkinitcmd SLEEP, move to POLL\n");
+		dd->ipath_flags &= ~IPATH_LINK_SLEEPING;
+		ibc = dd->ipath_ibcctrl;
+		/*
+		 * don't put linkinitcmd in ipath_ibcctrl, want that to
+		 * stay a NOP
+		 */
+		ibc |=
+		    INFINIPATH_IBCC_LINKINITCMD_POLL <<
+		    INFINIPATH_IBCC_LINKINITCMD_SHIFT;
+		ipath_kput_kreg(t, kr_ibcctrl, ibc);
+	}
+
+	/* limit qfull messages to ~one per minute per port */
+	if ((++cnt & 0x10)) {
+		for (val = devdata[t].ipath_cfgports - 1; ((int)val) >= 0;
+		     val--) {
+			if (dd->ipath_lastegrheads[val] != ~0)
+				dd->ipath_lastegrheads[val] = ~0;
+			if (dd->ipath_lastrcvhdrqtails[val] != ~0)
+				dd->ipath_lastrcvhdrqtails[val] = ~0;
+		}
+	}
+
+	if(dd->ipath_nosma_bufs) {
+		dd->ipath_nosma_secs += 5;
+		if(dd->ipath_nosma_secs >= 30) {
+			_IPATH_SMADBG("No SMA bufs avail %u seconds; cancelling pending sends\n",
+				dd->ipath_nosma_secs);
+			ipath_disarm_piobufs(t, dd->ipath_lastport_piobuf,
+				dd->ipath_piobcnt - dd->ipath_lastport_piobuf);
+			dd->ipath_nosma_secs = 0; /* start again, if necessary */
+		}
+		else 
+			_IPATH_SMADBG("No SMA bufs avail %u tries, after %u seconds\n",
+				dd->ipath_nosma_bufs, dd->ipath_nosma_secs);
+	}
+
+done:
+	mod_timer(&dd->ipath_stats_timer, jiffies + HZ * 5);
+}
+
+
+static void __devexit infinipath_remove_one(struct pci_dev *);
+static int infinipath_init_one(struct pci_dev *, const struct pci_device_id *);
+
+const struct pci_device_id infinipath_pci_tbl[] = {
+	{
+	 PCI_VENDOR_ID_PATHSCALE, PCI_DEVICE_ID_PATHSCALE_INFINIPATH2,
+	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
+	{0,}
+};
+
+MODULE_DEVICE_TABLE(pci, infinipath_pci_tbl);
+
+static struct pci_driver infinipath_driver = {
+	.name = MODNAME,
+	.driver.owner = THIS_MODULE,
+	.probe = infinipath_init_one,
+	.remove = __devexit_p(infinipath_remove_one),
+	.id_table = infinipath_pci_tbl,
+};
+
+#if defined (pgprot_writecombine) && defined(_PAGE_MA_WC)
+int remap_area_pages(unsigned long address, unsigned long phys_addr,
+		     unsigned long size, unsigned long flags);
+#endif
+
+static int infinipath_init_one(struct pci_dev *pdev,
+			       const struct pci_device_id *ent)
+{
+	int ret, len, j;
+	static int chip_idx = -1;
+	unsigned long addr;
+	uint64_t pioaddr, piolen, intconfig;
+	uint8_t rev;
+	ipath_type dev;
+
+	/*
+	 * XXX: Right now, we have a hardcoded array of devices.  We'll
+	 * change this in a future release, but not just yet.  For the
+	 * moment, we're limited to 4 infinipath devices per system.
+	 */
+
+	dev = ++chip_idx;
+
+	_IPATH_VDBG("initializing unit #%u\n", dev);
+	if ((!infinipath_cfgunits && (dev >= 1)) ||
+	    (infinipath_cfgunits && (dev >= infinipath_cfgunits)) ||
+	    (dev >= infinipath_max)) {
+		_IPATH_ERROR("Trying to initialize unit %u, max is %u\n",
+			     dev, infinipath_max - 1);
+		return -EINVAL;
+	}
+
+	devdata[dev].pci_registered = 1;
+	devdata[dev].ipath_unit = dev;
+
+	if ((ret = pci_enable_device(pdev))) {
+		_IPATH_DBG("pci_enable unit %u failed: %x\n", dev, ret);
+	}
+
+	if ((ret = pci_request_regions(pdev, MODNAME)))
+		_IPATH_INFO("pci_request_regions unit %u fails: %d\n", dev,
+			    ret);
+
+	if ((ret = pci_set_dma_mask(pdev, DMA_64BIT_MASK)) != 0)
+		_IPATH_INFO("pci_set_dma_mask unit %u fails: %d\n", dev, ret);
+
+	pci_set_master(pdev);	/* probably not be needed for HT */
+
+	addr = pci_resource_start(pdev, 0);
+	len = pci_resource_len(pdev, 0);
+	_IPATH_VDBG
+	    ("regbase (0) %lx len %d irq %x, vend %x/%x driver_data %lx\n",
+	     addr, len, pdev->irq, ent->vendor, ent->device, ent->driver_data);
+	devdata[dev].ipath_deviceid = ent->device;	/* save for later use */
+	devdata[dev].ipath_vendorid = ent->vendor;
+	for (j = 0; j < 6; j++) {
+		if (!pdev->resource[j].start)
+			continue;
+		_IPATH_VDBG("BAR %d start %lx, end %lx, len %lx\n",
+			    j, pdev->resource[j].start,
+			    pdev->resource[j].end, pci_resource_len(pdev, j));
+	}
+
+	if (!addr) {
+		_IPATH_UNIT_ERROR(dev, "No valid address in BAR 0!\n");
+		return -ENODEV;
+	}
+
+	if ((ret = pci_read_config_byte(pdev, PCI_REVISION_ID, &rev))) {
+		_IPATH_UNIT_ERROR(dev,
+				  "Failed to read PCI revision ID unit %u: %d\n",
+				  dev, ret);
+		return ret;	/* shouldn't ever happen */
+	} else
+		devdata[dev].ipath_pcirev = rev;
+
+	devdata[dev].ipath_kregbase = ioremap_nocache(addr, len);
+#if defined (pgprot_writecombine) && defined(_PAGE_MA_WC)
+	printk("Remapping pages WC\n");
+	remap_area_pages((u64) devdata[dev].ipath_kregbase + 1024 * 1024,
+			 addr + 1024 * 1024, 1024 * 1024, _PAGE_MA_WC);
+	/* devdata[dev].ipath_kregbase = __ioremap(addr, len, _PAGE_MA_WC); */
+#endif
+
+	if (!devdata[dev].ipath_kregbase) {
+		_IPATH_DBG("Unable to map io addr %lx to kvirt, failing\n",
+			   addr);
+		ret = -ENOMEM;
+		goto fail;
+	}
+	devdata[dev].ipath_kregend =
+	    (uint64_t *) ((void *)devdata[dev].ipath_kregbase + len);
+	devdata[dev].ipath_physaddr = addr;	/* used for io_remap, etc. */
+	/* for user mmap */
+	devdata[dev].ipath_kregvirt = phys_to_virt(addr);
+	_IPATH_VDBG("mapped io addr %lx to kregbase %p kregvirt %p\n", addr,
+		    devdata[dev].ipath_kregbase, devdata[dev].ipath_kregvirt);
+
+	/*
+	 * set these up before registering the interrupt handler, just
+	 * in case
+	 */
+	devdata[dev].pcidev = pdev;
+	pci_set_drvdata(pdev, &(devdata[dev]));
+
+	/*
+	 * set up our interrupt handler; SA_SHIRQ probably not needed,
+	 * but won't  hurt for now.
+	 */
+
+	if (!pdev->irq) {
+		_IPATH_UNIT_ERROR(dev, "irq is 0, failing init\n");
+		ret = -EINVAL;
+		goto fail;
+	}
+	if ((ret = request_irq(pdev->irq, ipath_intr,
+			       SA_SHIRQ, MODNAME, &devdata[dev]))) {
+		_IPATH_UNIT_ERROR(dev,
+				  "Couldn't setup interrupt handler, irq=%u: %d\n",
+				  pdev->irq, ret);
+		goto fail;
+	}
+
+	/*
+	 * clear ipath_flags here instead of in ipath_init_chip as it is set
+	 * by ipath_setup_htconfig.
+	 */
+	devdata[dev].ipath_flags = 0;
+	if (ipath_setup_htconfig(pdev, &intconfig, dev))
+		_IPATH_DBG
+		    ("Failed to setup HT config, continuing anyway for now\n");
+
+	ret = ipath_init_chip(dev);	/* do the chip-specific init */
+	if (!ret) {
+#ifdef CONFIG_MTRR
+		unsigned bits;
+		/*
+		 * Set the PIO buffers to be WCCOMB, so we get HT bursts
+		 * to the chip.  Linux (possibly the hardware) requires
+		 * it to be on a power of 2 address matching the length
+		 * (which has to be a power of 2).  For rev1, that means
+		 * the base address, for rev2, it will be just the PIO
+		 * buffers themselves.
+		 */
+		pioaddr = addr + devdata[dev].ipath_piobufbase;
+		piolen = devdata[dev].ipath_piobcnt *
+		    round_up(devdata[dev].ipath_piosize,
+			     devdata[dev].ipath_palign);
+
+		for (bits = 0; !(piolen & (1ULL << bits)); bits++) ;
+		if (piolen != (1ULL << bits)) {
+			_IPATH_DBG("piolen 0x%llx not power of 2, bits=%u\n",
+				   piolen, bits);
+			piolen >>= bits;
+			while (piolen >>= 1)
+				bits++;
+			piolen = 1ULL << (bits + 1);
+			_IPATH_DBG("Changed piolen to 0x%llx bits=%u\n", piolen,
+				   bits);
+		}
+		if (pioaddr & (piolen - 1)) {
+			uint64_t atmp;
+			_IPATH_DBG
+			    ("pioaddr %llx not on right boundary for size %llx, fixing\n",
+			     pioaddr, piolen);
+			atmp = pioaddr & ~(piolen - 1);
+			if (atmp < addr || (atmp + piolen) > (addr + len)) {
+				_IPATH_UNIT_ERROR(dev,
+						  "No way to align address/size (%llx/%llx), no WC mtrr\n",
+						  atmp, piolen << 1);
+				ret = -ENODEV;
+			} else {
+				_IPATH_DBG
+				    ("changing WC base from %llx to %llx, len from %llx to %llx\n",
+				     pioaddr, atmp, piolen, piolen << 1);
+				pioaddr = atmp;
+				piolen <<= 1;
+			}
+		}
+
+		if (!ret) {
+			int cookie;
+			_IPATH_VDBG
+			    ("Setting mtrr for chip to WC (addr %llx, len=0x%llx)\n",
+			     pioaddr, piolen);
+			cookie = mtrr_add(pioaddr, piolen, MTRR_TYPE_WRCOMB, 0);
+			if (cookie < 0) {
+				_IPATH_INFO
+				    ("mtrr_add(%llx,0x%llx,WC,0) failed (%d)\n",
+				     pioaddr, piolen, cookie);
+				ret = -EINVAL;
+			} else {
+				_IPATH_VDBG
+				    ("Set mtrr for chip to WC, cookie is %d\n",
+				     cookie);
+				devdata[dev].ipath_mtrr = (uint32_t) cookie;
+			}
+		}
+#endif				/* CONFIG_MTRR */
+	}
+
+	if (!ret && devdata[dev].ipath_kregbase && (devdata[dev].ipath_flags
+					    & IPATH_PRESENT)) {
+		/*
+		 * for the hardware, enable interrupts only after
+		 * kr_interruptconfig is written, if we could set it up
+		 */
+		if (intconfig) {
+			/* interrupt address */
+			ipath_kput_kreg(dev, kr_interruptconfig, intconfig);
+			/* enable all interrupts */
+			ipath_kput_kreg(dev, kr_intmask, ~0ULL);
+			/* force re-interrupt of any pending interrupts. */
+			ipath_kput_kreg(dev, kr_intclear, 0ULL);
+			/* OK, the chip is usable, marked it as initialized */
+			*devdata[dev].ipath_statusp |= IPATH_STATUS_INITTED;
+		} else
+			_IPATH_UNIT_ERROR(dev,
+					  "No interrupts enabled, couldn't setup interrupt address\n");
+	} else if(ret != -EPERM)
+		_IPATH_INFO("Not configuring unit %u interrupts, init failed\n",
+			    dev);
+
+	device_create_file(&(pdev->dev), &dev_attr_status);
+	device_create_file(&(pdev->dev), &dev_attr_status_str);
+	device_create_file(&(pdev->dev), &dev_attr_lid);
+	device_create_file(&(pdev->dev), &dev_attr_mlid);
+	device_create_file(&(pdev->dev), &dev_attr_guid);
+	device_create_file(&(pdev->dev), &dev_attr_nguid);
+	device_create_file(&(pdev->dev), &dev_attr_serial);
+	device_create_file(&(pdev->dev), &dev_attr_unit);
+
+	/*
+	 * We used to cleanup here, with pci_release_regions, etc. but that
+	 * can cause other problems if we want to run diags, etc., so instead
+	 * defer that until driver unload.
+	 */
+
+fail:	/* after we've done at least some of the pci setup */
+	if(ret == -EPERM) /* disabled device, don't want module load error;
+		* just want to carry status through to this point */
+		ret = 0;
+
+	return ret;
+}
+
+
+
+#define HT_CAPABILITY_ID   0x08 /* HT capabilities not defined in kernel */
+#define HT_INTR_DISC_CONFIG  0x80 /* HT interrupt and discovery cap */
+#define HT_INTR_REG_INDEX    2 /* intconfig requires indirect accesses */
+
+/*
+ * setup the interruptconfig register from the HT config info.
+ * Also clear CRC errors in HT linkcontrol, if necessary.
+ * This is done only for the real hardware.  It is done before
+ * chip address space is initted, so can't touch infinipath registers
+ */
+
+static int ipath_setup_htconfig(struct pci_dev *pdev, uint64_t * iaddr,
+				const ipath_type t)
+{
+	uint8_t cap_type;
+	uint32_t int_handler_addr_lower;
+	uint32_t int_handler_addr_upper;
+	uint64_t ihandler = 0;
+	int i, pos, ret = 0;
+
+	*iaddr = 0ULL;		/* init to zero in case not able to configure */
+
+	/*
+	 * Read the capability info to find the interrupt info, and also
+	 * handle clearing CRC errors in linkctrl register if necessary.
+	 * We do this early, before we ever enable errors or hardware errors,
+	 * mostly to avoid causing the chip to enter freeze mode.
+	 */
+	if(!(pos = pci_find_capability(pdev, HT_CAPABILITY_ID))) {
+		_IPATH_UNIT_ERROR(t,
+		    "Couldn't find HyperTransport capability; no interrupts\n");
+		return -ENODEV;
+	}
+	do {
+	    	/* the HT capability type byte is 3 bytes after the
+		 * capability byte.
+		 */
+		if(pci_read_config_byte(pdev, pos+3, &cap_type)) {
+			_IPATH_INFO
+			    ("Couldn't read config command @ %d\n", pos);
+			continue;
+		}
+		if(!(cap_type & 0xE0)) {
+			/* bits 13-15 of command==0 is slave/primary block.
+			 * Clear any HT CRC errors.  We only bother to
+			 * do this at load time, because it's OK if it
+			 * happened before we were loaded (first time
+			 * after boot/reset), but any time after that,
+			 * it's fatal anyway.  Also need to not check for
+			 * for upper byte errors if we are in 8 bit mode,
+			 * so figure out our width.  For now, at least,
+			 * also complain if it's 8 bit.
+			 */
+			uint8_t linkwidth = 0, linkerr, link_a_b_off, link_off;
+			uint16_t linkctrl = 0;
+
+			devdata[t].ipath_ht_slave_off = pos;
+			/* command word, master_host bit */
+			if((cap_type >> 2) & 1) /* master host || slave */
+				link_a_b_off = 4;
+			else
+				link_a_b_off = 0;
+			_IPATH_VDBG("HT%u (Link %c) connected to processor\n",
+				    link_a_b_off ? 1 : 0,
+				    link_a_b_off ? 'B' : 'A');
+
+			link_a_b_off += pos;
+
+			/*
+			 * check both link control registers; clear both
+			 * HT CRC sets if necessary.
+			 */
+
+			for (i = 0; i < 2; i++) {
+				link_off = pos + i * 4 + 0x4;
+				if (pci_read_config_word
+				    (pdev, link_off, &linkctrl))
+					_IPATH_UNIT_ERROR(t,
+					  "Couldn't read HT link control%d register\n",
+					  i);
+				else if (linkctrl & (0xf << 8)) {
+					_IPATH_VDBG
+					    ("Clear linkctrl%d CRC Error bits %x\n",
+					     i, linkctrl & (0xf << 8));
+					/*
+					 * now write them back to clear
+					 * the error.
+					 */
+					pci_write_config_byte(pdev, link_off,
+							      linkctrl & (0xf <<
+									  8));
+				}
+			}
+
+			/*
+			 * As with HT CRC bits, same for protocol errors
+			 * that might occur during boot.
+			 */
+
+			for (i = 0; i < 2; i++) {
+				link_off = pos + i * 4 + 0xd;
+				if (pci_read_config_byte
+				    (pdev, link_off, &linkerr))
+					_IPATH_INFO
+					    ("Couldn't read linkerror%d of HT slave/primary block\n",
+					     i);
+				else if (linkerr & 0xf0) {
+					_IPATH_VDBG
+					    ("HT linkerr%d bits 0x%x set, clearing\n",
+					     linkerr >> 4, i);
+					/*
+					 * writing the linkerr bits that
+					 * are set will clear them
+					 */
+					if (pci_write_config_byte
+					    (pdev, link_off, linkerr))
+						_IPATH_DBG
+						    ("Failed write to clear HT linkerror%d\n",
+						     i);
+					if (pci_read_config_byte
+					    (pdev, link_off, &linkerr))
+						_IPATH_INFO
+						    ("Couldn't reread linkerror%d of HT slave/primary block\n",
+						     i);
+					else if (linkerr & 0xf0)
+						_IPATH_INFO
+						    ("HT linkerror%d bits 0x%x couldn't be cleared\n",
+						     i, linkerr >> 4);
+				}
+			}
+
+			/*
+			 * this is just for our link to the host, not
+			 * devices connected through tunnel.
+			 */
+
+			if (pci_read_config_byte
+			    (pdev, link_a_b_off + 7, &linkwidth))
+				_IPATH_UNIT_ERROR(t,
+						  "Couldn't read HT link width config register\n");
+			else {
+				uint32_t width;
+				switch (linkwidth & 7) {
+				case 5:
+					width = 4;
+					break;
+				case 4:
+					width = 2;
+					break;
+				case 3:
+					width = 32;
+					break;
+				case 1:
+					width = 16;
+					break;
+				case 0:
+				default:	/* if wrong, assume 8 bit */
+					width = 8;
+					break;
+				}
+				((ipath_devdata *) pci_get_drvdata(pdev))->
+				    ipath_htwidth = width;
+
+				if (linkwidth != 0x11) {
+					_IPATH_UNIT_ERROR(t,
+							  "Not configured for 16 bit HT (%x)\n",
+							  linkwidth);
+					if (!(linkwidth & 0xf)) {
+						_IPATH_DBG
+						    ("Will ignore HT lane1 errors\n");
+						((ipath_devdata *)
+						 pci_get_drvdata(pdev))->
+				  ipath_flags |= IPATH_8BIT_IN_HT0;
+					}
+				}
+			}
+
+			/*
+			 * this is just for our link to the host, not
+			 * devices connected through tunnel.
+			 */
+
+			if (pci_read_config_byte
+			    (pdev, link_a_b_off + 0xd, &linkwidth))
+				_IPATH_UNIT_ERROR(t,
+						  "Couldn't read HT link frequency config register\n");
+			else {
+				uint32_t speed;
+				switch (linkwidth & 0xf) {
+				case 6:
+					speed = 1000;
+					break;
+				case 5:
+					speed = 800;
+					break;
+				case 4:
+					speed = 600;
+					break;
+				case 3:
+					speed = 500;
+					break;
+				case 2:
+					speed = 400;
+					break;
+				case 1:
+					speed = 300;
+					break;
+				default:
+					/*
+					 * assume reserved and
+					 * vendor-specific are 200...
+					 */
+				case 0:
+					speed = 200;
+					break;
+				}
+				((ipath_devdata *) pci_get_drvdata(pdev))->
+				    ipath_htspeed = speed;
+			}
+		} else if (cap_type == HT_INTR_DISC_CONFIG) {
+		    	/* use indirection register to get the intr handler */
+			uint32_t intvec;
+			pci_write_config_byte(pdev, pos + HT_INTR_REG_INDEX,
+					      0x10);
+			pci_read_config_dword(pdev, pos + 4,
+					      &int_handler_addr_lower);
+
+			pci_write_config_byte(pdev, pos + HT_INTR_REG_INDEX,
+					      0x11);
+			pci_read_config_dword(pdev, pos + 4,
+					      &int_handler_addr_upper);
+
+			ihandler = (uint64_t) int_handler_addr_lower |
+				((uint64_t) int_handler_addr_upper << 32);
+
+			/*
+			 * I'm unable to find an exported API to get
+			 * the the actual vector, either from the PCI
+			 * infrastructure, or from the APIC
+			 * infrastructure.  This heuristic seems to be
+			 * valid for Opteron on 2.6.x kernels, for irq's > 2.
+			 * It may not be universally true... Bug 2338
+			 *
+			 * Oh well; the heuristic doesn't work for the
+			 * AMI/Iwill BIOS...  But the good news is,
+			 * somewhere by 2.6.9, when CONFIG_PCI_MSI is
+			 * enabled, the irq field actually turned into
+			 * the vector number
+			 * We therefore require that MSI be enabled...
+			 */
+
+			intvec = pdev->irq;
+			/*
+			 * clear any bits there; normally not set but
+			 * we'll overload this for some debug purposes
+			 * (setting the HTC debug register value from
+			 * software, rather than GPIOs), so it might be
+			 * set on a driver reload.
+			 */
+
+			ihandler &= ~0xff0000;
+			/* x86 vector goes in intrinfo[23:16] */
+			ihandler |= intvec << 16;
+			_IPATH_VDBG
+			    ("ihandler lower %x, upper %x, intvec %x, interruptconfig %llx\n",
+			     int_handler_addr_lower, int_handler_addr_upper,
+			     intvec, ihandler);
+
+			/* return to caller, can't program yet. */
+			*iaddr = ihandler;
+			/*
+			 * no break, have to be sure we find link control
+			 * stuff also
+			 */
+		}
+
+	} while((pos=pci_find_next_capability(pdev, pos, HT_CAPABILITY_ID)));
+
+	if (!ihandler) {
+		_IPATH_UNIT_ERROR(t,
+			"Couldn't find interrupt handler in config space\n");
+		ret = -ENODEV;
+	}
+	return ret;
+}
+
+/*
+ * get the GUID from the i2c device
+ * When we add the multi-chip support, we will probably have to add
+ * the ability to use the number of guids field, and get the guid from
+ * the first chip's flash, to use for all of them.
+ */
+
+static void ipath_get_guid(const ipath_type t)
+{
+	void *buf;
+	struct ipath_flash *ifp;
+	uint64_t guid;
+	int len;
+	uint8_t csum, *bguid;
+
+	if (t && devdata[0].ipath_nguid > 1 && t <= devdata[0].ipath_nguid) {
+		uint8_t oguid;
+		devdata[t].ipath_guid = devdata[0].ipath_guid;
+		bguid = (uint8_t *) & devdata[t].ipath_guid;
+
+		oguid = bguid[7];
+		bguid[7] += t;
+		if (oguid > bguid[7]) {
+			if (bguid[6] == 0xff) {
+				if (bguid[5] == 0xff) {
+					_IPATH_UNIT_ERROR(t,
+							  "Can't set %s GUID from base GUID, wraps to OUI!\n",
+							  ipath_get_unit_name
+							  (t));
+					devdata[t].ipath_guid = 0;
+					return;
+				}
+				bguid[5]++;
+			}
+			bguid[6]++;
+		}
+		devdata[t].ipath_nguid = 1;
+
+		_IPATH_DBG
+		    ("nguid %u, so adding %u to device 0 guid, for %llx (big-endian)\n",
+		     devdata[0].ipath_nguid, t, devdata[t].ipath_guid);
+		return;
+	}
+
+	len = offsetof(struct ipath_flash, if_future);
+	if (!(buf = vmalloc(len))) {
+		_IPATH_UNIT_ERROR(t,
+				  "Couldn't allocate memory to read %u bytes from eeprom for GUID\n",
+				  len);
+		return;
+	}
+
+	if (ipath_eeprom_read(t, 0, buf, len)) {
+		_IPATH_UNIT_ERROR(t, "Failed reading GUID from eeprom\n");
+		goto done;
+	}
+	ifp = (struct ipath_flash *)buf;
+
+	csum = ipath_flash_csum(ifp, 0);
+	if (csum != ifp->if_csum) {
+		_IPATH_INFO("Bad I2C flash checksum: 0x%x, not 0x%x\n",
+			    csum, ifp->if_csum);
+		goto done;
+	}
+	if (*(uint64_t *) ifp->if_guid == 0ULL
+	    || *(uint64_t *) ifp->if_guid == ~0ULL) {
+		_IPATH_UNIT_ERROR(t, "Invalid GUID %llx from flash; ignoring\n",
+				  *(uint64_t *) ifp->if_guid);
+		goto done;	/* don't allow GUID if all 0 or all 1's */
+	}
+
+	/* complain, but allow it */
+	if (*(uint64_t *) ifp->if_guid == 0x100007511000000)
+		_IPATH_INFO
+		    ("Warning, GUID %llx is default, probabaly not correct!\n",
+		     *(uint64_t *) ifp->if_guid);
+
+	bguid = ifp->if_guid;
+	if(!bguid[0] && !bguid[1] && !bguid[2]) {
+		/* original incorrect GUID format in flash; fix in core copy, by
+		 * shifting up 2 octets; don't need to change top octet, since both
+		 * it and shifted are 0.. */
+		bguid[1] = bguid[3];
+		bguid[2] = bguid[4];
+		bguid[3] = bguid[4] = 0;
+		guid = *(uint64_t *)ifp->if_guid;
+		_IPATH_VDBG("Old GUID format in flash, top 3 zero, shifting 2 octets\n");
+	}
+	else
+		guid = *(uint64_t *)ifp->if_guid;
+	devdata[t].ipath_guid = guid;
+	devdata[t].ipath_nguid = ifp->if_numguid;
+	memcpy(devdata[t].ipath_serial, ifp->if_serial, sizeof(ifp->if_serial));
+	_IPATH_VDBG("Initted GUID to %llx (big-endian) from i2c flash\n",
+		    devdata[t].ipath_guid);
+
+done:
+	vfree(buf);
+}
+
+static void __devexit infinipath_remove_one(struct pci_dev *pdev)
+{
+	ipath_devdata *dd;
+
+	_IPATH_VDBG("pci_release, pdev=%p\n", pdev);
+	if (pdev) {
+		device_remove_file(&(pdev->dev), &dev_attr_status);
+		device_remove_file(&(pdev->dev), &dev_attr_status_str);
+		device_remove_file(&(pdev->dev), &dev_attr_lid);
+		device_remove_file(&(pdev->dev), &dev_attr_mlid);
+		device_remove_file(&(pdev->dev), &dev_attr_guid);
+		device_remove_file(&(pdev->dev), &dev_attr_nguid);
+		device_remove_file(&(pdev->dev), &dev_attr_serial);
+		device_remove_file(&(pdev->dev), &dev_attr_unit);
+		dd = pci_get_drvdata(pdev);
+		pci_set_drvdata(pdev, NULL);
+		_IPATH_VDBG
+		    ("Releasing pci memory regions, devdata %p, unit %u\n", dd,
+		     (uint32_t) (dd - devdata));
+		if (dd && dd->ipath_kregbase) {
+			_IPATH_VDBG("Unmapping kregbase %p\n",
+				    dd->ipath_kregbase);
+			iounmap((void *)dd->ipath_kregbase);
+			dd->ipath_kregbase = NULL;
+		}
+		pci_release_regions(pdev);
+		_IPATH_VDBG("calling pci_disable_device\n");
+		pci_disable_device(pdev);
+	}
+}
+
+int ipath_open(struct inode *, struct file *);
+static int ipath_opensma(struct inode *, struct file *);
+int ipath_close(struct inode *, struct file *);
+static unsigned int ipath_poll(struct file *, struct poll_table_struct *);
+long ipath_ioctl(struct file *, unsigned int, unsigned long);
+static loff_t ipath_llseek(struct file *, loff_t, int);
+static int ipath_mmap(struct file *, struct vm_area_struct *);
+
+static struct file_operations ipath_fops = {
+	.owner = THIS_MODULE,
+	.open = ipath_open,
+	.release = ipath_close,
+	.poll = ipath_poll,
+	/*
+	 * all of ours are completely compatible and don't require the
+	 * kernel lock
+	 */
+	.compat_ioctl = ipath_ioctl,
+	/* we don't need kernel lock for our ioctls */
+	.unlocked_ioctl = ipath_ioctl,
+	.llseek = ipath_llseek,
+	.mmap = ipath_mmap
+};
+
+static DECLARE_MUTEX(ipath_mutex);	/* general driver use */
+spinlock_t ipath_pioavail_lock;
+
+/*
+ * For now, at least (and probably forever), we don't require root
+ * or equivalent permissions to use the device.
+ */
+
+int ipath_open(struct inode *in, struct file *fp)
+{
+	int ret = 0, minor, i, prefunit=-1, devmax;
+	int maxofallports, npresent = 0, notup = 0;
+	ipath_type ndev;
+
+	down(&ipath_mutex);
+
+	minor = iminor(in);
+	_IPATH_VDBG("open on dev %lx (minor %d)\n", (long)in->i_rdev, minor);
+
+	/* This code is present to allow a knowledgeable person to specify the
+	 * layout of processes to processors before opening this driver, and
+	 * then we'll assign the process to the "closest" HT-400 to
+	 * that processor * (we assume reasonable connectivity, for now).
+	 * This code assumes that if affinity has been set before this
+	 * point, that at most one cpu is set; for now this is reasonable.
+	 * I check for both cpus_empty() and cpus_full(), in case some
+	 * kernel variant sets none of the bits when no affinity is set.
+	 * 2.6.11 and 12 kernels have all present cpus set.
+	 * Some day we'll have to fix it up further to handle a cpu subset.
+	 * This algorithm fails for two HT-400's connected in tunnel fashion.
+	 * Eventually this needs real topology information.
+	 * There may be some issues with dual core numbering as well.  This
+	 * needs more work prior to release.
+	 */
+	if(minor != IPATH_SMA
+#ifdef IPATH_DIAG
+		&& minor != IPATH_DIAG
+#endif
+		&& minor != IPATH_CTRL
+		&& !cpus_empty(current->cpus_allowed)
+		&& !cpus_full(current->cpus_allowed)) {
+		int ncpus = num_online_cpus(), curcpu = -1;
+		for(i=0; i<ncpus; i++)
+			if(cpu_isset(i, current->cpus_allowed)) {
+				_IPATH_PRDBG("%s[%u] affinity set for cpu %d\n",
+					current->comm, current->pid, i);
+				curcpu = i;
+			}
+		if(curcpu != -1) {
+			for(ndev = 0; ndev < infinipath_max; ndev++)
+				if((devdata[ndev].ipath_flags & IPATH_PRESENT)
+				    && devdata[ndev].ipath_kregbase)
+					npresent++;
+			if(npresent) {
+				prefunit = curcpu/(ncpus/npresent);
+				_IPATH_DBG("%s[%u] %d chips, %d cpus, "
+					   "%d cpus/chip, select unit %d\n",
+					   current->comm, current->pid,
+					   npresent, ncpus, ncpus/npresent,
+					   prefunit);
+			}
+		}
+	}
+
+	if (minor == IPATH_SMA) {
+		ret = ipath_opensma(in, fp);
+		/* for ipath_ioctl */
+		fp->private_data = (void *)(unsigned long)minor;
+		goto done;
+	}
+#ifdef IPATH_DIAG
+	else if (minor == IPATH_DIAG) {
+		ret = ipath_opendiag(in, fp);
+		/* for ipath_ioctl */
+		fp->private_data = (void *)(unsigned long)minor;
+		goto done;
+	}
+#endif
+	else if (minor == IPATH_CTRL) {
+		/* for ipath_ioctl */
+		fp->private_data = (void *)(unsigned long)minor;
+		ret = 0;
+		goto done;
+	}
+	else if (minor) {
+		/*
+		 * minor number 0 is used for all chips, we choose available
+		 * chip ourselves, it isn't based on what they open.
+		 */
+
+		_IPATH_DBG("open on invalid minor %u\n", minor);
+		ret = -ENXIO;
+		goto done;
+	}
+
+	/*
+	 * for now, we use all ports on one, then all ports on the
+	 * next, etc.  Eventually we want to tweak this to be cpu/chip
+	 * topology aware, and round-robin across chips that are 
+	 * configured and connected, placing processes on the closest
+	 * available processor that isn't already over-allocated.
+	 * multi-HT400 topology could be better handled
+	 */
+
+	npresent = maxofallports = 0;
+	for (ndev = 0; ndev < infinipath_max; ndev++) {
+		if (!(devdata[ndev].ipath_flags & IPATH_PRESENT) ||
+		    !devdata[ndev].ipath_kregbase)
+			continue;
+		npresent++;
+		if ((devdata[ndev].
+		     ipath_flags & (IPATH_LINKDOWN | IPATH_LINKUNK))) {
+			_IPATH_VDBG("unit %u present, but link not ready\n",
+				    ndev);
+			notup++;
+			continue;
+		} else if (!devdata[ndev].ipath_lid) {
+			_IPATH_VDBG
+			    ("unit %u present, but LID not assigned, down\n",
+			     ndev);
+			notup++;
+			continue;
+		}
+		if (devdata[ndev].ipath_cfgports > maxofallports)
+			maxofallports = devdata[ndev].ipath_cfgports;
+	}
+
+	/*
+	 * user ports start at 1, kernel port is 0
+	 * For now, we do round-robin access across all chips
+	 */
+
+	devmax = prefunit!=-1 ? prefunit+1 : infinipath_max;
+recheck:
+	for (i = 1; i < maxofallports; i++) {
+		for (ndev = prefunit!=-1?prefunit:0; ndev < devmax; ndev++) {
+			if (!(devdata[ndev].ipath_flags & IPATH_PRESENT) ||
+			    !devdata[ndev].ipath_kregbase
+			    || !devdata[ndev].ipath_lid
+			    || (devdata[ndev].
+				ipath_flags & (IPATH_LINKDOWN | IPATH_LINKUNK)))
+				break;	/* can't use this chip */
+			if (i >= devdata[ndev].ipath_cfgports)
+				break;	/* max'ed out on users of this chip */
+			if (!devdata[ndev].ipath_pd[i]) {
+				void *p, *ptmp;
+				p = kmalloc(sizeof(ipath_portdata), GFP_KERNEL);
+
+				/*
+				 * allocate memory for use in
+				 * ipath_tid_update() just once at open,
+				 * not per call.  Reduces cost of expected
+				 * send setup
+				 */
+
+				ptmp =
+				    kmalloc(devdata[ndev].ipath_rcvtidcnt *
+					    sizeof(uint16_t)
+					    +
+					    devdata[ndev].ipath_rcvtidcnt *
+					    sizeof(struct page **), GFP_KERNEL);
+				if (!p || !ptmp) {
+					_IPATH_UNIT_ERROR(ndev,
+							  "Unable to allocate portdata memory, failing open\n");
+					ret = -ENOMEM;
+					kfree(p);
+					kfree(ptmp);
+					goto done;
+				}
+				memset(p, 0, sizeof(ipath_portdata));
+				devdata[ndev].ipath_pd[i] = p;
+				devdata[ndev].ipath_pd[i]->port_port = i;
+				devdata[ndev].ipath_pd[i]->port_unit = ndev;
+				devdata[ndev].ipath_pd[i]->port_tid_pg_list =
+				    ptmp;
+				init_waitqueue_head(&devdata[ndev].ipath_pd[i]->
+						    port_wait);
+			}
+			if (!devdata[ndev].ipath_pd[i]->port_cnt) {
+				devdata[ndev].ipath_pd[i]->port_cnt = 1;
+				fp->private_data =
+				    (void *)devdata[ndev].ipath_pd[i];
+				_IPATH_PRDBG("%s[%u] opened unit:port %u:%u\n",
+					     current->comm, current->pid, ndev,
+					     i);
+				devdata[ndev].ipath_pd[i]->port_pid =
+				    current->pid;
+				strncpy(devdata[ndev].ipath_pd[i]->port_comm,
+					current->comm,
+					sizeof(devdata[ndev].ipath_pd[i]->
+					       port_comm));
+				ipath_stats.sps_ports++;
+				goto done;
+			}
+		}
+	}
+
+	if (npresent) {
+		if (notup) {
+			ret = -ENETDOWN;
+			_IPATH_DBG
+			    ("No ports available (none initialized and ready)\n");
+		} else {
+			if(prefunit > 0) { /* if we started above unit 0, retry from 0 */
+				_IPATH_PRDBG("%s[%u] no ports on prefunit %d, clear and re-check\n",
+					current->comm, current->pid, prefunit);
+				devmax = infinipath_max;
+				prefunit = -1;
+				goto recheck;
+			}
+			ret = -EBUSY;
+			_IPATH_DBG("No ports available\n");
+		}
+	} else {
+		ret = -ENXIO;
+		_IPATH_DBG("No boards found\n");
+	}
+
+done:
+	up(&ipath_mutex);
+	return ret;
+}
+
+static int ipath_opensma(struct inode *in, struct file *fp)
+{
+	ipath_type s;
+
+	if (ipath_sma_alive) {
+		_IPATH_DBG("SMA already running (pid %u), failing\n",
+			   ipath_sma_alive);
+		return -EBUSY;
+	}
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;	/* all SMA functions are root-only */
+
+	for (s = 0; s < infinipath_max; s++) {
+		/* we need at least one infinipath device to be initialized. */
+		if (devdata[s].ipath_flags & IPATH_INITTED) {
+			ipath_sma_alive = current->pid;
+			*devdata[s].ipath_statusp |= IPATH_STATUS_SMA;
+			*devdata[s].ipath_statusp &= ~IPATH_STATUS_OIB_SMA;
+		}
+	}
+	if (ipath_sma_alive) {
+		_IPATH_SMADBG
+			("SMA device now open, SMA active as PID %u\n",
+			 ipath_sma_alive);
+		return 0;
+	}
+	_IPATH_DBG("No hardware yet found and initted, failing\n");
+	return -ENODEV;
+}
+
+
+#ifdef IPATH_DIAG
+static int ipath_opendiag(struct inode *in, struct file *fp)
+{
+	ipath_type s;
+
+	if (ipath_diag_alive) {
+		_IPATH_DBG("Diags already running (pid %u), failing\n",
+			   ipath_diag_alive);
+		return -EBUSY;
+	}
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;	/* all diags functions are root-only */
+
+	for (s = 0; s < infinipath_max; s++)
+		/*
+		 * we need at least one infinipath device to be present
+		 * (don't use INITTED, because we want to be able to open
+		 * even if device is in freeze mode, which cleared INITTED.
+		 * There is s small amount of risk to this, which is
+		 * why we also verify kregbase is set.
+		 */
+
+		if ((devdata[s].ipath_flags & IPATH_PRESENT)
+		    && devdata[s].ipath_kregbase) {
+			ipath_diag_alive = current->pid;
+			_IPATH_DBG("diag device now open, active as PID %u\n",
+				   ipath_diag_alive);
+			return 0;
+		}
+	_IPATH_DBG("No hardware yet found and initted, failing diags\n");
+	return -ENODEV;
+}
+#endif
+
+/*
+ * clear all TID entries for a port, expected and eager.
+ * Used from ipath_close(), and at chip initialization.
+ */
+
+static void ipath_clear_tids(const ipath_type t, unsigned port)
+{
+	volatile uint64_t *tidbase;
+	int i;
+	ipath_devdata *dd;
+	uint64_t tidval;
+	dd = &devdata[t];
+
+	if (!dd->ipath_kregbase)
+		return;
+
+	/*
+	 * chip errata bug 7358, try to work around it by marking invalid
+	 * tids as having max length
+	 */
+
+	tidval =
+	    (~0ULL & INFINIPATH_RT_BUFSIZE_MASK) << INFINIPATH_RT_BUFSIZE_SHIFT;
+
+	/*
+	 * need to invalidate all of the expected TID entries for this
+	 * port, so we don't have valid entries that might somehow get
+	 * used (early in next use of this port, or through some bug)
+	 * We don't bother with the eager, because they are initialized
+	 * each time before receives are enabled; expected aren't
+	 */
+
+	tidbase = (volatile uint64_t *)((char *)(dd->ipath_kregbase) +
+					dd->ipath_rcvtidbase +
+					port * dd->ipath_rcvtidcnt *
+					sizeof(*tidbase));
+	_IPATH_VDBG("Invalidate expected TIDs for port %u, tidbase=%p\n", port,
+		    tidbase);
+	for (i = 0; i < dd->ipath_rcvtidcnt; i++)
+		ipath_kput_memq(t, &tidbase[i], tidval);
+	yield();		/* don't hog the cpu */
+
+	/* zero the eager TID entries */
+	tidbase = (volatile uint64_t *)((char *)(dd->ipath_kregbase) +
+					dd->ipath_rcvegrbase +
+					port * dd->ipath_rcvegrcnt *
+					sizeof(*tidbase));
+
+	for (i = 0; i < dd->ipath_rcvegrcnt; i++)
+		ipath_kput_memq(t, &tidbase[i], tidval);
+	yield();		/* don't hog the cpu */
+}
+
+int ipath_close(struct inode *in, struct file *fp)
+{
+	int ret = 0;
+	ipath_portdata *pd;
+
+	_IPATH_VDBG("close on dev %lx, private data %p\n", (long)in->i_rdev,
+		    fp->private_data);
+
+	down(&ipath_mutex);
+	if (iminor(in) == IPATH_SMA) {
+		ipath_type s;
+
+		ipath_sma_alive = 0;
+		_IPATH_SMADBG("Closing SMA device\n");
+		for (s = 0; s < infinipath_max; s++) {
+			if (!(devdata[s].ipath_flags & IPATH_INITTED))
+				continue;
+			*devdata[s].ipath_statusp &= ~IPATH_STATUS_SMA;
+			if (devdata[s].verbs_layer.l_flags &
+			    IPATH_VERBS_KERNEL_SMA)
+				*devdata[s].ipath_statusp |=
+				    IPATH_STATUS_OIB_SMA;
+		}
+	}
+#ifdef IPATH_DIAG
+	else if (iminor(in) == IPATH_DIAG) {
+		ipath_diag_alive = 0;
+		_IPATH_DBG("Closing DIAG device\n");
+	}
+#endif
+	else if (fp->private_data && 255UL < (unsigned long)fp->private_data) {
+		ipath_type t;
+		unsigned port;
+		ipath_devdata *dd;
+
+		pd = (ipath_portdata *) fp->private_data;
+		port = pd->port_port;
+		fp->private_data = NULL;
+		t = pd->port_unit;
+		if (t > infinipath_max) {
+			_IPATH_ERROR
+			    ("closing, fp %p, pd %p, but unit %x not valid!\n",
+			     fp, pd, t);
+			goto done;
+		}
+		dd = &devdata[t];
+
+		if (pd->port_hdrqfull) {
+			_IPATH_PRDBG
+			    ("%s[%u] had %u rcvhdrqfull errors during run\n",
+			     pd->port_comm, pd->port_pid, pd->port_hdrqfull);
+			pd->port_hdrqfull = 0;
+		}
+
+		if (pd->port_rcvwait_to || pd->port_piowait_to
+		    || pd->port_rcvnowait || pd->port_pionowait) {
+			_IPATH_VDBG
+			    ("port%u, %u rcv, %u pio wait timeo; %u rcv %u, pio already\n",
+			     pd->port_port, pd->port_rcvwait_to,
+			     pd->port_piowait_to, pd->port_rcvnowait,
+			     pd->port_pionowait);
+			pd->port_rcvwait_to = pd->port_piowait_to =
+			    pd->port_rcvnowait = pd->port_pionowait = 0;
+		}
+		if (pd->port_flag) {
+			_IPATH_DBG("port %u port_flag still set to 0x%x\n",
+				   pd->port_port, pd->port_flag);
+			pd->port_flag = 0;
+		}
+
+		if (devdata[t].ipath_kregbase) {
+			if (pd->port_rcvhdrtail_uaddr) {
+				pd->port_rcvhdrtail_uaddr = 0;
+				pd->port_rcvhdrtail_kvaddr = NULL;
+				ipath_munlock(1, &pd->port_rcvhdrtail_pagep);
+				pd->port_rcvhdrtail_pagep = NULL;
+				ipath_stats.sps_pageunlocks++;
+			}
+			ipath_kput_kreg_port(t, kr_rcvhdrtailaddr, port, 0ULL);
+			ipath_kput_kreg_port(pd->port_unit, kr_rcvhdraddr,
+					     pd->port_port, 0);
+
+			/* clean up the pkeys for this port user */
+			ipath_clean_partkey(pd, dd);
+
+			if (port < dd->ipath_cfgports) {
+				int i = dd->ipath_pbufsport * (port - 1);
+				ipath_disarm_piobufs(t, i, dd->ipath_pbufsport);
+
+				/* atomically clear receive enable port. */
+				atomic_clear_mask(1U <<
+						  (INFINIPATH_R_PORTENABLE_SHIFT
+						   + port),
+						  &devdata[t].ipath_rcvctrl);
+				ipath_kput_kreg(t, kr_rcvctrl,
+						devdata[t].ipath_rcvctrl);
+
+				if (dd->ipath_pageshadow) {
+					/*
+					 * unlock any expected TID
+					 * entries port still had in use
+					 */
+					int port_tidbase =
+					    pd->port_port * dd->ipath_rcvtidcnt;
+					int i, cnt = 0, maxtid =
+					    port_tidbase + dd->ipath_rcvtidcnt;
+
+					_IPATH_VDBG
+					    ("Port %u unlocking any locked expTID pages\n",
+					     pd->port_port);
+					for (i = port_tidbase; i < maxtid; i++) {
+						if (dd->ipath_pageshadow[i]) {
+							ipath_munlock(1,
+								      &dd->
+								      ipath_pageshadow
+								      [i]);
+							dd->ipath_pageshadow[i]
+							    = NULL;
+							cnt++;
+							ipath_stats.
+							    sps_pageunlocks++;
+						}
+					}
+					if (cnt)
+						_IPATH_VDBG
+						    ("Port %u had %u expTID entries locked\n",
+						     pd->port_port, cnt);
+					if (ipath_stats.sps_pagelocks
+					    || ipath_stats.sps_pageunlocks)
+						_IPATH_VDBG
+						    ("%llu pages locked, %llu unlocked with"
+						     " ipath_m{un}lock\n",
+						     ipath_stats.sps_pagelocks,
+						     ipath_stats.
+						     sps_pageunlocks);
+				}
+				ipath_stats.sps_ports--;
+				_IPATH_PRDBG("%s[%u] closed port %u:%u\n",
+					     pd->port_comm, pd->port_pid, t,
+					     port);
+			}
+		}
+
+		pd->port_cnt = 0;
+		pd->port_pid = 0;
+
+		ipath_clear_tids(t, pd->port_port);
+
+		ipath_free_pddata(dd, pd->port_port, 0);
+	}
+
+done:
+	up(&ipath_mutex);
+
+	return ret;
+}
+
+/*
+ * cancel a range of PIO buffers, used when they might be armed, but
+ * not triggered.  Used at init to ensure buffer state, and also user
+ * process close, in case it died while writing to a PIO buffer
+ */
+
+static void ipath_disarm_piobufs(const ipath_type t, unsigned first,
+				 unsigned cnt)
+{
+	unsigned i, last = first + cnt;
+	uint64_t sendctrl;
+	for (i = first; i < last; i++) {
+		sendctrl = devdata[t].ipath_sendctrl | INFINIPATH_S_DISARM |
+		    (i << INFINIPATH_S_DISARMPIOBUF_SHIFT);
+		ipath_kput_kreg(t, kr_sendctrl, sendctrl);
+	}
+}
+
+static void ipath_clean_partkey(ipath_portdata * pd, ipath_devdata * dd)
+{
+	int i, j, pchanged = 0;
+	uint64_t oldpkey;
+
+	/* for debugging only */
+	oldpkey =
+	    (uint64_t) dd->ipath_pkeys[0] | ((uint64_t) dd->
+					     ipath_pkeys[1] << 16)
+	    | ((uint64_t) dd->ipath_pkeys[2] << 32)
+	    | ((uint64_t) dd->ipath_pkeys[3] << 48);
+
+	for (i = 0; i < (sizeof(pd->port_pkeys) / sizeof(pd->port_pkeys[0]));
+	     i++) {
+		if (!pd->port_pkeys[i])
+			continue;
+		_IPATH_VDBG("look for key[%d] %hx in pkeys\n", i,
+			    pd->port_pkeys[i]);
+		for (j = 0;
+		     j < (sizeof(dd->ipath_pkeys) / sizeof(dd->ipath_pkeys[0]));
+		     j++) {
+			/* check for match independent of the global bit */
+			if ((dd->ipath_pkeys[j] & 0x7fff) ==
+			    (pd->port_pkeys[i] & 0x7fff)) {
+				if (atomic_dec_and_test(&dd->ipath_pkeyrefs[j])) {
+					_IPATH_VDBG
+					    ("p%u clear key %x matches #%d\n",
+					     pd->port_port, pd->port_pkeys[i],
+					     j);
+					ipath_stats.sps_pkeys[j] =
+					    dd->ipath_pkeys[j] = 0;
+					pchanged++;
+				} else
+					_IPATH_VDBG
+					    ("p%u key %x matches #%d, but ref still %d\n",
+					     pd->port_port, pd->port_pkeys[i],
+					     j,
+					     atomic_read(&dd->
+							 ipath_pkeyrefs[j]));
+				break;
+			}
+		}
+		pd->port_pkeys[i] = 0;
+	}
+	if (pchanged) {
+		uint64_t pkey;
+		pkey =
+		    (uint64_t) dd->ipath_pkeys[0] | ((uint64_t) dd->
+						     ipath_pkeys[1] << 16)
+		    | ((uint64_t) dd->ipath_pkeys[2] << 32)
+		    | ((uint64_t) dd->ipath_pkeys[3] << 48);
+		_IPATH_VDBG("p%u old pkey reg %llx, new pkey reg %llx\n",
+			    pd->port_port, oldpkey, pkey);
+		ipath_kput_kreg(pd->port_unit, kr_partitionkey, pkey);
+	}
+}
+
+static unsigned int ipath_poll(struct file *fp, struct poll_table_struct *pt)
+{
+	int ret;
+	ipath_portdata *pd;
+
+	pd = port_fp(fp);
+	/* nothing for select/poll in this driver, at least for now */
+	ret = 0;
+
+	return ret;
+}
+
+/*
+ * wait up to msecs milliseconds for IB link state change to occur
+ * for now, take the easy polling route.  Currently used only by
+ * the SMA ioctls.  Returns 0 if state reached, otherwise -ETIMEDOUT
+ * state can have multiple states set, for any of several transitions.
+ */
+
+int ipath_wait_linkstate(const ipath_type t, uint32_t state, int msecs)
+{
+	devdata[t].ipath_sma_state_wanted = state;
+	wait_event_interruptible_timeout(ipath_sma_state_wait,
+					 (devdata[t].ipath_flags & state),
+					 msecs_to_jiffies(msecs));
+	devdata[t].ipath_sma_state_wanted = 0;
+
+	if (!(devdata[t].ipath_flags & state))
+		_IPATH_DBG
+		    ("Didn't reach linkstate %s within %u ms (ibcc %llx %s)\n",
+		     /* test INIT ahead of DOWN, both can be set */
+		     (state & IPATH_LINKINIT) ? "INIT" :
+		     ((state & IPATH_LINKDOWN) ? "DOWN" :
+		      ((state & IPATH_LINKARMED) ? "ARM" : "ACTIVE")),
+		     msecs, ipath_kget_kreg64(t, kr_ibcctrl),
+		     ipath_ibcstatus_str[ipath_kget_kreg64(t, kr_ibcstatus) &
+					 0xf]);
+	return (devdata[t].ipath_flags & state) ? 0 : -ETIMEDOUT;
+}
+
+/* unit number is already validated in ipath_ioctl() */
+static int ipath_kset_lid(uint32_t arg)
+{
+	unsigned unit = (arg >> 16) & 0xffff;
+
+	if (unit >= infinipath_max
+	    || !(devdata[unit].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", unit);
+		return -ENODEV;
+	}
+
+	arg &= 0xffff;
+	_IPATH_SMADBG("Unit %u setting lid to 0x%x, was 0x%x\n", unit, arg,
+		      devdata[unit].ipath_lid);
+	ipath_set_sps_lid(unit, arg);
+	return 0;
+}
+
+static int ipath_kset_mlid(uint32_t arg)
+{
+	unsigned unit = (arg >> 16) & 0xffff;
+
+	if (unit >= infinipath_max
+	    || !(devdata[unit].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", unit);
+		return -ENODEV;
+	}
+
+	arg &= 0xffff;
+	_IPATH_SMADBG("Unit %u setting mlid to 0x%x, was 0x%x\n", unit, arg,
+		      devdata[unit].ipath_mlid);
+	ipath_stats.sps_mlid[unit] = devdata[unit].ipath_mlid = arg;
+	if (devdata[unit].ipath_layer.l_intr)
+		devdata[unit].ipath_layer.l_intr(unit, IPATH_LAYER_INT_BCAST);
+	return 0;
+}
+
+/* unit number is in incoming, overwritten on return with data */
+
+static int ipath_get_devstatus(uint64_t * a)
+{
+	int ret;
+	uint64_t unit64;
+	uint32_t unit;
+	uint64_t devstatus;
+
+	if ((ret = copy_from_user(&unit64, a, sizeof unit64))) {
+		_IPATH_DBG("Failed to copy in unit: %d\n", ret);
+		return ret;
+	}
+	unit = unit64;
+	if (unit >= infinipath_max
+	    || !(devdata[unit].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", unit);
+		return -ENODEV;
+	}
+
+	devstatus = *devdata[unit].ipath_statusp;
+
+	if ((ret = copy_to_user(a, &devstatus, sizeof devstatus)))
+		_IPATH_DBG("Failed to copy out device status: %d\n", ret);
+	return ret;
+}
+
+/* unit number is in incoming, overwritten on return with data */
+
+static int ipath_get_mlid(uint32_t * a)
+{
+	int ret;
+	uint32_t unit;
+	uint32_t mlid;
+
+	if ((ret = copy_from_user(&unit, a, sizeof unit))) {
+		_IPATH_DBG("Failed to copy in mlid: %d\n", ret);
+		return ret;
+	}
+	if (unit >= infinipath_max
+	    || !(devdata[unit].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", unit);
+		return -ENODEV;
+	}
+
+	mlid = devdata[unit].ipath_mlid;
+
+	if ((ret = copy_to_user(a, &mlid, sizeof mlid)))
+		_IPATH_DBG("Failed to copy out MLID: %d\n", ret);
+	return ret;
+}
+
+static int ipath_kset_guid(struct ipath_setguid *a)
+{
+	struct ipath_setguid setguid;
+	int ret;
+
+	if ((ret = copy_from_user(&setguid, a, sizeof setguid))) {
+		_IPATH_DBG("Failed to copy in guid info: %d\n", ret);
+		return ret;
+	}
+	if (setguid.sunit >= infinipath_max ||
+	    !(devdata[setguid.sunit].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %llu\n", setguid.sunit);
+		return -ENODEV;
+	}
+	if (setguid.sguid == 0ULL || setguid.sguid == ~0ULL) {
+		/*
+		 * use INFO, not DBG, because ipath_mux doesn't yet
+		 * complain about errors on this
+		 */
+
+		_IPATH_INFO("Ignoring attempt to set invalid GUID %llx\n",
+			    setguid.sguid);
+		return -EINVAL;
+	}
+	devdata[setguid.sunit].ipath_guid = setguid.sguid;
+	devdata[setguid.sunit].ipath_nguid = 1;
+	_IPATH_DBG("SMA set hardware GUID unit %llu to %llx (network order)\n",
+		   setguid.sunit, devdata[setguid.sunit].ipath_guid);
+	return 0;
+}
+
+/*
+ * receive an IB packet with QP 0 or 1.  For now, we have no timeout implemented
+ * We put the actual received count into the iov on return, and the unit we 
+ * received from goes into the lower 16 bits of sps_flags.
+ * This receives from all/any of the active chips, and we currently do not
+ * allow specifying just one (we could, by filling in unit in the library
+ * before the syscall, and checking here).
+ */
+
+static int ipath_rcvsma_pkt(struct ipath_sendpkt * p)
+{
+	struct ipath_sendpkt rpkt;
+	int i, any, ret;
+	unsigned long flags;
+
+	if ((ret = copy_from_user(&rpkt, p, sizeof rpkt))) {
+		_IPATH_DBG("Failed to copy in pkt struct (%d)\n", ret);
+		return ret;
+	}
+	if (!ipath_sma_data_spare) {
+		_IPATH_DBG("can't do receive, sma not initialized\n");
+		return -ENETDOWN;
+	}
+
+	for (any = i = 0; i < infinipath_max; i++)
+		if (devdata[i].ipath_flags & IPATH_INITTED)
+			any++;
+	if (!any) {		/* no hardware, freeze, etc. */
+		_IPATH_SMADBG("Didn't find any initialized and usable chips\n");
+		return -ENODEV;
+	}
+
+	wait_event_interruptible(ipath_sma_wait,
+				 ipath_sma_data[ipath_sma_first].len);
+
+	spin_lock_irqsave(&ipath_sma_lock, flags);
+	if (ipath_sma_data[ipath_sma_first].len) {
+		int len;
+		uint32_t slen;
+		uint8_t *sdata;
+		struct _ipath_sma_rpkt *smpkt =
+		    &ipath_sma_data[ipath_sma_first];
+
+		/*
+		 * we swap out the buffer we are going to use with the
+		 * spare buffer and set spare to that buffer.  This code
+		 * is the only code that ever manipulates spare, other
+		 * than the initialization code.  This code should never
+		 * be entered by more than one process at a time, and
+		 * if it is, the user  code doing so deserves what it gets;
+		 * it won't break anything in the driver by doing so.
+		 * We do it this way to avoid holding a lock across the
+		 * copy_to_user, which could fault, or delay a long time
+		 * while paging occurs; ditto for printks
+		 */
+
+		slen = smpkt->len;
+		sdata = smpkt->buf;
+		rpkt.sps_flags = smpkt->unit;
+		smpkt->buf = ipath_sma_data_spare;
+		ipath_sma_data_spare = sdata;
+		smpkt->len = 0;	/* it's available again */
+		if (++ipath_sma_first >= IPATH_NUM_SMAPKTS)
+			ipath_sma_first = 0;
+		spin_unlock_irqrestore(&ipath_sma_lock, flags);
+
+		len = min((uint32_t) rpkt.sps_iov[0].iov_len, slen);
+		ret =
+		    copy_to_user((void *)rpkt.sps_iov[0].iov_base, sdata, len);
+		_IPATH_VDBG
+		    ("SMA packet (index=%d), len %d (actual %d) buf %p, ubuf %llx\n",
+		     ipath_sma_first, slen, len, sdata,
+		     rpkt.sps_iov[0].iov_base);
+		if (!ret) {
+			/* actual length read. */
+			rpkt.sps_iov[0].iov_len = len;
+			rpkt.sps_cnt = 1;	/* received one packet */
+			if ((ret = copy_to_user(p, &rpkt, sizeof rpkt)))
+				_IPATH_DBG
+				    ("Failed to copy out pkt struct (%d)\n",
+				     ret);
+		} else
+			_IPATH_DBG("copyout failed: %d\n", ret);
+	} else {
+		/* usually means SMA process received a signal */
+		spin_unlock_irqrestore(&ipath_sma_lock, flags);
+		return -EAGAIN;
+	}
+
+	return ret;
+}
+
+/* unit number is in first word incoming, overwritten on return with data */
+static int ipath_get_portinfo(uint32_t * a)
+{
+	int ret;
+	uint32_t unit, tmp, tmp2;
+	ipath_devdata *dd;
+	uint32_t portinfo[13];	/* just the data for Portinfo, in host horder */
+
+	if ((ret = copy_from_user(&unit, a, sizeof unit))) {
+		_IPATH_DBG("Failed to copy in portinfo: %d\n", ret);
+		return ret;
+	}
+	if (unit >= infinipath_max
+	    || !(devdata[unit].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", unit);
+		return -ENODEV;
+	}
+	dd = &devdata[unit];
+	/* so we only initialize non-zero fields. */
+	memset(portinfo, 0, sizeof portinfo);
+
+	/*
+	 * Notimpl yet M_Key (64)
+	 * Notimpl yet GID (64)
+	 */
+
+	portinfo[4] = (dd->ipath_lid << 16);
+
+	/*
+	 * Notimpl yet SMLID (should we store this in the driver, in
+	 * case SMA dies?)
+	 * CapabilityMask is 0, we don't support any of these
+	 * DiagCode is 0; we don't store any diag info for now
+	 * Notimpl yet M_KeyLeasePeriod (we don't support M_Key)
+	 */
+
+	/* LocalPortNum is whichever port number they ask for */
+	portinfo[7] = (unit << 24)
+	    /* LinkWidthEnabled */
+	    |(2 << 16)
+	    /* LinkWidthSupported (really 2, but that's not IB valid...) */
+	    |(3 << 8)
+	    /* LinkWidthActive */
+	    |(2 << 0);
+	tmp = dd->ipath_lastibcstat & 0xff;
+	tmp2 = 5;
+	if (tmp == 0x11)
+		tmp = 2;
+	else if (tmp == 0x21)
+		tmp = 3;
+	else if (tmp == 0x31)
+		tmp = 4;
+	else {
+		tmp = 0;	/* down */
+		tmp2 = tmp & 0xf;
+	}
+	portinfo[8] = (1 << 28)	/* LinkSpeedSupported */
+	    |(tmp << 24)	/* PortState */
+	    |(tmp2 << 20)	/* PortPhysicalState */
+	    |(2 << 16)
+
+	    /* LinkDownDefaultState */
+	    /* M_KeyProtectBits == 0 */
+	    /* NotImpl yet LMC == 0 (we can support all values) */
+	    |(1 << 4)		/* LinkSpeedActive */
+	    |(1 << 0);		/* LinkSpeedEnabled */
+	switch (dd->ipath_ibmtu) {
+	case 4096:
+		tmp = 5;
+		break;
+	case 2048:
+		tmp = 4;
+		break;
+	case 1024:
+		tmp = 3;
+		break;
+	case 512:
+		tmp = 2;
+		break;
+	case 256:
+		tmp = 1;
+		break;
+	default:		/* oops, something is wrong */
+		_IPATH_DBG
+		    ("Problem, ipath_ibmtu 0x%x not a valid IB MTU, treat as 2048\n",
+		     dd->ipath_ibmtu);
+		tmp = 4;
+		break;
+	}
+	portinfo[9] = (tmp << 28)
+	    /* NeighborMTU */
+	    /* Notimpl MasterSMSL */
+	    |(1 << 20)
+
+	    /* VLCap */
+	    /* Notimpl InitType (actually, an SMA decision) */
+	    /* VLHighLimit is 0 (only one VL) */
+	    ;			/* VLArbitrationHighCap is 0 (only one VL) */
+	portinfo[10] =		/* VLArbitrationLowCap is 0 (only one VL) */
+	    /* InitTypeReply is SMA decision */
+	    (5 << 16)		/* MTUCap 4096 */
+	    |(7 << 13)		/* VLStallCount */
+	    |(0x1f << 8)	/* HOQLife */
+	    |(1 << 4)		/* OperationalVLs 0 */
+	    |(1 << 3)
+
+	    /* PartitionEnforcementInbound */
+	    /* PartitionEnforcementOutbound not enforced */
+	    /* FilterRawinbound not enforced */
+	    ;			/* FilterRawOutbound not enforced */
+	/* M_KeyViolations are not counted by hardware, SMA can count */
+	tmp = ipath_kget_creg32(unit, cr_errpkey);
+	/* P_KeyViolations are counted by hardware. */
+	portinfo[11] = ((tmp & 0xffff) << 0);
+	portinfo[12] =
+	    /* Q_KeyViolations are not counted by hardware */
+	    (1 << 8)
+
+	    /* GUIDCap */
+	    /* SubnetTimeOut handled by SMA */
+	    /* RespTimeValue handled by SMA */
+	    ;
+	/* LocalPhyErrors are programmed to max */
+	portinfo[12] |= (0xf << 20)
+	    |(0xf << 16)	/* OverRunErrors are programmed to max */
+	    ;
+
+	if ((ret = copy_to_user(a, portinfo, sizeof portinfo)))
+		_IPATH_DBG("Failed to copy out portinfo: %d\n", ret);
+	return ret;
+}
+
+/* unit number is in first word incoming, overwritten on return with data */
+static int ipath_get_nodeinfo(uint32_t * a)
+{
+	int ret;
+	uint32_t unit;		/*, tmp, tmp2; */
+	ipath_devdata *dd;
+	uint32_t nodeinfo[10];	/* just the data for Nodeinfo, in host horder */
+
+	if ((ret = copy_from_user(&unit, a, sizeof unit))) {
+		_IPATH_DBG("Failed to copy in nodeinfo: %d\n", ret);
+		return ret;
+	}
+	if (unit >= infinipath_max
+	    || !(devdata[unit].ipath_flags & IPATH_INITTED)) {
+		/* VDBG because sma normally probes for all possible units */
+		_IPATH_VDBG("Invalid unit %u\n", unit);
+		return -ENODEV;
+	}
+	dd = &devdata[unit];
+
+	/* so we only initialize non-zero fields. */
+	memset(nodeinfo, 0, sizeof nodeinfo);
+
+	nodeinfo[0] =		/* BaseVersion is SMA */
+	    /* ClassVersion is SMA */
+	    (1 << 8)		/* NodeType  */
+	    |(1 << 0);		/* NumPorts */
+	nodeinfo[1] = (uint32_t) (dd->ipath_guid >> 32);
+	nodeinfo[2] = (uint32_t) (dd->ipath_guid & 0xffffffff);
+	nodeinfo[3] = nodeinfo[1];	/* PortGUID == SystemImageGUID for us */
+	nodeinfo[4] = nodeinfo[2];	/* PortGUID == SystemImageGUID for us */
+	nodeinfo[5] = nodeinfo[3];	/* PortGUID == NodeGUID for us */
+	nodeinfo[6] = nodeinfo[4];	/* PortGUID == NodeGUID for us */
+	nodeinfo[7] = (4 << 16)	/* we support 4 pkeys */
+	    |(dd->ipath_deviceid << 0);
+	/* our chip version as 16 bits major, 16 bits minor */
+	nodeinfo[8] = dd->ipath_minrev | (dd->ipath_majrev << 16);
+	nodeinfo[9] = (unit << 24) | (dd->ipath_vendorid << 0);
+
+	if ((ret = copy_to_user(a, nodeinfo, sizeof nodeinfo)))
+		_IPATH_DBG("Failed to copy out nodeinfo: %d\n", ret);
+	return ret;
+}
+
+static int ipath_sma_ioctl(struct file *fp, unsigned int cmd, unsigned long a)
+{
+	int ret = 0;
+	switch (cmd) {
+	case IPATH_SEND_SMA_PKT:	/* send SMA packet */
+		if (!(ret = ipath_send_smapkt((struct ipath_sendpkt *) a)))
+			/* another SMA packet sent */
+			ipath_stats.sps_sma_spkts++;
+		break;
+	case IPATH_RCV_SMA_PKT:	/* recieve an SMA or MAD packet */
+		ret = ipath_rcvsma_pkt((struct ipath_sendpkt *) a);
+		break;
+	case IPATH_SET_LID:	/* set our lid, (SMA) */
+		ret = ipath_kset_lid((uint32_t) a);
+		break;
+	case IPATH_SET_MTU:	/* set the IB mtu (not maxpktlen) (SMA) */
+		ret = ipath_kset_mtu((uint32_t) a);
+		break;
+	case IPATH_SET_LINKSTATE:
+		/* walk through the linkstate states (SMA) */
+		ret = ipath_kset_linkstate((uint32_t) a);
+		break;
+	case IPATH_GET_PORTINFO:	/* get the SMA portinfo */
+		ret = ipath_get_portinfo((uint32_t *) a);
+		break;
+	case IPATH_GET_NODEINFO:	/* get the SMA nodeinfo */
+		ret = ipath_get_nodeinfo((uint32_t *) a);
+		break;
+	case IPATH_SET_GUID:
+		/*
+		 * set our guid, (SMA).  This is not normally
+		 * used, but provides a way to set the GUID when the i2c flash
+		 * has a problem, or for special testing.
+		 */
+		ret = ipath_kset_guid((struct ipath_setguid *)a);
+		break;
+	case IPATH_SET_MLID:	/* set multicast LID for ipath broadcast */
+		ret = ipath_kset_mlid((uint32_t) a);
+		break;
+	case IPATH_GET_MLID:	/* get multicast LID for ipath broadcast */
+		ret = ipath_get_mlid((uint32_t *) a);
+		break;
+	case IPATH_GET_DEVSTATUS:	/* get device status */
+		ret = ipath_get_devstatus((uint64_t *) a);
+		break;
+	default:
+		_IPATH_DBG("%x not a valid SMA ioctl for infinipath\n", cmd);
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+static int ipath_get_unit_counters(struct infinipath_getunitcounters *a)
+{
+	struct infinipath_getunitcounters c;
+
+	if(copy_from_user(&c, (void *)a, sizeof c))
+		return -EFAULT;
+	return ipath_get_counters(c.unit, (struct infinipath_counters *)c.data);
+}
Signature · 1 line
0.99.9n
#6 lore raw

RFC PATCH 5/13 ipath LLD core, part 2

Next part of ipath core driver

---

 drivers/infiniband/hw/ipath/ipath_driver.c | 2290 ++++++++++++++++++++++++++++
 1 files changed, 2290 insertions(+), 0 deletions(-)

fc2b052ff2abadc8547dc1b319883f9c942b0ae4

drivers/infiniband/hw/ipath/ipath_driver.c +2290 -0
index df650d6..0dee4ce 100644
@@ -2587,3 +2587,2293 @@ static int ipath_get_unit_counters(struc
 		return -EFAULT;
 	return ipath_get_counters(c.unit, (struct infinipath_counters *)c.data);
 }
+
+/*
+ * ioctls for the control device, which is useful when you don't want
+ * to open the main device and use up a port.
+ */
+
+static int ipath_ctrl_ioctl(struct file *fp, unsigned int cmd, unsigned long a)
+{
+	int ret = 0;
+
+	switch (cmd) {
+	case IPATH_GETSTATS:		/* return driver stats */
+		ret = ipath_get_stats((struct infinipath_stats *) a);
+		break;
+	case IPATH_GETUNITCOUNTERS:	/* return chip counters */
+		ret = ipath_get_unit_counters((struct infinipath_getunitcounters *) a);
+		break;
+	default:
+		_IPATH_DBG("%x not a valid CTRL ioctl for infinipath\n", cmd);
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+long ipath_ioctl(struct file *fp, unsigned int cmd, unsigned long a)
+{
+	int ret = 0;
+	ipath_portdata *pd;
+	ipath_type unit;
+	uint32_t tmp, i, nactive = 0;
+
+	if (cmd == IPATH_GETUNITS) {
+		/*
+		 * Return number of units supported.  This is called
+		 * here as this ioctl is needed via both the normal and
+		 * diags interface, and it does not need the device to
+		 * be opened.
+		 */
+		return ipath_get_units();
+	}
+
+	pd = port_fp(fp);
+	if (!pd) {
+		if (IPATH_SMA == (unsigned long)fp->private_data)
+			/* sma separate; no pd */
+			return (long)ipath_sma_ioctl(fp, cmd, a);
+#ifdef IPATH_DIAG
+		else if (IPATH_DIAG == (unsigned long)fp->private_data)
+			/* diags separate; no pd */
+			return (long)ipath_diags_ioctl(fp, cmd, a);
+#endif
+		else if (IPATH_CTRL == (unsigned long)fp->private_data)
+			/* ctrl separate; no pd */
+			return (long)ipath_ctrl_ioctl(fp, cmd, a);
+		else {
+			_IPATH_DBG("NULL pd from fp (%p), cmd=%x\n", fp, cmd);
+			return -ENODEV;	/* bad; shouldn't ever happen */
+		}
+	}
+
+	unit = pd->port_unit;
+
+	if ((devdata[unit].ipath_flags & IPATH_PRESENT)
+	    && (cmd == IPATH_GETCOUNTERS || cmd == IPATH_GETSTATS
+		|| cmd == IPATH_READ_EEPROM || cmd == IPATH_WRITE_EEPROM)) {
+		/* allowed to do these, as long as chip is accessible */
+	} else if (!(devdata[unit].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG
+		    ("%s not initialized (flags=0x%x), failing ioctl #%u\n",
+		     ipath_get_unit_name(unit), devdata[unit].ipath_flags,
+		     _IOC_NR(cmd));
+		ret = -ENODEV;
+	} else
+	    if ((devdata[unit].
+		 ipath_flags & (IPATH_LINKDOWN | IPATH_LINKUNK))) {
+		_IPATH_DBG("%s link is down, failing ioctl #%u\n",
+			   ipath_get_unit_name(unit), _IOC_NR(cmd));
+		ret = -ENETDOWN;
+	}
+
+	if (ret)
+		return ret;
+
+	/* normal driver ioctls, not sim-specific */
+	switch (cmd) {
+	case IPATH_USERINIT:
+		/* real application is starting on a port */
+		ret = ipath_do_user_init(pd, (struct ipath_user_info *) a);
+		break;
+	case IPATH_BASEINFO:
+		/* it's done the init, now return the info it needs */
+		ret = ipath_get_baseinfo(pd, (struct ipath_base_info *) a);
+		break;
+	case IPATH_GETPORT:
+		/*
+		 * just return the unit:port that we were assigned,
+		 * and the number of active chips.  This is is used for
+		 * doing sched_setaffinity() before initialization.
+		 */
+		for (i = 0; i < infinipath_max; i++)
+			if ((devdata[i].ipath_flags & IPATH_PRESENT)
+			    && devdata[i].ipath_kregbase
+			    && devdata[i].ipath_lid
+			    && !(devdata[i].ipath_flags &
+				 (IPATH_LINKDOWN | IPATH_LINKUNK)))
+				nactive++;
+		tmp = (nactive << 24) | (unit << 16) | unit;
+		if(copy_to_user((void *)a, &tmp, sizeof(unit)))
+			ret = EFAULT;
+		break;
+	case IPATH_GETLID:
+		/* get LID for given unit # */
+		ret = ipath_layer_get_lid(a);
+		break;
+	case IPATH_UPDM_TID:	/* update expected TID entries */
+		ret = ipath_tid_update(pd, (struct _tidupd *)a);
+		break;
+	case IPATH_FREE_TID:	/* free expected TID entries */
+		ret = ipath_tid_free(pd, (struct _tidupd *)a);
+		break;
+	case IPATH_GETCOUNTERS:	/* return chip counters */
+		ret = ipath_get_counters(unit, (struct infinipath_counters *)a);
+		break;
+	case IPATH_GETSTATS:	/* return driver stats */
+		ret = ipath_get_stats((struct infinipath_stats *) a);
+		break;
+	case IPATH_GETUNITCOUNTERS:	/* return chip counters */
+		ret = ipath_get_unit_counters((struct infinipath_getunitcounters *) a);
+		break;
+	case IPATH_SET_PKEY:	/* set a partition key */
+		ret = ipath_set_partkey(pd, (uint16_t) a);
+		break;
+	case IPATH_RCVCTRL:	/* error handling to manage the rcvq */
+		ret = ipath_manage_rcvq(pd, (uint16_t) a);
+		break;
+	case IPATH_WRITE_EEPROM:
+		/* write the eeprom (for GUID) */
+		ret = ipath_wr_eeprom(pd, (struct ipath_eeprom_req *)a);
+		break;
+	case IPATH_READ_EEPROM:	/* read the eeprom (for GUID) */
+		ret = ipath_rd_eeprom(pd->port_unit,
+				      (struct ipath_eeprom_req *)a);
+		break;
+	case IPATH_WAIT:
+		/*
+		 * wait for a receive intr for this port, or PIO avail
+		 */
+		ret = ipath_wait_intr(pd, (uint32_t) a);
+		break;
+
+	default:
+		_IPATH_DBG("cmd %x (%c,%u) not a valid ioctl\n", cmd,
+			   _IOC_TYPE(cmd), _IOC_NR(cmd));
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static loff_t ipath_llseek(struct file *fp, loff_t off, int whence)
+{
+	loff_t ret;
+
+	/* range checking is done where offset is used, not here. */
+	down(&fp->f_dentry->d_inode->i_sem);
+	if (!whence)
+		ret = fp->f_pos = off;
+	else if (whence == 1) {
+		fp->f_pos += off;
+		ret = fp->f_pos;
+	} else
+		ret = -EINVAL;
+	up(&fp->f_dentry->d_inode->i_sem);
+	_IPATH_DBG("New offset %llx from seek %llx whence=%d\n", fp->f_pos, off,
+		   whence);
+
+	return ret;
+}
+
+/*
+ * We use this to have a shared buffer between the kernel and the user
+ * code for the rcvhdr queue, egr buffers, and the per-port user regs and pio
+ * buffers in the chip.  We have the open and close entries so we can bump
+ * the ref count and keep the driver from being unloaded while still mapped.
+ */
+
+static struct vm_operations_struct ipath_vmops = {
+	.nopage = ipath_nopage,
+};
+
+static int ipath_mmap(struct file *fp, struct vm_area_struct *vm)
+{
+	int setlen = 0, ret = -EINVAL;
+	ipath_portdata *pd;
+
+	if (fp->private_data && 255UL < (unsigned long)fp->private_data) {
+		pd = port_fp(fp);
+		{
+			/*
+			 * This is the ipath_do_user_init() code,
+			 * mapping the shared buffers into the user
+			 * process. The address referred to by vm_pgoff
+			 * is the virtual, not physical, address; we only
+			 * do one mmap for each space mapped.
+			 */
+			uint64_t pgaddr, ureg;
+
+			pgaddr = vm->vm_pgoff << PAGE_SHIFT;
+
+			/*
+			 * note that ureg does *NOT* have the kregvirt
+			 * as part of it, to be sure that for 32 bit
+			 * programs, we don't end up trying to map
+			 * a > 44 address.  Has to match ipath_get_baseinfo()
+			 * code that sets __spi_uregbase
+			 */
+
+			ureg = devdata[pd->port_unit].ipath_uregbase +
+			    devdata[pd->port_unit].ipath_palign * pd->port_port;
+
+			_IPATH_MMDBG
+			    ("ushare: pgaddr %llx vm_start=%lx, vmlen %lx\n",
+			     pgaddr, vm->vm_start, vm->vm_end - vm->vm_start);
+
+			if (pgaddr == ureg) {
+				/* it's the real hardware, so io_remap works */
+				unsigned long phys;
+				if ((vm->vm_end - vm->vm_start) > PAGE_SIZE) {
+					_IPATH_INFO
+					    ("FAIL mmap userreg: reqlen %lx > PAGE\n",
+					     vm->vm_end - vm->vm_start);
+					ret = -EFAULT;
+				} else {
+					phys =
+					    devdata[pd->port_unit].
+					    ipath_physaddr + ureg;
+					vm->vm_page_prot =
+					    pgprot_noncached(vm->vm_page_prot);
+
+					vm->vm_flags |=
+					    VM_DONTCOPY | VM_DONTEXPAND | VM_IO
+					    | VM_SHM | VM_LOCKED;
+					ret =
+			io_remap_pfn_range(vm, vm->vm_start, phys >> PAGE_SHIFT,
+					   vm->vm_end - vm->vm_start,
+					   vm->vm_page_prot);
+				}
+			} else if (pgaddr == pd->port_piobufs) {
+				/*
+				 * We use io_remap, so there is not a
+				 * nopage handler for this case!
+				 * when we map the PIO buffers, we want
+				 * to map them as writeonly, no read possible.
+				 */
+
+				unsigned long phys;
+				if ((vm->vm_end - vm->vm_start) >
+				    (devdata[pd->port_unit].ipath_pbufsport *
+				     devdata[pd->port_unit].ipath_palign)) {
+					_IPATH_INFO
+					    ("FAIL mmap userreg: reqlen %lx > PAGE\n",
+					     vm->vm_end - vm->vm_start);
+					ret = -EFAULT;
+				} else {
+					phys =
+					    devdata[pd->port_unit].
+					    ipath_physaddr + pd->port_piobufs;
+					/*
+					 * Do *NOT* mark this as
+					 * non-cached (PWT bit), or we
+					 * don't get the write combining
+					 * behavior we want on the
+					 * PIO buffers!
+					 * vm->vm_page_prot = pgprot_noncached(vm->vm_page_prot);
+					 */
+
+#if defined (pgprot_writecombine) && defined(_PAGE_MA_WC)
+					/* Enable WC */
+					vm->vm_page_prot =
+					    pgprot_writecombine(vm->
+								vm_page_prot);
+#endif
+
+					if (vm->vm_flags & VM_READ) {
+						_IPATH_INFO
+						    ("Can't map piobufs as readable (flags=%lx)\n",
+						     vm->vm_flags);
+						ret = -EPERM;
+					} else {
+						/*
+						 * don't allow them to
+						 * later change to readable
+						 * with mprotect
+						 */
+
+						vm->vm_flags &= ~VM_MAYWRITE;
+
+						vm->vm_flags |=
+						    VM_DONTCOPY | VM_DONTEXPAND
+						    | VM_IO | VM_SHM |
+						    VM_LOCKED;
+						ret =
+			io_remap_pfn_range(vm, vm->vm_start, phys >> PAGE_SHIFT,
+					   vm->vm_end - vm->vm_start,
+					   vm->vm_page_prot);
+					}
+				}
+			} else if (pgaddr == (uint64_t) pd->port_rcvegr_phys) {
+				if (!pd->port_rcvegrbuf_virt)
+					return -EFAULT;
+				/*
+				 * page_alloc'ed egr memory, not
+				 * physically contiguous
+				 * *BUT* to work around the 32 bit mmap64
+				 * only handling 44 bits, we have remapped
+				 * the first page to kernel virtual, so
+				 * we have to do the conversion here to
+				 * get back to the original virtual
+				 * address (not contig pages) so we have
+				 * to mark this for special handling.
+				 */
+
+				/*
+				 * not egrbufs * egrsize since they are
+				 * no longer virtually contiguous.
+				 */
+				setlen = pd->port_rcvegrbuf_chunks * PAGE_SIZE *
+				    (1 << pd->port_rcvegrbuf_order);
+				if ((vm->vm_end - vm->vm_start) > setlen) {
+					_IPATH_INFO
+					    ("FAIL on egr bufs: reqlen %lx > actual %x\n",
+					     vm->vm_end - vm->vm_start, setlen);
+					ret = -EFAULT;
+				} else {
+					vm->vm_ops = &ipath_vmops;
+					vm->vm_private_data =
+					    (void *)(3 | (uint64_t) pd);
+					if (vm->vm_flags & VM_WRITE) {
+						_IPATH_INFO
+						    ("Can't map eager buffers as writable (flags=%lx)\n",
+						     vm->vm_flags);
+						ret = -EPERM;
+					} else {
+						/*
+						 * don't allow them to
+						 * later change to writeable
+						 * with mprotect
+						 */
+
+						vm->vm_flags &= ~VM_MAYWRITE;
+						_IPATH_MMDBG
+						    ("egrbufs, set private to %p, not %llx\n",
+						     vm->vm_private_data,
+						     pgaddr);
+						ret = 0;
+					}
+				}
+			} else if (pgaddr == (uint64_t) pd->port_rcvhdrq_phys) {
+				/*
+				 * kmalloc'ed memory, physically
+				 * contiguous; this is from
+				 * spi_rcvhdr_base; we allow user to
+				 * map read-write so they can write
+				 * hdrq entries to allow protocol code
+				 * to directly poll whether a hdrq entry
+				 * has been written.
+				 */
+				setlen =
+				    round_up(devdata[pd->port_unit].
+					     ipath_rcvhdrcnt *
+					     devdata[pd->port_unit].
+					     ipath_rcvhdrentsize *
+					     sizeof(uint32_t), PAGE_SIZE);
+				if ((vm->vm_end - vm->vm_start) > setlen) {
+					_IPATH_INFO
+					    ("FAIL on rcvhdrq: reqlen %lx > actual %x\n",
+					     vm->vm_end - vm->vm_start, setlen);
+					ret = -EFAULT;
+				} else {
+					vm->vm_ops = &ipath_vmops;
+					vm->vm_private_data =
+					    (void *)(pgaddr | 1);
+					ret = 0;
+				}
+			}
+			/*
+			 * when we map the PIO bufferavail registers,
+			 * we want to map them as readonly, no read
+			 * possible.
+			 */
+			else if (pgaddr ==
+				 devdata[pd->port_unit].
+				 ipath_pioavailregs_phys) {
+				/*
+				 * kmalloc'ed memory, physically
+				 * contiguous, one page only, readonly
+				 */
+				setlen = PAGE_SIZE;
+				if ((vm->vm_end - vm->vm_start) > setlen) {
+					_IPATH_INFO
+					    ("FAIL on pioavailregs_dma: reqlen %lx > actual %x\n",
+					     vm->vm_end - vm->vm_start, setlen);
+					ret = -EFAULT;
+				} else if (vm->vm_flags & VM_WRITE) {
+					_IPATH_INFO
+					    ("Can't map pioavailregs as writable (flags=%lx)\n",
+					     vm->vm_flags);
+					ret = -EPERM;
+				} else {
+					/*
+					 * don't allow them to later
+					 * change with mprotect
+					 */
+					vm->vm_flags &= ~VM_MAYWRITE;
+					vm->vm_ops = &ipath_vmops;
+					vm->vm_private_data =
+					    (void *)(pgaddr | 2);
+					ret = 0;
+				}
+			}
+			if (!ret && setlen) {
+				/* keep page(s) from being swapped, etc. */
+				vm->vm_flags |=
+				    VM_DONTEXPAND | VM_DONTCOPY | VM_RESERVED |
+				    VM_IO | VM_SHM;
+			} else {
+				/* failure, or io_remap case */
+				vm->vm_private_data = NULL;
+				if (ret)
+					_IPATH_INFO
+					    ("Failure %d, setlen %d, on addr %lx, off %lx\n",
+					     ret, setlen, vm->vm_start,
+					     vm->vm_pgoff);
+			}
+		}
+	} else			/* something very wrong */
+		_IPATH_INFO("fp_private wasn't set, no mmaping\n");
+
+	return ret;
+}
+
+/* page fault handler.  For each page that is first faulted in from the
+ * mmap'ed shared address buffer, this routine is called.
+ * It's always for a single page.
+ * We use the low bits of the private_data field to tell us which case
+ * we are dealing with.
+ */
+
+static struct page *ipath_nopage(struct vm_area_struct *vma, unsigned long addr,
+				 int *type)
+{
+	unsigned long avirt,	/* the original [kv]malloc virtual address */
+	 paddr,			/* physical address */
+	 off;			/* calculated page offset */
+	uint32_t which, chunk;
+	void *vaddr = NULL;
+	ipath_portdata *pd;
+	struct page *vpage = NOPAGE_SIGBUS;
+
+	if (!(avirt = (unsigned long)vma->vm_private_data)) {
+		_IPATH_DBG("NULL private_data, vm_pgoff %lx\n", vma->vm_pgoff);
+		which = 0;	/* quiet incorrect gcc warning */
+		goto done;
+	}
+	which = avirt & 3;
+	avirt &= ~3ULL;
+
+	if (addr > vma->vm_end) {
+		_IPATH_DBG("trying to fault in addr %lx past end\n", addr);
+		goto done;
+	}
+
+	/*
+	 * most of our memory is vmalloc'ed, but rcvhdr Q is physically
+	 * contiguous, either from kmalloc or alloc_pages()
+	 * pgoff is virtual.
+	 */
+	switch (which) {
+	case 1:		/* rcvhdrq_phys */
+		/* should always be 0 */
+		off = vma->vm_pgoff - (avirt >> PAGE_SHIFT);
+		paddr = addr - vma->vm_start + (off << PAGE_SHIFT) + avirt;
+		_IPATH_MMDBG("hdrq %lx (u=%lx)\n", paddr, addr);
+		vpage = pfn_to_page(paddr >> PAGE_SHIFT);
+		break;
+	case 2:		/* PIO buffer avail regs */
+		/* should always be 0 */
+		off = vma->vm_pgoff - (avirt >> PAGE_SHIFT);
+		paddr = (addr - vma->vm_start + (off << PAGE_SHIFT) + avirt);
+		_IPATH_MMDBG("pioav %lx\n", paddr);
+		vpage = pfn_to_page(paddr >> PAGE_SHIFT);
+		break;
+	case 3:
+		/*
+		 * rcvegrbufs; page_alloc()'ed like rcvhdrq, but we
+		 * have to pick out which page_alloc()'ed chunk it is.
+		 */
+		pd = (ipath_portdata *) avirt;
+		/* this should always be 0 */
+		off =
+		    vma->vm_pgoff -
+		    ((unsigned long)pd->port_rcvegr_phys >> PAGE_SHIFT);
+		off = (addr - vma->vm_start + (off << PAGE_SHIFT));
+
+		chunk = off / (PAGE_SIZE * (1 << pd->port_rcvegrbuf_order));
+		if (chunk > pd->port_rcvegrbuf_chunks)
+			_IPATH_DBG("Bad egrbuf chunk %u (max %u); off = %lx\n",
+				   chunk, pd->port_rcvegrbuf_chunks, off);
+		vaddr = pd->port_rcvegrbuf_virt[chunk] +
+		    off % (PAGE_SIZE * (1 << pd->port_rcvegrbuf_order));
+		paddr = virt_to_phys(vaddr);
+		vpage = pfn_to_page(paddr >> PAGE_SHIFT);
+		_IPATH_MMDBG("egrb %p,%lx\n", vaddr, paddr);
+		break;
+	default:
+		_IPATH_DBG
+		    ("trying to fault in mmap addr %lx (avirt %lx) that isn't known (case %u)\n",
+		     addr, avirt, which);
+	}
+
+done:
+	if (vpage != NOPAGE_SIGBUS && vpage != NOPAGE_OOM) {
+		if (which == 2)
+			/*
+			 * media/video/video-buf.c doesn't do get_page() for
+			 * buffer from alloc_page().  Hmmm.
+			 *
+			 * keep it from being swapped, complaints if
+			 * process exits before we [vf]free it, etc,
+			 * and keep shared page counts correct, etc.
+			 */
+			get_page(vpage);
+		mark_page_accessed(vpage);
+		if (type)
+			*type = VM_FAULT_MINOR;
+	} else
+		_IPATH_DBG("faultin of addr %lx vaddr %p avirt %lx failed\n",
+			   addr, vaddr, avirt);
+
+	return vpage;
+}
+
+/* this is separate to allow for better optimization of ipath_intr() */
+
+static void ipath_bad_intr(const ipath_type t, uint32_t * unexpectp)
+{
+	ipath_devdata *dd = &devdata[t];
+
+	/*
+	 * sometimes happen during driver init and unload, don't want
+	 * to process any interrupts at that point
+	 */
+
+	/* this is just a bandaid, not a fix, if something goes badly wrong */
+	if (++*unexpectp > 100) {
+		if (++*unexpectp > 105) {
+			/*
+			 * ok, we must be taking somebody else's interrupts,
+			 * due to a messed up mptable and/or PIRQ table, so
+			 * unregister the interrupt.  We've seen this
+			 * during linuxbios development work, and it
+			 * may happen in the future again.
+			 */
+			if (dd->pcidev && dd->pcidev->irq) {
+				_IPATH_UNIT_ERROR(t,
+						  "Now %u unexpected interrupts, unregistering interrupt handler\n",
+						  *unexpectp);
+				_IPATH_DBG("free_irq of irq %x\n",
+					   dd->pcidev->irq);
+				free_irq(dd->pcidev->irq, dd);
+				dd->pcidev->irq = 0;
+			}
+		}
+		if (ipath_kget_kreg32(t, kr_intmask)) {
+			_IPATH_UNIT_ERROR(t,
+					  "%u unexpected interrupts, disabling interrupts completely\n",
+					  *unexpectp);
+			/* disable all interrupts, something is very wrong */
+			ipath_kput_kreg(t, kr_intmask, 0ULL);
+		}
+	} else if (*unexpectp > 1)
+		_IPATH_DBG
+		    ("Interrupt when not ready, should not happen, ignoring\n");
+}
+
+/* separate routine, for better optimization of ipath_intr() */
+
+static void ipath_bad_regread(const ipath_type t)
+{
+	static int allbits;
+	ipath_devdata *dd = &devdata[t];
+
+	/*
+	 * We print the message and disable interrupts, in hope of
+	 * having a better chance of debugging the problem.
+	 */
+	_IPATH_UNIT_ERROR(t,
+			  "Read of interrupt status failed (all bits set)\n");
+	if (allbits++) {
+		/* disable all interrupts, something is very wrong */
+		ipath_kput_kreg(t, kr_intmask, 0ULL);
+		if (allbits == 2) {
+			_IPATH_UNIT_ERROR(t,
+					  "Still bad interrupt status, unregistering interrupt\n");
+			free_irq(dd->pcidev->irq, dd);
+			dd->pcidev->irq = 0;
+		} else if (allbits > 2) {
+			if ((allbits % 10000) == 0)
+				printk(".");
+		} else
+			_IPATH_UNIT_ERROR(t,
+					  "Disabling interrupts, multiple errors\n");
+	}
+}
+
+static irqreturn_t ipath_intr(int irq, void *data, struct pt_regs *regs)
+{
+	ipath_devdata *dd = data;
+	const ipath_type t = IPATH_UNIT(dd);
+	uint32_t istat = ipath_kget_kreg32(t, kr_intstatus);
+	uint64_t estat = 0;
+	static unsigned unexpected = 0;
+
+	if (unlikely(!istat)) {
+		ipath_stats.sps_nullintr++;
+		/* not our interrupt, or already handled */
+		return IRQ_NONE;
+	}
+	if (unlikely(istat == ~0)) {
+		ipath_bad_regread(t);
+		/* don't know if it was our interrupt or not */
+		return IRQ_NONE;
+	}
+
+	ipath_stats.sps_ints++;
+
+	/*
+	 * this needs to be flags&initted, not statusp, so we keep
+	 * taking interrupts even after link goes down, etc.
+	 * Also, we *must* clear the interrupt at some point, or we won't
+	 * take it again, which can be real bad for errors, etc...
+	 */
+
+	if (!(dd->ipath_flags & IPATH_INITTED)) {
+		ipath_bad_intr(t, &unexpected);
+		return IRQ_NONE;
+	}
+	if (unexpected)
+		unexpected = 0;
+
+	if (istat & ~infinipath_i_bitsextant)
+		_IPATH_UNIT_ERROR(t,
+				  "interrupt with unknown interrupts %x set\n",
+				  istat & (uint32_t) ~ infinipath_i_bitsextant);
+
+	if (istat & INFINIPATH_I_ERROR) {
+		ipath_stats.sps_errints++;
+		estat = ipath_kget_kreg64(t, kr_errorstatus);
+		if (!estat)
+			_IPATH_INFO
+			    ("error interrupt (%x), but no error bits set!\n",
+			     istat);
+		else if (estat == ~0ULL)
+			/*
+			 * should we try clearing all, or hope next read
+			 * works?
+			 */
+			_IPATH_UNIT_ERROR(t,
+					  "Read of error status failed (all bits set); ignoring\n");
+		else
+			ipath_handle_errors(t, estat);
+	}
+
+	if (istat & INFINIPATH_I_GPIO) {
+		/* Clear GPIO status bit 2 */
+		ipath_kput_kreg(t, kr_gpio_clear, (uint64_t)(1 << 2));
+
+		/*
+		 * Packets are available in the port 0 receive queue.
+		 * Eventually this needs to be generalized to check
+		 * IPATH_GPIO_INTR, and the specific GPIO bit, when
+		 * GPIO interrupts start being used for other things.
+		 * We skip that now to improve performance.
+		 */
+		ipath_kreceive(t);
+	}
+
+	/*
+	 * clear the ones we will deal with on this round
+	 * We clear it early, mostly for receive interrupts, so we
+	 * know the chip will have seen this by the time we process
+	 * the queue, and will re-interrupt if necessary.  The processor
+	 * itself won't take the interrupt again until we return.
+	 */
+	ipath_kput_kreg(t, kr_intclear, istat);
+
+	if (istat & INFINIPATH_I_SPIOBUFAVAIL) {
+		atomic_clear_mask(INFINIPATH_S_PIOINTBUFAVAIL,
+				  &dd->ipath_sendctrl);
+		ipath_kput_kreg(t, kr_sendctrl, dd->ipath_sendctrl);
+
+		if (dd->ipath_portpiowait) {
+			uint32_t i;
+			/*
+			 * start from port 1, since for now port 0  is
+			 * never using wait_event for PIO
+			 */
+			for (i = 1;
+			     dd->ipath_portpiowait && i < dd->ipath_cfgports;
+			     i++) {
+				if (dd->ipath_pd[i]
+				    && dd->ipath_portpiowait & (1U << i)) {
+					atomic_clear_mask(1U << i,
+							  &dd->
+							  ipath_portpiowait);
+					if (dd->ipath_pd[i]->
+					    port_flag & IPATH_PORT_WAITING_PIO)
+					{
+						dd->ipath_pd[i]->port_flag &=
+						    ~IPATH_PORT_WAITING_PIO;
+						wake_up_interruptible(&dd->
+								      ipath_pd
+								      [i]->
+								      port_wait);
+					}
+				}
+			}
+		}
+
+		if (dd->ipath_layer.l_intr) {
+			if (dd->ipath_layer.l_intr(t,
+				IPATH_LAYER_INT_SEND_CONTINUE)) {
+				atomic_set_mask(INFINIPATH_S_PIOINTBUFAVAIL,
+						&dd->ipath_sendctrl);
+				ipath_kput_kreg(t, kr_sendctrl,
+						dd->ipath_sendctrl);
+			} 
+		}
+
+		if (dd->verbs_layer.l_piobufavail) {
+			if (!dd->verbs_layer.l_piobufavail(t)) {
+				atomic_set_mask(INFINIPATH_S_PIOINTBUFAVAIL,
+						&dd->ipath_sendctrl);
+				ipath_kput_kreg(t, kr_sendctrl,
+						dd->ipath_sendctrl);
+			}
+		}
+	}
+
+	/*
+	 * we check for both transition from empty to non-empty, and urgent
+	 * packets (those with the interrupt bit set in the header)
+	 */
+
+	if (istat & ((infinipath_i_rcvavail_mask << INFINIPATH_I_RCVAVAIL_SHIFT)
+		     | (infinipath_i_rcvurg_mask << INFINIPATH_I_RCVURG_SHIFT))) {
+		uint64_t portr;
+		int i;
+		uint32_t rcvdint = 0;
+
+		portr = ((istat >> INFINIPATH_I_RCVAVAIL_SHIFT) &
+			 infinipath_i_rcvavail_mask)
+		    | ((istat >> INFINIPATH_I_RCVURG_SHIFT) &
+		       infinipath_i_rcvurg_mask);
+		for (i = 0; i < dd->ipath_cfgports; i++) {
+			if (portr & (1 << i) && dd->ipath_pd[i]) {
+				if (i == 0)
+					ipath_kreceive(t);
+				else if (dd->ipath_pd[i]->
+					 port_flag & IPATH_PORT_WAITING_RCV) {
+					atomic_clear_mask
+					    (IPATH_PORT_WAITING_RCV,
+					     &dd->ipath_pd[i]->port_flag);
+					wake_up_interruptible(&dd->ipath_pd[i]->
+							      port_wait);
+					rcvdint |= 1U << i;
+				}
+			}
+		}
+		if (rcvdint) {
+			/*
+			 * only want to take one interrupt, so turn off
+			 * the rcv interrupt for all the ports that we
+			 * did the wakeup on (but never for kernel port)
+			 */
+			atomic_clear_mask(rcvdint <<
+					  INFINIPATH_R_INTRAVAIL_SHIFT,
+					  &dd->ipath_rcvctrl);
+			ipath_kput_kreg(t, kr_rcvctrl, dd->ipath_rcvctrl);
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void ipath_decode_err(char *buf, size_t blen, uint64_t err)
+{
+	*buf = '\0';
+	if (err & INFINIPATH_E_RHDRLEN)
+		strlcat(buf, "rhdrlen ", blen);
+	if (err & INFINIPATH_E_RBADTID)
+		strlcat(buf, "rbadtid ", blen);
+	if (err & INFINIPATH_E_RBADVERSION)
+		strlcat(buf, "rbadversion ", blen);
+	if (err & INFINIPATH_E_RHDR)
+		strlcat(buf, "rhdr ", blen);
+	if (err & INFINIPATH_E_RLONGPKTLEN)
+		strlcat(buf, "rlongpktlen ", blen);
+	if (err & INFINIPATH_E_RSHORTPKTLEN)
+		strlcat(buf, "rshortpktlen ", blen);
+	if (err & INFINIPATH_E_RMAXPKTLEN)
+		strlcat(buf, "rmaxpktlen ", blen);
+	if (err & INFINIPATH_E_RMINPKTLEN)
+		strlcat(buf, "rminpktlen ", blen);
+	if (err & INFINIPATH_E_RFORMATERR)
+		strlcat(buf, "rformaterr ", blen);
+	if (err & INFINIPATH_E_RUNSUPVL)
+		strlcat(buf, "runsupvl ", blen);
+	if (err & INFINIPATH_E_RUNEXPCHAR)
+		strlcat(buf, "runexpchar ", blen);
+	if (err & INFINIPATH_E_RIBFLOW)
+		strlcat(buf, "ribflow ", blen);
+	if (err & INFINIPATH_E_REBP)
+		strlcat(buf, "EBP ", blen);
+	if (err & INFINIPATH_E_SUNDERRUN)
+		strlcat(buf, "sunderrun ", blen);
+	if (err & INFINIPATH_E_SPIOARMLAUNCH)
+		strlcat(buf, "spioarmlaunch ", blen);
+	if (err & INFINIPATH_E_SUNEXPERRPKTNUM)
+		strlcat(buf, "sunexperrpktnum ", blen);
+	if (err & INFINIPATH_E_SDROPPEDDATAPKT)
+		strlcat(buf, "sdroppeddatapkt ", blen);
+	if (err & INFINIPATH_E_SDROPPEDSMPPKT)
+		strlcat(buf, "sdroppedsmppkt ", blen);
+	if (err & INFINIPATH_E_SMAXPKTLEN)
+		strlcat(buf, "smaxpktlen ", blen);
+	if (err & INFINIPATH_E_SMINPKTLEN)
+		strlcat(buf, "sminpktlen ", blen);
+	if (err & INFINIPATH_E_SUNSUPVL)
+		strlcat(buf, "sunsupVL ", blen);
+	if (err & INFINIPATH_E_SPKTLEN)
+		strlcat(buf, "spktlen ", blen);
+	if (err & INFINIPATH_E_INVALIDADDR)
+		strlcat(buf, "invalidaddr ", blen);
+	if (err & INFINIPATH_E_RICRC)
+		strlcat(buf, "CRC ", blen);
+	if (err & INFINIPATH_E_RVCRC)
+		strlcat(buf, "VCRC ", blen);
+	if (err & INFINIPATH_E_RRCVEGRFULL)
+		strlcat(buf, "rcvegrfull ", blen);
+	if (err & INFINIPATH_E_RRCVHDRFULL)
+		strlcat(buf, "rcvhdrfull ", blen);
+	if (err & INFINIPATH_E_IBSTATUSCHANGED)
+		strlcat(buf, "ibcstatuschg ", blen);
+	if (err & INFINIPATH_E_RIBLOSTLINK)
+		strlcat(buf, "riblostlink ", blen);
+	if (err & INFINIPATH_E_HARDWARE)
+		strlcat(buf, "hardware ", blen);
+	if (err & INFINIPATH_E_RESET)
+		strlcat(buf, "reset ", blen);
+}
+
+/* decode RHF errors; only used one place now, may want more later */
+static void get_rhf_errstring(uint32_t err, char *msg, size_t len)
+{
+	/* if no errors, and so don't need to check what's first */
+	*msg = '\0';
+
+	if (err & INFINIPATH_RHF_H_ICRCERR)
+		strlcat(msg, "icrcerr ", len);
+	if (err & INFINIPATH_RHF_H_VCRCERR)
+		strlcat(msg, "vcrcerr ", len);
+	if (err & INFINIPATH_RHF_H_PARITYERR)
+		strlcat(msg, "parityerr ", len);
+	if (err & INFINIPATH_RHF_H_LENERR)
+		strlcat(msg, "lenerr ", len);
+	if (err & INFINIPATH_RHF_H_MTUERR)
+		strlcat(msg, "mtuerr ", len);
+	if (err & INFINIPATH_RHF_H_IHDRERR)
+		/* infinipath hdr checksum error */
+		strlcat(msg, "ipathhdrerr ", len);
+	if (err & INFINIPATH_RHF_H_TIDERR)
+		strlcat(msg, "tiderr ", len);
+	if (err & INFINIPATH_RHF_H_MKERR)
+		/* bad port, offset, etc. */
+		strlcat(msg, "invalid ipathhdr ", len);
+	if (err & INFINIPATH_RHF_H_IBERR)
+		strlcat(msg, "iberr ", len);
+	if (err & INFINIPATH_RHF_L_SWA)
+		strlcat(msg, "swA ", len);
+	if (err & INFINIPATH_RHF_L_SWB)
+		strlcat(msg, "swB ", len);
+}
+
+static void ipath_handle_errors(const ipath_type t, uint64_t errs)
+{
+	char msg[512];
+	uint32_t piobcnt;
+	uint64_t sbuf[4], ignore_this_time = 0;
+	int i;
+	int chkerrpkts = 0, noprint = 0;
+	cycles_t nc;
+	static cycles_t nextmsg_time;
+	static unsigned nmsgs, supp_msgs;
+	ipath_devdata *dd = &devdata[t];
+
+#define E_SUM_PKTERRS (INFINIPATH_E_RHDRLEN | INFINIPATH_E_RBADTID \
+     | INFINIPATH_E_RBADVERSION \
+     | INFINIPATH_E_RHDR | INFINIPATH_E_RLONGPKTLEN | INFINIPATH_E_RSHORTPKTLEN \
+     | INFINIPATH_E_RMAXPKTLEN | INFINIPATH_E_RMINPKTLEN \
+     | INFINIPATH_E_RFORMATERR | INFINIPATH_E_RUNSUPVL | INFINIPATH_E_RUNEXPCHAR \
+     | INFINIPATH_E_REBP)
+
+#define E_SUM_ERRS ( INFINIPATH_E_SPIOARMLAUNCH \
+    | INFINIPATH_E_SUNEXPERRPKTNUM | INFINIPATH_E_SDROPPEDDATAPKT \
+    | INFINIPATH_E_SDROPPEDSMPPKT | INFINIPATH_E_SMAXPKTLEN \
+    | INFINIPATH_E_SUNSUPVL | INFINIPATH_E_SMINPKTLEN | INFINIPATH_E_SPKTLEN \
+    | INFINIPATH_E_INVALIDADDR)
+
+	/*
+	 * throttle back "fast" messages to no more than 10 per 5 seconds
+	 * (1.4-2GHz clock).  This isn't perfect, but it's a reasonable
+	 * heuristic
+	 * If we get more than 10, give a 5x longer delay
+	 */
+	nc = get_cycles();
+	if (nmsgs > 10) {
+		if (nc < nextmsg_time) {
+			noprint = 1;
+			if (!supp_msgs++)
+				nextmsg_time = nc + 50000000000ULL;
+		} else if (supp_msgs) {
+			/*
+			 * Print the message unless it's ibc status
+			 * change only, which happens so often we never
+			 * want to count it.
+			 */
+			if (dd->ipath_lasterror & ~INFINIPATH_E_IBSTATUSCHANGED) {
+				ipath_decode_err(msg, sizeof msg,
+						 dd->
+						 ipath_lasterror &
+						 ~INFINIPATH_E_IBSTATUSCHANGED);
+				if (dd->
+				    ipath_lasterror & ~(INFINIPATH_E_RRCVEGRFULL
+							|
+							INFINIPATH_E_RRCVHDRFULL))
+					_IPATH_UNIT_ERROR(t,
+							  "Suppressed %u messages for fast-repeating errors (%s) (%llx)\n",
+							  supp_msgs, msg,
+							  dd->ipath_lasterror);
+				else {
+					/*
+					 * rcvegrfull and rcvhdrqfull are
+					 * "normal", for some types of
+					 * processes (mostly benchmarks)
+					 * that send huge numbers of
+					 * messages, while not processing
+					 * them.  So only complain about
+					 * these at debug level.
+					 */
+					_IPATH_DBG
+					    ("Suppressed %u messages for %s\n",
+					     supp_msgs, msg);
+				}
+			}
+			supp_msgs = 0;
+			nmsgs = 0;
+		}
+	} else if (!nmsgs++ || nc > nextmsg_time)	/* start timer */
+		nextmsg_time = nc + 10000000000ULL;
+
+	/*
+	 * don't report errors that are masked (includes those always
+	 * ignored)
+	 */
+	errs &= ~dd->ipath_maskederrs;
+
+	/* do these first, they are most important */
+	if (errs & INFINIPATH_E_HARDWARE) {
+		/* reuse same msg buf */
+		ipath_handle_hwerrors(t, msg, sizeof msg);
+	}
+
+	if (!noprint && (errs & ~infinipath_e_bitsextant))
+		_IPATH_UNIT_ERROR(t,
+				  "error interrupt with unknown errors %llx set\n",
+				  errs & ~infinipath_e_bitsextant);
+
+	if (errs & E_SUM_ERRS) {
+		/* if possible that sendbuffererror could be valid */
+		piobcnt = dd->ipath_piobcnt;
+		/* read these before writing errorclear */
+		sbuf[0] = ipath_kget_kreg64(t, kr_sendbuffererror);
+		sbuf[1] = ipath_kget_kreg64(t, kr_sendbuffererror + 1);
+		if (piobcnt > 128) {
+			sbuf[2] = ipath_kget_kreg64(t, kr_sendbuffererror + 2);
+			sbuf[3] = ipath_kget_kreg64(t, kr_sendbuffererror + 3);
+		}
+
+		if (sbuf[0] || sbuf[1]
+		    || (piobcnt > 128 && (sbuf[2] || sbuf[3]))) {
+			_IPATH_PDBG("SendbufErrs %llx %llx ", sbuf[0], sbuf[1]);
+			if (infinipath_debug & __IPATH_PKTDBG && piobcnt > 128)
+				printk("%llx %llx ", sbuf[2], sbuf[3]);
+			for (i = 0; i < piobcnt; i++) {
+				if (test_bit(i, sbuf)) {
+					uint32_t sendctrl;
+					if (infinipath_debug & __IPATH_PKTDBG)
+						printk("%u ", i);
+					sendctrl =
+					    dd->
+					    ipath_sendctrl | INFINIPATH_S_DISARM
+					    | (i <<
+					       INFINIPATH_S_DISARMPIOBUF_SHIFT);
+					ipath_kput_kreg(t, kr_sendctrl,
+							sendctrl);
+				}
+			}
+			if (infinipath_debug & __IPATH_PKTDBG)
+				printk("\n");
+		}
+		if ((errs &
+		     (INFINIPATH_E_SDROPPEDDATAPKT | INFINIPATH_E_SDROPPEDSMPPKT
+		      | INFINIPATH_E_SMINPKTLEN))
+		    && !(dd->ipath_flags & IPATH_LINKACTIVE)) {
+			/*
+			 * This can happen when SMA is trying to bring
+			 * the link up, but the IB link changes state
+			 * at the "wrong" time.  The IB logic then
+			 * complains that the packet isn't valid.
+			 * We don't want to confuse people, so we just
+			 * don't print them, except at debug
+			 */
+			_IPATH_DBG
+			    ("Ignoring pktsend errors %llx, because not yet active\n",
+			     errs);
+			ignore_this_time |=
+			    INFINIPATH_E_SDROPPEDDATAPKT |
+			    INFINIPATH_E_SDROPPEDSMPPKT |
+			    INFINIPATH_E_SMINPKTLEN;
+		}
+	}
+
+	if (supp_msgs == 250000) {
+		/*
+		 * It's not entirely reasonable assuming that the errors
+		 * set in the last clear period are all responsible for
+		 * the problem, but the alternative is to assume it's the only
+		 * ones on this particular interrupt, which also isn't great
+		 */
+		dd->ipath_maskederrs |= dd->ipath_lasterror | errs;
+		ipath_kput_kreg(t, kr_errormask, ~dd->ipath_maskederrs);
+		ipath_decode_err(msg, sizeof msg,
+				 (dd->ipath_maskederrs & ~dd->
+				  ipath_ignorederrs));
+
+		if ((dd->ipath_maskederrs & ~dd->ipath_ignorederrs)
+		    & ~(INFINIPATH_E_RRCVEGRFULL | INFINIPATH_E_RRCVHDRFULL))
+			_IPATH_UNIT_ERROR(t,
+					  "Disabling error(s) %llx because occuring too frequently (%s)\n",
+					  (dd->ipath_maskederrs & ~dd->
+					   ipath_ignorederrs), msg);
+		else {
+			/*
+			 * rcvegrfull and rcvhdrqfull are "normal",
+			 * for some types of processes (mostly benchmarks)
+			 * that send huge numbers of messages, while not
+			 * processing them.  So only complain about
+			 * these at debug level.
+			 */
+			_IPATH_DBG
+			    ("Disabling frequent queue full errors (%s)\n",
+			     msg);
+		}
+
+		/*
+		 * re-enable the masked errors after around 3 minutes.
+		 * in ipath_get_faststats().  If we have a series of
+		 * fast repeating but different errors, the interval will keep
+		 * stretching out, but that's OK, as that's pretty catastrophic.
+		 */
+		dd->ipath_unmasktime = nc + 400000000000ULL;
+	}
+
+	ipath_kput_kreg(t, kr_errorclear, errs);
+	if (ignore_this_time)
+		errs &= ~ignore_this_time;
+	if (errs & ~dd->ipath_lasterror) {
+		errs &= ~dd->ipath_lasterror;
+		/* never suppress duplicate hwerrors or ibstatuschange */
+		dd->ipath_lasterror |= errs &
+		    ~(INFINIPATH_E_HARDWARE | INFINIPATH_E_IBSTATUSCHANGED);
+	}
+	if (!errs)
+		return;
+
+	if (!noprint)
+		/* the ones we mask off are handled specially below or above */
+		ipath_decode_err(msg, sizeof msg,
+				 errs & ~(INFINIPATH_E_IBSTATUSCHANGED |
+					  INFINIPATH_E_RRCVEGRFULL |
+					  INFINIPATH_E_RRCVHDRFULL |
+					  INFINIPATH_E_HARDWARE));
+	else
+		/* so we don't need if(!noprint) at strlcat's below */
+		*msg = 0;
+
+	if (errs & E_SUM_PKTERRS) {
+		ipath_stats.sps_pkterrs++;
+		chkerrpkts = 1;
+	}
+	if (errs & E_SUM_ERRS)
+		ipath_stats.sps_errs++;
+
+	if (errs & (INFINIPATH_E_RICRC | INFINIPATH_E_RVCRC)) {
+		ipath_stats.sps_crcerrs++;
+		chkerrpkts = 1;
+	}
+
+	/*
+	 * We don't want to print these two as they happen, or we can make
+	 * the situation even worse, because it takes so long to print messages.
+	 * to serial consoles.  kernel ports get printed from fast_stats, no
+	 * more than every 5 seconds, user ports get printed on close
+	 */
+	if (errs & INFINIPATH_E_RRCVHDRFULL) {
+		int any;
+		uint32_t hd, tl;
+		ipath_stats.sps_hdrqfull++;
+		for (any = i = 0; i < dd->ipath_cfgports; i++) {
+			if (i == 0) {
+				hd = dd->ipath_port0head;
+				tl = *dd->ipath_hdrqtailptr;
+			} else if (dd->ipath_pd[i] &&
+				   dd->ipath_pd[i]->port_rcvhdrtail_kvaddr) {
+				/*
+				 * don't report same point multiple times,
+				 * except kernel
+				 */
+				tl = (uint32_t) *
+				    dd->ipath_pd[i]->port_rcvhdrtail_kvaddr;
+				if (tl == dd->ipath_lastrcvhdrqtails[i])
+					continue;
+				hd = ipath_kget_ureg32(t, ur_rcvhdrhead, i);
+			} else
+				continue;
+			if (hd == (tl + 1) || (!hd && tl == dd->ipath_hdrqlast)) {
+				dd->ipath_lastrcvhdrqtails[i] = tl;
+				dd->ipath_pd[i]->port_hdrqfull++;
+				if (i == 0)
+					chkerrpkts = 1;
+			}
+		}
+	}
+	if (errs & INFINIPATH_E_RRCVEGRFULL) {
+		/*
+		 * since this is of less importance and not likely to
+		 * happen without also getting hdrfull, only count
+		 * occurrences; don't check each port (or even the kernel
+		 * vs user)
+		 */
+		ipath_stats.sps_etidfull++;
+		if (dd->ipath_port0head != *dd->ipath_hdrqtailptr)
+			chkerrpkts = 1;
+	}
+
+	/*
+	 * do this before IBSTATUSCHANGED, in case both bits set in a single
+	 * interrupt; we want the STATUSCHANGE to "win", so we do our 
+	 * internal copy of state machine correctly
+	 */
+	if (errs & INFINIPATH_E_RIBLOSTLINK) {
+		/* force through block below */
+		errs |= INFINIPATH_E_IBSTATUSCHANGED;
+		ipath_stats.sps_iblink++;
+		dd->ipath_flags |= IPATH_LINKDOWN;
+		dd->ipath_flags &= ~(IPATH_LINKUNK | IPATH_LINKINIT
+				     | IPATH_LINKARMED | IPATH_LINKACTIVE);
+		if (!noprint)
+			_IPATH_DBG("Lost link, link now down (%s)\n",
+				   ipath_ibcstatus_str[ipath_kget_kreg64
+						       (t,
+							kr_ibcstatus) & 0xf]);
+	}
+
+	if ((errs & INFINIPATH_E_IBSTATUSCHANGED) && (!ipath_diags_enabled)) {
+		uint64_t val;
+		uint32_t ltstate;
+
+		val = ipath_kget_kreg64(t, kr_ibcstatus);
+		ltstate = val & 0xff;
+		if(ltstate == 0x11 || ltstate == 0x21 || ltstate == 0x31)
+			_IPATH_DBG("Link state changed unit %u to 0x%x, last was 0x%llx\n",
+				t, ltstate, dd->ipath_lastibcstat);
+		else {
+			ltstate = dd->ipath_lastibcstat & 0xff;
+			if(ltstate == 0x11 || ltstate == 0x21 || ltstate == 0x31)
+				_IPATH_DBG("Link state unit %u changed to down state 0x%llx, last was 0x%llx\n",
+					t, val, dd->ipath_lastibcstat);
+			else
+				_IPATH_VDBG("Link state unit %u changed to 0x%llx from one of down states\n",
+					t, val);
+		}
+		ltstate = (val >> INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT) &
+		    INFINIPATH_IBCS_LINKTRAININGSTATE_MASK;
+
+		if (ltstate == 2 || ltstate == 3) {
+			uint32_t last_ltstate;
+
+			/*
+			 * ignore cycling back and forth from states 2 to 3
+			 * while waiting for other end of link to come up
+			 * except that if it keeps happening, we switch between
+			 * linkinitstate SLEEP and POLL.  While we cycle
+			 * back and forth between them, we aren't seeing
+			 * any other device, either no cable plugged in,
+			 * other device powered off, other device is
+			 * switch that hasn't yet polled us, etc.
+			 */
+			last_ltstate = (dd->ipath_lastibcstat >>
+					INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT)
+			    & INFINIPATH_IBCS_LINKTRAININGSTATE_MASK;
+			if (last_ltstate == 2 || last_ltstate == 3) {
+				if (++dd->ipath_ibpollcnt > 4) {
+					uint64_t ibc;
+					dd->ipath_flags |=
+					    IPATH_LINK_SLEEPING | IPATH_NOCABLE;
+					*dd->ipath_statusp |=
+					    IPATH_STATUS_IB_NOCABLE;
+					_IPATH_VDBG
+					    ("linkinitcmd POLL, move to SLEEP\n");
+					ibc = dd->ipath_ibcctrl;
+					ibc |= INFINIPATH_IBCC_LINKINITCMD_SLEEP
+					    <<
+					    INFINIPATH_IBCC_LINKINITCMD_SHIFT;
+					/*
+					 * don't put linkinitcmd in
+					 * ipath_ibcctrl, want that to
+					 * stay a NOP
+					 */
+					ipath_kput_kreg(t, kr_ibcctrl, ibc);
+					dd->ipath_ibpollcnt = 0;
+				}
+				goto skip_ibchange;
+			}
+		}
+		/* some state other than 2 or 3 */
+		dd->ipath_ibpollcnt = 0;
+		ipath_stats.sps_iblink++;
+		/*
+		 * Note:  We try to match the Mellanox HCA LED behavior
+		 * as best we can.  That changed around Oct 2003.
+		 * Green indicates link state (something is plugged in,
+		 * and we can train).  Amber indicates the link is
+		 * logically up (ACTIVE).  Mellanox further blinks the
+		 * amber LED to indicate data packet activity, but we
+		 * have no hardware support for that, so it would require
+		 * waking up every 10-20 msecs and checking the counters
+		 * on the chip, and then turning the LED off if
+		 * appropriate.  That's visible overhead, so not something
+		 * we will do.
+		 */
+		if (ltstate != 1 || ((dd->ipath_lastibcstat & 0x30) == 0x30 &&
+				     (val & 0x30) != 0x30)) {
+			dd->ipath_flags |= IPATH_LINKDOWN;
+			dd->ipath_flags &= ~(IPATH_LINKUNK | IPATH_LINKINIT
+					     | IPATH_LINKACTIVE |
+					     IPATH_LINKARMED);
+			*dd->ipath_statusp &= ~IPATH_STATUS_IB_READY;
+			if (!noprint) {
+				if ((dd->ipath_lastibcstat & 0x30) == 0x30)
+					/* if from up to down be more vocal */
+					_IPATH_DBG("Link unit %u is now down (%s)\n",
+						   t, ipath_ibcstatus_str
+						   [ltstate]);
+				else
+					_IPATH_VDBG("Link unit %u is down (%s)\n",
+						    t, ipath_ibcstatus_str
+						    [ltstate]);
+			}
+
+			if (val & 0x30) {
+				/* leave just green on, 0x11 and 0x21 */
+				dd->ipath_extctrl &=
+				    ~INFINIPATH_EXTC_LEDPRIPORTYELLOWON;
+				dd->ipath_extctrl |=
+				    INFINIPATH_EXTC_LEDPRIPORTGREENON;
+			} else	/* not up at all, so turn the leds off */
+				dd->ipath_extctrl &=
+				    ~(INFINIPATH_EXTC_LEDPRIPORTGREENON |
+				      INFINIPATH_EXTC_LEDPRIPORTYELLOWON);
+			ipath_kput_kreg(t, kr_extctrl,
+					(uint64_t) dd->ipath_extctrl);
+			if (ltstate == 1
+			    && (dd->
+				ipath_flags & (IPATH_LINK_TOARMED |
+					       IPATH_LINK_TOACTIVE))) {
+				ipath_set_ib_lstate(t,
+						    INFINIPATH_IBCC_LINKCMD_INIT);
+			}
+		} else if ((val & 0x31) == 0x31) {
+			if (!noprint)
+				_IPATH_DBG("Link unit %u is now in active state\n", t);
+			dd->ipath_flags |= IPATH_LINKACTIVE;
+			dd->ipath_flags &=
+			    ~(IPATH_LINKUNK | IPATH_LINKINIT | IPATH_LINKDOWN |
+			      IPATH_LINKARMED | IPATH_NOCABLE |
+			      IPATH_LINK_TOACTIVE | IPATH_LINK_SLEEPING);
+			*dd->ipath_statusp &= ~IPATH_STATUS_IB_NOCABLE;
+			*dd->ipath_statusp |=
+			    IPATH_STATUS_IB_READY | IPATH_STATUS_IB_CONF;
+			/* set the externally visible LEDs to indicate state */
+			dd->ipath_extctrl |= INFINIPATH_EXTC_LEDPRIPORTGREENON
+			    | INFINIPATH_EXTC_LEDPRIPORTYELLOWON;
+			ipath_kput_kreg(t, kr_extctrl,
+					(uint64_t) dd->ipath_extctrl);
+
+			/*
+			 * since we are now active, set the linkinitcmd
+			 * to NOP (0) it was probably either POLL or SLEEP
+			 */
+			dd->ipath_ibcctrl &=
+			    ~(INFINIPATH_IBCC_LINKINITCMD_MASK <<
+			      INFINIPATH_IBCC_LINKINITCMD_SHIFT);
+			ipath_kput_kreg(t, kr_ibcctrl, dd->ipath_ibcctrl);
+
+			if (devdata[t].ipath_layer.l_intr)
+				devdata[t].ipath_layer.l_intr(t,
+							      IPATH_LAYER_INT_IF_UP);
+		} else if ((val & 0x31) == 0x11) {
+			/*
+			 * set set INIT and DOWN.  Down is checked by
+			 * most of the other code, but INIT is useful
+			 * to know in a few places.
+			 */
+			dd->ipath_flags |= IPATH_LINKINIT | IPATH_LINKDOWN;
+			dd->ipath_flags &=
+			    ~(IPATH_LINKUNK | IPATH_LINKACTIVE | IPATH_LINKARMED
+			      | IPATH_NOCABLE | IPATH_LINK_SLEEPING);
+			*dd->ipath_statusp &= ~(IPATH_STATUS_IB_NOCABLE
+				| IPATH_STATUS_IB_READY);
+
+			/* set the externally visible LEDs to indicate state */
+			dd->ipath_extctrl &=
+			    ~INFINIPATH_EXTC_LEDPRIPORTYELLOWON;
+			dd->ipath_extctrl |= INFINIPATH_EXTC_LEDPRIPORTGREENON;
+			ipath_kput_kreg(t, kr_extctrl,
+					(uint64_t) dd->ipath_extctrl);
+			if (dd->
+			    ipath_flags & (IPATH_LINK_TOARMED |
+					   IPATH_LINK_TOACTIVE)) {
+				/*
+				 * if we got here while trying to bring
+				 * the link up, try again, but only once more!
+				 */
+				ipath_set_ib_lstate(t,
+						    INFINIPATH_IBCC_LINKCMD_ARMED);
+				dd->ipath_flags &=
+				    ~(IPATH_LINK_TOARMED | IPATH_LINK_TOACTIVE);
+			}
+		} else if ((val & 0x31) == 0x21) {
+			dd->ipath_flags |= IPATH_LINKARMED;
+			dd->ipath_flags &=
+			    ~(IPATH_LINKUNK | IPATH_LINKDOWN | IPATH_LINKINIT |
+			      IPATH_LINKACTIVE | IPATH_NOCABLE |
+			      IPATH_LINK_TOARMED | IPATH_LINK_SLEEPING);
+			*dd->ipath_statusp &= ~(IPATH_STATUS_IB_NOCABLE
+				| IPATH_STATUS_IB_READY);
+			/*
+			 * set the externally visible LEDs to indicate
+			 * state (same as 0x11)
+			 */
+			dd->ipath_extctrl &=
+			    ~INFINIPATH_EXTC_LEDPRIPORTYELLOWON;
+			dd->ipath_extctrl |= INFINIPATH_EXTC_LEDPRIPORTGREENON;
+			ipath_kput_kreg(t, kr_extctrl,
+					(uint64_t) dd->ipath_extctrl);
+			if (dd->ipath_flags & IPATH_LINK_TOACTIVE) {
+				/*
+				 * if we got here while trying to bring
+				 * the link up, try again, but only once more!
+				 */
+				ipath_set_ib_lstate(t,
+						    INFINIPATH_IBCC_LINKCMD_ACTIVE);
+				dd->ipath_flags &= ~IPATH_LINK_TOACTIVE;
+			}
+		} else {
+			if (dd->
+			    ipath_flags & (IPATH_LINK_TOARMED |
+					   IPATH_LINK_TOACTIVE))
+				ipath_set_ib_lstate(t,
+						    INFINIPATH_IBCC_LINKCMD_INIT);
+			else if (!noprint)
+				_IPATH_DBG("IBstatuschange unit %u: %s\n",
+					  t, ipath_ibcstatus_str[ltstate]);
+		}
+		dd->ipath_lastibcstat = val;
+	}
+
+skip_ibchange:
+
+	if (errs & INFINIPATH_E_RESET) {
+		if (!noprint)
+			_IPATH_UNIT_ERROR(t,
+					  "Got reset, requires re-initialization (unload and reload driver)\n");
+		dd->ipath_flags &= ~IPATH_INITTED;	/* needs re-init */
+		/* mark as having had error */
+		*dd->ipath_statusp |= IPATH_STATUS_HWERROR;
+		*dd->ipath_statusp &= ~IPATH_STATUS_IB_CONF;
+	}
+
+	if (!noprint && *msg)
+		_IPATH_UNIT_ERROR(t, "%s error\n", msg);
+	if (dd->ipath_sma_state_wanted & dd->ipath_flags) {
+		_IPATH_VDBG("sma wanted state %x, iflags now %x, waking\n",
+			    dd->ipath_sma_state_wanted, dd->ipath_flags);
+		wake_up_interruptible(&ipath_sma_state_wait);
+	}
+
+	if (chkerrpkts)
+		/* process possible error packets in hdrq */
+		ipath_kreceive(t);
+}
+
+/* must only be called if ipath_pd[port] is known to be allocated */
+static __inline__ void *ipath_get_egrbuf(const ipath_type t, uint32_t bufnum,
+					 int err)
+{
+	return devdata[t].ipath_port0_skbs ?
+	    (void *)devdata[t].ipath_port0_skbs[bufnum]->data : NULL;
+
+#ifdef _USE_FOR_DEBUGGING_ONLY
+	/*
+	 * want routine to be inlined and fast this is here so if we do ports
+	 * other than 0, I don't have to rewrite the code, since it's slightly
+	 * complicated
+	 */
+	if (port != 1) {
+		void *chunkbase;
+		/*
+		 * This calculation takes about 50 cycles.  Could do
+		 * what I did for protocol code, and have an array of
+		 * addresses, getting it down to just a few cycles per
+		 * lookup, at the cost of 16KB of memory.
+		 */
+		if (!devdata[t].ipath_pd[port]->port_rcvegrbuf_virt)
+			return NULL;
+		chunkbase = devdata[t].ipath_pd[port]->port_rcvegrbuf_virt
+		    [bufnum /
+		     devdata[t].ipath_pd[port]->port_rcvegrbufs_perchunk];
+		return (void *)(chunkbase +
+				(bufnum %
+				 devdata[t].ipath_pd[port]->
+				 port_rcvegrbufs_perchunk)
+				* devdata[t].ipath_rcvegrbufsize);
+	}
+#endif
+}
+
+/* receive an sma packet.  Separate for better overall optimization */
+static void ipath_rcv_sma(const ipath_type t, uint32_t tlen,
+			  uint64_t * rc, void *ebuf)
+{
+	int sindex, slen, elen;
+	void *smbuf;
+	uint8_t pad, *bthbytes;
+
+	ipath_stats.sps_sma_rpkts++;	/* another SMA packet received */
+
+	bthbytes = (uint8_t *) ((ips_message_header_typ *) & rc[1])->bth;
+
+	pad = (bthbytes[1] >> 4) & 3;
+	elen = tlen - (IPATH_SMA_HDRSZ + pad + (uint32_t) sizeof(uint32_t));
+	if (elen > (SMA_MAX_PKTSZ - IPATH_SMA_HDRSZ))
+		elen = SMA_MAX_PKTSZ - IPATH_SMA_HDRSZ;
+
+	spin_lock_irq(&ipath_sma_lock);
+	sindex = ipath_sma_next;
+	smbuf = ipath_sma_data[sindex].buf;
+	ipath_sma_data[sindex].unit = t;
+	slen = ipath_sma_data[ipath_sma_next].len;
+	memcpy(smbuf, &rc[1], IPATH_SMA_HDRSZ);
+	memcpy(smbuf + IPATH_SMA_HDRSZ, ebuf, elen);
+	if (slen) {
+		/*
+		 * overwriting a yet unread old one (buffer wrap), have to
+		 * advance ipath_sma_first to next oldest
+		 */
+
+		/* count OK packets that we drop */
+		ipath_stats.sps_krdrops++;
+		if (++ipath_sma_first >= IPATH_NUM_SMAPKTS)
+			ipath_sma_first = 0;
+	}
+	slen = ipath_sma_data[sindex].len = elen + IPATH_SMA_HDRSZ;
+	if (++ipath_sma_next >= IPATH_NUM_SMAPKTS)
+		ipath_sma_next = 0;
+	spin_unlock_irq(&ipath_sma_lock);
+}
+
+/*
+ * receive a packet for the layered (ethernet) driver.
+ * Separate routine for better overall optimization
+ */
+static void ipath_rcv_layer(const ipath_type t, uint32_t etail,
+			    uint32_t tlen, ether_header_typ * hdr)
+{
+	uint32_t elen;
+	uint8_t pad, *bthbytes;
+	struct sk_buff *skb;
+	struct sk_buff *nskb;
+	ipath_devdata *dd = &devdata[t];
+	ipath_portdata *pd;
+	unsigned long pa, pent;
+	uint64_t *egrbase;
+	uint64_t lenvalid;	/* in words */
+
+	if (dd->ipath_port0_skbs && hdr->sub_opcode == OPCODE_ENCAP) {
+		/*
+		 * Allocate a new sk_buff to replace the one we give
+		 * to the network stack.
+		 */
+		if (!(nskb = dev_alloc_skb(dd->ipath_ibmaxlen + 4))) {
+			/* count OK packets that we drop */
+			ipath_stats.sps_krdrops++;
+			return;
+		}
+
+		bthbytes = (uint8_t *) hdr->bth;
+		pad = (bthbytes[1] >> 4) & 3;
+		/* +CRC32 */
+		elen = tlen - (sizeof(*hdr) + pad + sizeof(uint32_t));
+
+		skb_reserve(nskb, 4);
+
+		skb = dd->ipath_port0_skbs[etail];
+		dd->ipath_port0_skbs[etail] = nskb;
+		skb_put(skb, elen);
+
+		pd = dd->ipath_pd[0];
+		lenvalid = (dd->ipath_ibmaxlen - pd->port_egrskip) >> 2;
+		lenvalid <<= INFINIPATH_RT_BUFSIZE_SHIFT;
+		lenvalid |= INFINIPATH_RT_VALID;
+		pa = virt_to_phys(nskb->data);
+		pa += pd->port_egrskip;
+		pent = (pa & INFINIPATH_RT_ADDR_MASK) | lenvalid;
+		/* This is simplified for port 0 */
+		egrbase = (uint64_t *) ((char *)(dd->ipath_kregbase) +
+					dd->ipath_rcvegrbase);
+		ipath_kput_memq(t, &egrbase[etail], pent);
+
+		dd->ipath_layer.l_rcv(t, hdr, skb);
+
+		/* another ether packet received */
+		ipath_stats.sps_ether_rpkts++;
+	} else if (hdr->sub_opcode == OPCODE_LID_ARP) {
+		if (dd->ipath_layer.l_rcv_lid)
+			dd->ipath_layer.l_rcv_lid(t, hdr);
+	}
+
+}
+
+/* called from interrupt handler for errors or receive interrupt */
+void ipath_kreceive(const ipath_type t)
+{
+	uint64_t *rc;
+	void *ebuf;
+	ipath_devdata *dd = &devdata[t];
+	const uint32_t rsize = dd->ipath_rcvhdrentsize;	/* words */
+	const uint32_t maxcnt = dd->ipath_rcvhdrcnt * rsize;	/* in words */
+	uint32_t etail = ~0U, l, hdrqtail, sma_this_time = 0;
+	ips_message_header_typ *hdr;
+	uint32_t eflags, i, etype, tlen, pkttot=0;
+	static uint64_t totcalls; /* stats, may eventually remove */
+	char emsg[128];
+
+	if (!dd->ipath_hdrqtailptr) {
+		_IPATH_UNIT_ERROR(t,
+				  "hdrqtailptr not set, can't do receives\n");
+		return;
+	}
+
+	if (test_and_set_bit(0, &dd->ipath_rcv_pending)) {
+		/* There is already a thread processing this queue. */
+		return;
+	}
+
+	if (dd->ipath_port0head == *dd->ipath_hdrqtailptr)
+		goto done;
+
+gotmore:
+	/*
+	 * read only once at start.  If in flood situation, this helps
+	 * performance slightly.  If more arrive while we are processing,
+	 * we'll come back here and do them
+	 */
+	hdrqtail = *dd->ipath_hdrqtailptr;
+
+	for (i = 0, l = dd->ipath_port0head; l != hdrqtail; i++) {
+		uint32_t qp;
+		uint8_t *bthbytes;
+
+
+		rc = (uint64_t *) (dd->ipath_pd[0]->port_rcvhdrq + (l << 2));
+		hdr = (ips_message_header_typ *) & rc[1];
+		/*
+		 * could make a network order version of IPATH_KD_QP, and
+		 * do the obvious shift before masking to speed this up.
+		 */
+		qp = ntohl(hdr->bth[1]) & 0xffffff;
+		bthbytes = (uint8_t *) hdr->bth;
+
+		eflags = ips_get_hdr_err_flags(rc);
+		etype = ips_get_rcv_type(rc);
+		tlen = ips_get_length_in_bytes(rc);	/* total length */
+		ebuf = NULL;
+		if (etype != RCVHQ_RCV_TYPE_EXPECTED) {
+			/*
+			 * it turns out that the chips uses an eager buffer for
+			 * all non-expected packets, whether it "needs"
+			 * one or not.	So always get the index, but
+			 * don't set ebuf (so we try to copy data)
+			 * unless the length requires it.
+			 */
+			etail = ips_get_index(rc);
+			if (tlen > sizeof(*hdr)
+			    || etype == RCVHQ_RCV_TYPE_NON_KD) {
+				ebuf = ipath_get_egrbuf(t, etail, 0);
+			}
+		}
+
+		/*
+		 * both tiderr and ipathhdrerr are set for all plain IB
+		 * packets; only ipathhdrerr should be set.
+		 */
+
+		if (etype != RCVHQ_RCV_TYPE_NON_KD
+		    && etype != RCVHQ_RCV_TYPE_ERROR
+		    && ips_get_ipath_ver(hdr->iph.ver_port_tid_offset) !=
+		    IPS_PROTO_VERSION) {
+			_IPATH_PDBG("Bad InfiniPath protocol version %x\n",
+				    etype);
+		}
+
+		if (eflags &
+		    ~(INFINIPATH_RHF_H_TIDERR | INFINIPATH_RHF_H_IHDRERR)) {
+			get_rhf_errstring(eflags, emsg, sizeof emsg);
+			_IPATH_PDBG
+			    ("RHFerrs %x hdrqtail=%x typ=%u tlen=%x opcode=%x egridx=%x: %s\n",
+			     eflags, l, etype, tlen, bthbytes[0],
+			     ips_get_index(rc), emsg);
+		} else if (etype == RCVHQ_RCV_TYPE_NON_KD) {
+			/*
+			 * If there is a userland SMA and this is a MAD packet,
+			 * then pass it to the userland SMA.
+			 */
+			if (ipath_sma_alive && qp <= 1) {
+				/*
+				 * count OK packets that we drop because
+				 * SMA isn't yet running, or because we
+				 * are in an sma flood (no point in
+				 * constantly acquiring the spin lock, and
+				 * overwriting previous packets).
+				 * Eventually things will recover.
+				 * Similarly if the sma consumer is
+				 * so far behind that we would overwrite
+				 * (yes, it's outside the lock)
+				 */
+				if (!ipath_sma_data_spare ||
+				    ipath_sma_data[ipath_sma_next].len ||
+				    ++sma_this_time > IPATH_NUM_SMAPKTS) {
+					ipath_stats.sps_krdrops++;
+				} else if (ebuf) {
+					ipath_rcv_sma(t, tlen, rc, ebuf);
+				}
+			} else if (dd->verbs_layer.l_rcv) {
+				dd->verbs_layer.l_rcv(t, rc + 1, ebuf, tlen);
+			} else {
+				_IPATH_VDBG("received IB packet, not SMA (QP=%x)\n",
+					    qp);
+			}
+		} else if (etype == RCVHQ_RCV_TYPE_EAGER) {
+			if (qp == IPATH_KD_QP && bthbytes[0] ==
+			    dd->ipath_layer.l_rcv_opcode && ebuf)
+				ipath_rcv_layer(t, etail, tlen,
+						(ether_header_typ *) hdr);
+			else
+				_IPATH_PDBG
+				    ("typ %x, opcode %x (eager, qp=%x), len %x; ignored\n",
+				     etype, bthbytes[0], qp, tlen);
+		} else if (etype == RCVHQ_RCV_TYPE_EXPECTED) {
+			_IPATH_DBG("Bug: Expected TID, opcode %x; ignored\n",
+				   hdr->bth[0] & 0xff);
+		} else if (eflags &
+			   (INFINIPATH_RHF_H_TIDERR | INFINIPATH_RHF_H_IHDRERR))
+		{
+			/*
+			 * This is a type 3 packet, only the LRH is in
+			 * the rcvhdrq, the rest of the header is in
+			 * the eager buffer.
+			 */
+			uint8_t opcode;
+			if (ebuf) {
+				bthbytes = (uint8_t *) ebuf;
+				opcode = *bthbytes;
+			} else
+				opcode = 0;
+			get_rhf_errstring(eflags, emsg, sizeof emsg);
+			_IPATH_DBG
+			    ("Err %x (%s), opcode %x, egrbuf %x, len %x\n",
+			     eflags, emsg, opcode, etail, tlen);
+		} else {
+			/*
+			 * error packet, type of error	unknown.
+			 * Probably type 3, but we don't know, so don't
+			 * even try to print the opcode, etc.
+			 */
+			_IPATH_DBG
+			    ("Error Pkt, but no eflags! egrbuf %x, len %x\n"
+			     "hdrq@%lx;hdrq+%x rhf: %llx; hdr %llx %llx %llx %llx %llx\n",
+			     etail, tlen, (unsigned long)rc, l, rc[0], rc[1],
+			     rc[2], rc[3], rc[4], rc[5]);
+		}
+		l += rsize;
+		if (l >= maxcnt)
+			l = 0;
+		/*
+		 * update for each packet, to help prevent overflows if we have
+		 * lots of packets.
+		 */
+		(void)ipath_kput_ureg(t, ur_rcvhdrhead, l, 0);
+		if (etype != RCVHQ_RCV_TYPE_EXPECTED)
+			(void)ipath_kput_ureg(t, ur_rcvegrindexhead, etail, 0);
+	}
+
+	pkttot += i;
+
+	dd->ipath_port0head = l;
+
+	if (hdrqtail != *dd->ipath_hdrqtailptr)
+		goto gotmore;	/* more arrived while we handled first batch */
+
+	if(pkttot > ipath_stats.sps_maxpkts_call)
+		ipath_stats.sps_maxpkts_call = pkttot;
+	ipath_stats.sps_port0pkts += pkttot;
+	ipath_stats.sps_avgpkts_call = ipath_stats.sps_port0pkts / ++totcalls;
+
+	if (sma_this_time)	/* only once at end, not each time */
+		wake_up_interruptible(&ipath_sma_wait);
+
+done:
+	clear_bit(0, &dd->ipath_rcv_pending);
+	smp_mb__after_clear_bit();
+}
+
+/*
+ * Update our shadow copy of the PIO availability register map, called
+ * whenever our local copy indicates we have run out of send buffers
+ * NOTE: This can be called from interrupt context by ipath_bufavail()
+ * and from non-interrupt context by ipath_getpiobuf().
+ */
+
+static void ipath_update_pio_bufs(const ipath_type t)
+{
+	unsigned long flags;
+	int i;
+	const unsigned piobregs = (unsigned)devdata[t].ipath_pioavregs;
+
+	/* If the generation (check) bits have changed, then we update the
+	 * busy bit for the corresponding PIO buffer.  This algorithm will
+	 * modify positions to the value they already have in some cases
+	 * (i.e., no change), but it's faster than changing only the bits
+	 * that have changed.
+	 *
+	 * We would like to do this atomicly, to avoid spinlocks in the
+	 * critical send path, but that's not really possible, given the
+	 * type of changes, and that this routine could be called on multiple
+	 * cpu's simultaneously, so we lock in this routine only, to avoid
+	 * conflicting updates; all we change is the shadow, and it's a
+	 * single 64 bit memory location, so by definition the update is
+	 * atomic in terms of what other cpu's can see in testing the
+	 * bits.  The spin_lock overhead isn't too bad, since it only
+	 * happens when all buffers are in use, so only cpu overhead,
+	 * not latency or bandwidth is affected.
+	 */
+#define _IPATH_ALL_CHECKBITS 0x5555555555555555ULL
+	if (!devdata[t].ipath_pioavailregs_dma) {
+		_IPATH_DBG("Update shadow pioavail, but regs_dma NULL!\n");
+		return;
+	}
+	if (infinipath_debug & __IPATH_VERBDBG) {
+		/* only if packet debug and verbose */
+		_IPATH_PDBG("Refill avail, dma0=%llx shad0=%llx, "
+			    "d1=%llx s1=%llx, d2=%llx s2=%llx, d3=%llx s3=%llx\n",
+			    devdata[t].ipath_pioavailregs_dma[0],
+			    devdata[t].ipath_pioavailshadow[0],
+			    devdata[t].ipath_pioavailregs_dma[1],
+			    devdata[t].ipath_pioavailshadow[1],
+			    devdata[t].ipath_pioavailregs_dma[2],
+			    devdata[t].ipath_pioavailshadow[2],
+			    devdata[t].ipath_pioavailregs_dma[3],
+			    devdata[t].ipath_pioavailshadow[3]);
+		if (piobregs > 4)
+			_IPATH_PDBG("2nd group, dma4=%llx shad4=%llx, "
+				    "d5=%llx s5=%llx, d6=%llx s6=%llx, d7=%llx s7=%llx\n",
+				    devdata[t].ipath_pioavailregs_dma[4],
+				    devdata[t].ipath_pioavailshadow[4],
+				    devdata[t].ipath_pioavailregs_dma[5],
+				    devdata[t].ipath_pioavailshadow[5],
+				    devdata[t].ipath_pioavailregs_dma[6],
+				    devdata[t].ipath_pioavailshadow[6],
+				    devdata[t].ipath_pioavailregs_dma[7],
+				    devdata[t].ipath_pioavailshadow[7]);
+	}
+	spin_lock_irqsave(&ipath_pioavail_lock, flags);
+	for (i = 0; i < piobregs; i++) {
+		uint64_t pchbusy, pchg, piov, pnew;
+		/* Chip Errata: bug 6641; even and odd qwords>3 are swapped */
+		piov = devdata[t].ipath_pioavailregs_dma[i > 3 ? i ^ 1 : i];
+		pchg =
+		    _IPATH_ALL_CHECKBITS & ~(devdata[t].
+					     ipath_pioavailshadow[i] ^ piov);
+		pchbusy = pchg << INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT;
+		if (pchg && (pchbusy & devdata[t].ipath_pioavailshadow[i])) {
+			pnew = devdata[t].ipath_pioavailshadow[i] & ~pchbusy;
+			pnew |= piov & pchbusy;
+			devdata[t].ipath_pioavailshadow[i] = pnew;
+		}
+	}
+	spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
+}
+
+static int ipath_do_user_init(ipath_portdata * pd,
+			      struct ipath_user_info *uinfo)
+{
+	int ret = 0;
+	ipath_type t = pd->port_unit;
+	ipath_devdata *dd = &devdata[t];
+	struct ipath_user_info kinfo;
+
+	if (copy_from_user(&kinfo, uinfo, sizeof kinfo))
+		ret = -EFAULT;
+	else {
+		/* for now, if major version is different, bail */
+		if ((kinfo.spu_userversion >> 16) != IPATH_USER_SWMAJOR) {
+			_IPATH_INFO
+			    ("User major version %d not same as driver major %d\n",
+			     kinfo.spu_userversion >> 16, IPATH_USER_SWMAJOR);
+			ret = -ENODEV;
+		} else {
+			if ((kinfo.spu_userversion & 0xffff) !=
+			    IPATH_USER_SWMINOR)
+				_IPATH_DBG
+				    ("User minor version %d not same as driver minor %d\n",
+				     kinfo.spu_userversion & 0xffff,
+				     IPATH_USER_SWMINOR);
+			if (kinfo.spu_rcvhdrsize) {
+				if ((ret =
+				     ipath_setrcvhdrsize(t,
+							 kinfo.spu_rcvhdrsize)))
+					goto done;
+			} else if (!dd->ipath_rcvhdrsize) {
+				/*
+				 * first user of field, kernel or user
+				 * code, and using default
+				 */
+				dd->ipath_rcvhdrsize = IPATH_DFLT_RCVHDRSIZE;
+				ipath_kput_kreg(pd->port_unit, kr_rcvhdrsize,
+						dd->ipath_rcvhdrsize);
+				_IPATH_VDBG
+				    ("Use default protocol header size %u\n",
+				     dd->ipath_rcvhdrsize);
+			}
+
+			pd->port_egrskip = kinfo.spu_egrskip;
+			if (pd->port_egrskip) {
+				if (pd->port_egrskip & 3) {
+					_IPATH_DBG
+					    ("eager skip 0x%x invalid, must be word multiple; using 0x%x\n",
+					     pd->port_egrskip,
+					     pd->port_egrskip & ~3);
+					pd->port_egrskip &= ~3;
+				}
+				_IPATH_DBG
+				    ("user reserves 0x%x bytes at start of eager TIDs\n",
+				     pd->port_egrskip);
+			}
+
+			/*
+			 * for now we do nothing with rcvhdrcnt:
+			 * kinfo.spu_rcvhdrcnt
+			 */
+
+			/*
+			 * set up for the rcvhdr Q tail register writeback
+			 * to user memory
+			 */
+			if (kinfo.spu_rcvhdraddr &&
+			    access_ok(VERIFY_WRITE, kinfo.spu_rcvhdraddr,
+				      sizeof(uint64_t))) {
+				uint64_t physaddr, uaddr, off, atmp;
+				struct page *pagep;
+				off = offset_in_page(kinfo.spu_rcvhdraddr);
+				uaddr =
+				    PAGE_MASK & (unsigned long)kinfo.
+				    spu_rcvhdraddr;
+				if ((ret = ipath_mlock_nocopy(uaddr, &pagep))) {
+					_IPATH_INFO
+					    ("Failed to lookup and lock address %llx for rcvhdrtail: errno %d\n",
+					     kinfo.spu_rcvhdraddr, -ret);
+					goto done;
+				}
+				ipath_stats.sps_pagelocks++;
+				pd->port_rcvhdrtail_uaddr = uaddr;
+				pd->port_rcvhdrtail_pagep = pagep;
+				pd->port_rcvhdrtail_kvaddr =
+				    page_address(pagep);
+				pd->port_rcvhdrtail_kvaddr += off;
+				physaddr = page_to_phys(pagep) + off;
+				_IPATH_VDBG
+				    ("port %d user addr %llx hdrtailaddr, %llx physical (off=%llx)\n",
+				     pd->port_port, kinfo.spu_rcvhdraddr,
+				     physaddr, off);
+				ipath_kput_kreg_port(t, kr_rcvhdrtailaddr,
+						     pd->port_port, physaddr);
+				atmp =
+				    ipath_kget_kreg64_port(t, kr_rcvhdrtailaddr,
+							   pd->port_port);
+				if (physaddr != atmp) {
+					_IPATH_UNIT_ERROR(t,
+							  "Catastrophic software error, RcvHdrTailAddr%u written as %llx, read back as %llx\n",
+							  pd->port_port,
+							  physaddr, atmp);
+					ret = -EINVAL;
+					goto done;
+				}
+			} else {
+				_IPATH_DBG
+				    ("Port %d rcvhdrtail addr %llx not valid\n",
+				     pd->port_port, kinfo.spu_rcvhdraddr);
+				ret = -EINVAL;
+				goto done;
+			}
+
+			/*
+			 * for right now, kernel piobufs are at end,
+			 * so port 1 is at 0
+			 */
+			pd->port_piobufs = dd->ipath_piobufbase +
+			    dd->ipath_pbufsport * (pd->port_port -
+						   1) * dd->ipath_palign;
+			_IPATH_VDBG("Set base of piobufs for port %u to 0x%x\n",
+				    pd->port_port, pd->port_piobufs);
+
+			/*
+			 * Now allocate the rcvhdr Q and eager TIDs;
+			 * skip the TID array for time being.
+			 * If pd->port_port > chip-supported, we need
+			 * to do extra stuff here to handle by handling
+			 * overflow through port 0, someday
+			 */
+			if (!(ret = ipath_create_rcvhdrq(pd)))
+				ret = ipath_create_user_egr(pd);
+			if (!ret) {	/* enable receives now */
+				uint64_t head;
+				uint32_t head32;
+				/* atomically set enable bit for this port */
+				atomic_set_mask(1U <<
+						(INFINIPATH_R_PORTENABLE_SHIFT +
+						 pd->port_port),
+						&dd->ipath_rcvctrl);
+
+				/*
+				 * set the head registers for this port
+				 * to the current values of the tail
+				 * pointers, since we don't know if they
+				 * were updated on last use of the port.
+				 */
+				head32 =
+				    ipath_kget_ureg32(t, ur_rcvhdrtail,
+						      pd->port_port);
+				head = (uint64_t) head32;
+				ipath_kput_ureg(t, ur_rcvhdrhead, head,
+						pd->port_port);
+				head32 =
+				    ipath_kget_ureg32(t, ur_rcvegrindextail,
+						      pd->port_port);
+				ipath_kput_ureg(t, ur_rcvegrindexhead, head32,
+						pd->port_port);
+				dd->ipath_lastegrheads[pd->port_port] = ~0;
+				dd->ipath_lastrcvhdrqtails[pd->port_port] = ~0;
+				_IPATH_VDBG
+				    ("Wrote port%d head %llx, egrhead %x from tail regs\n",
+				     pd->port_port, head, head32);
+				/* start at beginning after open */
+				pd->port_tidcursor = 0;
+				{
+					/*
+					 * now enable the port; the tail
+					 * registers will be written to
+					 * memory by the chip as soon
+					 * as it sees the write to
+					 * kr_rcvctrl.  The update only
+					 * happens on transition from 0
+					 * to 1, so clear it first, then
+					 * set it as part of enabling
+					 * the port.  This will (very
+					 * briefly) affect any other open
+					 * ports, but it shouldn't be long
+					 * enough to be an issue.
+					 */
+					ipath_kput_kreg(t, kr_rcvctrl,
+							dd->
+							ipath_rcvctrl &
+							~INFINIPATH_R_TAILUPD);
+					ipath_kput_kreg(t, kr_rcvctrl,
+							dd->ipath_rcvctrl);
+				}
+			}
+		}
+	}
+
+done:
+	return ret;
+}
+
+static int ipath_get_baseinfo(ipath_portdata * pd,
+			      struct ipath_base_info *ubase)
+{
+	int ret = 0;
+	struct ipath_base_info kbase;
+	ipath_devdata *dd = &devdata[pd->port_unit];
+
+	/* be sure anything we don't set is 0ed */
+	memset(&kbase, 0, sizeof kbase);
+	kbase.spi_rcvhdr_cnt = dd->ipath_rcvhdrcnt;
+	kbase.spi_rcvhdrent_size = dd->ipath_rcvhdrentsize;
+	kbase.spi_tidegrcnt = dd->ipath_rcvegrcnt;
+	kbase.spi_rcv_egrbufsize = dd->ipath_rcvegrbufsize;
+	kbase.spi_rcv_egrbuftotlen = pd->port_rcvegrbuf_chunks * PAGE_SIZE * (1 << pd->port_rcvegrbuf_order);	/* have to mmap whole thing */
+	kbase.spi_rcv_egrperchunk = pd->port_rcvegrbufs_perchunk;
+	kbase.spi_rcv_egrchunksize = kbase.spi_rcv_egrbuftotlen /
+	    pd->port_rcvegrbuf_chunks;
+	kbase.spi_tidcnt = dd->ipath_rcvtidcnt;
+	/*
+	 * for this use, may be ipath_cfgports summed over all chips that
+	 * are are configured and present
+	 */
+	kbase.spi_nports = dd->ipath_cfgports;
+	kbase.spi_unit = pd->port_unit;	/* unit (chip/board) our port is on */
+	/* for now, only a single page */
+	kbase.spi_tid_maxsize = PAGE_SIZE;
+
+	/*
+	 * doing this per port, and based on the skip value, etc.
+	 * This has to be the actual buffer size, since the protocol
+	 * code treats it as an array.
+	 *
+	 * These have to be set to user addresses in the user code via mmap
+	 * These values are used on return to user code for the mmap target
+	 * addresses only.  For 32 bit, same 44 bit address problem, so use
+	 * the physical address, not virtual.  Before 2.6.11, using the
+	 * page_address() macro worked, but in 2.6.11, even that returns
+	 * the full 64 bit address (upper bits all 1's).
+	 * So far, using the physical addresses (or chip offsets, for
+	 * chip mapping) works, but no doubt some future kernel release
+	 * will chang that, and we'll be on to yet another method of
+	 * dealing with this
+	 */
+	kbase.spi_rcvhdr_base = (uint64_t) pd->port_rcvhdrq_phys;
+	kbase.spi_rcv_egrbufs = (uint64_t) pd->port_rcvegr_phys;
+	kbase.spi_pioavailaddr = (uint64_t) dd->ipath_pioavailregs_phys;
+	kbase.spi_status = (uint64_t) kbase.spi_pioavailaddr +
+	    (void *)dd->ipath_statusp - (void *)dd->ipath_pioavailregs_dma;
+	kbase.spi_piobufbase = (uint64_t) pd->port_piobufs;
+	kbase.__spi_uregbase =
+	    dd->ipath_uregbase + dd->ipath_palign * pd->port_port;
+
+	kbase.spi_pioindex = dd->ipath_pbufsport * (pd->port_port - 1);
+	kbase.spi_piocnt = dd->ipath_pbufsport;
+	kbase.spi_pioalign = dd->ipath_palign;
+
+	kbase.spi_qpair = IPATH_KD_QP;
+	kbase.spi_piosize = dd->ipath_ibmaxlen;
+	kbase.spi_mtu = dd->ipath_ibmaxlen;	/* maxlen, not ibmtu */
+	kbase.spi_port = pd->port_port;
+	kbase.spi_sw_version = IPATH_KERN_SWVERSION;
+	kbase.spi_hw_version = dd->ipath_revision;
+
+	if (copy_to_user(ubase, &kbase, sizeof kbase))
+		ret = -EFAULT;
+
+	return ret;
+}
+
+/*
+ * return number of units supported by driver.  This is infinipath_max,
+ * unless there are no initted units.
+ */
+static int ipath_get_units(void)
+{
+	int i;
+
+	for (i = 0; i < infinipath_max; i++)
+		if (devdata[i].ipath_flags & IPATH_INITTED)
+			return infinipath_max;
+	return 0;
+}
+
+/* write data to the EEPROM on the board */
+static int ipath_wr_eeprom(ipath_portdata * pd, struct ipath_eeprom_req *req)
+{
+	int ret = 0;
+	struct ipath_eeprom_req kreq;
+	void *buf = NULL;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;	/* not just any old user can write flash */
+	if (copy_from_user(&kreq, req, sizeof kreq))
+		return -EFAULT;
+	if (!kreq.addr || (kreq.offset + kreq.len) > 128) {
+		_IPATH_DBG
+		    ("called with NULL addr %llx, or bad cnt %u or offset %u\n",
+		     kreq.addr, kreq.len, kreq.offset);
+		return -EINVAL;
+	}
+
+	if (!(buf = vmalloc(kreq.len))) {
+		ret = -ENOMEM;
+		_IPATH_UNIT_ERROR(pd->port_unit,
+				  "Couldn't allocate memory to write %u bytes from eeprom\n",
+				  kreq.len);
+		goto done;
+	}
+	if (copy_from_user(buf, (void *)kreq.addr, kreq.len)) {
+		ret = -EFAULT;
+		goto done;
+	}
+	if (ipath_eeprom_write(pd->port_unit, kreq.offset, buf, kreq.len)) {
+		ret = -ENXIO;
+		_IPATH_UNIT_ERROR(pd->port_unit,
+				  "Failed write to eeprom %u bytes offset %u\n",
+				  kreq.len, kreq.offset);
+	}
+
+done:
+	if (buf)
+		vfree(buf);
+	return ret;
+}
+
+/* read data from the EEPROM on the board */
+int ipath_rd_eeprom(const ipath_type port_unit, struct ipath_eeprom_req *req)
+{
+	int ret = 0;
+	struct ipath_eeprom_req kreq;
+	void *buf = NULL;
+
+	if (copy_from_user(&kreq, req, sizeof kreq))
+		return -EFAULT;
+	if (!kreq.addr || (kreq.offset + kreq.len) > 128) {
+		_IPATH_DBG
+		    ("called with NULL addr %llx, or bad cnt %u or offset %u\n",
+		     kreq.addr, kreq.len, kreq.offset);
+		return -EINVAL;
+	}
+
+	if (!(buf = vmalloc(kreq.len))) {
+		ret = -ENOMEM;
+		_IPATH_UNIT_ERROR(port_unit,
+				  "Couldn't allocate memory to read %u bytes from eeprom\n",
+				  kreq.len);
+		goto done;
+	}
+	if (ipath_eeprom_read(port_unit, kreq.offset, buf, kreq.len)) {
+		ret = -ENXIO;
+		_IPATH_UNIT_ERROR(port_unit,
+				  "Failed reading %u bytes offset %u from eeprom\n",
+				  kreq.len, kreq.offset);
+	}
+	if (copy_to_user((void *)kreq.addr, buf, kreq.len))
+		ret = -EFAULT;
+
+done:
+	if (buf)
+		vfree(buf);
+	return ret;
+}
+
+/*
+ * wait for something to happen on a port.  Currently this is 
+ * PIO buffer available, or a packet being received.  For now, at
+ * least, we wait no longer than 1/2 seconds on rcv, 1 tick on PIO, so
+ * we recover from any bugs (or, as we see in ips.c init and close, cases
+ * where other side isn't yet ready).
+ * NOTE: currently called only with PIO or RCV, never both, so path with both
+ * has not been tested
+ */
+static int ipath_wait_intr(ipath_portdata * pd, uint32_t flag)
+{
+	ipath_devdata *dd = &devdata[pd->port_unit];
+	/* stupid compiler can't tell it's initialized */
+	uint32_t im = 0;
+	uint32_t head, tail, timeo = 0, wflag = 0;
+
+	if (!(flag & (IPATH_WAIT_RCV | IPATH_WAIT_PIO)))
+		return -EINVAL;
+	if (flag & IPATH_WAIT_RCV) {
+		head = flag >> 16;
+		im = (1U << pd->port_port) << INFINIPATH_R_INTRAVAIL_SHIFT;
+		atomic_set_mask(im, &dd->ipath_rcvctrl);
+		/*
+		 * now, before blocking, make sure that head is still == tail,
+		 * reading from the chip, so we can be sure the interrupt enable
+		 * has made it to the chip.  If not equal, disable
+		 * interrupt again and return immediately.  This avoids
+		 * races, and the overhead of the chip read doesn't
+		 * matter much at this point, since we are waiting for
+		 * something anyway.
+		 */
+		ipath_kput_kreg(pd->port_unit, kr_rcvctrl, dd->ipath_rcvctrl);
+		tail =
+		    ipath_kget_ureg32(pd->port_unit, ur_rcvhdrtail,
+				      pd->port_port);
+		if (tail == head) {
+			timeo = HZ / 2;
+			wflag = IPATH_PORT_WAITING_RCV;
+		} else {
+			atomic_clear_mask(im, &dd->ipath_rcvctrl);
+			ipath_kput_kreg(pd->port_unit, kr_rcvctrl,
+					dd->ipath_rcvctrl);
+		}
+	}
+	if (flag & IPATH_WAIT_PIO) {
+		/*
+		 * this one's a bit worse than the receive case, in that we
+		 * can't really verify that at least one interrupt
+		 * will happen...
+		 * We do use a really short timeout, however
+		 */
+		timeo = 1;	/* if both, the short PIO timeout wins */
+		atomic_set_mask(1U << pd->port_port, &dd->ipath_portpiowait);
+		wflag |= IPATH_PORT_WAITING_PIO;
+		/*
+		 * this has a possible race with the ipath stuff, so do
+		 * it atomicly
+		 */
+		atomic_set_mask(INFINIPATH_S_PIOINTBUFAVAIL,
+				&dd->ipath_sendctrl);
+		ipath_kput_kreg(pd->port_unit, kr_sendctrl, dd->ipath_sendctrl);
+	}
+	if (wflag) {
+		pd->port_flag |= wflag;
+		wait_event_interruptible_timeout(pd->port_wait,
+						 (pd->port_flag & wflag) !=
+						 wflag, timeo);
+		if (wflag & pd->port_flag & IPATH_PORT_WAITING_PIO) {
+			/* timed out, no PIO interrupts */
+			atomic_clear_mask(IPATH_PORT_WAITING_PIO,
+					  &pd->port_flag);
+			pd->port_piowait_to++;
+			atomic_clear_mask(1U << pd->port_port,
+					  &dd->ipath_portpiowait);
+			/*
+			 * *don't* clear the pio interrupt enable;
+			 * let that happen in the interrupt handler;
+			 * else we have a race condition.
+			 */
+		}
+		if (wflag & pd->port_flag & IPATH_PORT_WAITING_RCV) {
+			/* timed out, no packets received */
+			atomic_clear_mask(IPATH_PORT_WAITING_RCV,
+					  &pd->port_flag);
+			pd->port_rcvwait_to++;
+			atomic_clear_mask(im, &dd->ipath_rcvctrl);
+			ipath_kput_kreg(pd->port_unit, kr_rcvctrl,
+					dd->ipath_rcvctrl);
+		}
+	} else {
+		/* else it's already happened, don't do wait_event overhead */
+		if (flag & IPATH_WAIT_RCV)
+			pd->port_rcvnowait++;
+		if (flag & IPATH_WAIT_PIO)
+			pd->port_pionowait++;
+	}
+	return 0;
+}
Signature · 1 line
0.99.9n
#7 lore raw

RFC PATCH 6/13 ipath LLD core, part 3

Last part of core driver

---

 drivers/infiniband/hw/ipath/ipath_driver.c | 2380 ++++++++++++++++++++++++++++
 1 files changed, 2380 insertions(+), 0 deletions(-)

f7ffc0cabd62be5e13ad84027d5712e6f92d9cc1

drivers/infiniband/hw/ipath/ipath_driver.c +2380 -0
index 0dee4ce..87b6dae 100644
@@ -4877,3 +4877,2383 @@ static int ipath_wait_intr(ipath_portdat
 	}
 	return 0;
 }
+
+/*
+ * The new implementation as of Oct 2004 is that the driver assigns
+ * the tid and returns it to the caller.   To make it easier to
+ * catch bugs, and to reduce search time, we keep a cursor for
+ * each port, walking the shadow tid array to find one that's not
+ * in use.
+ *
+ * For now, if we can't allocate the full list, we fail, although
+ * in the long run, we'll allocate as many as we can, and the
+ * caller will deal with that by trying the remaining pages later.
+ * That means that when we fail, we have to mark the tids as not in
+ * use again, in our shadow copy.
+ *
+ * It's up to the caller to free the tids when they are done.
+ * We'll unlock the pages as they free them.
+ *
+ * Also, right now we are locking one page at a time, but since
+ * the intended use of this routine is for a single group of
+ * virtually contiguous pages, that should change to improve
+ * performance.
+ */
+static int ipath_tid_update(ipath_portdata * pd, struct _tidupd *tidu)
+{
+	int ret = 0, ntids;
+	uint32_t tid, porttid, cnt, i, tidcnt;
+	struct _tidupd tu;
+	uint16_t *tidlist;
+	ipath_devdata *dd = &devdata[pd->port_unit];
+	uint64_t vaddr, physaddr, lenvalid;
+	volatile uint64_t *tidbase;
+	uint64_t tidmap[8];
+	struct page **pagep = NULL;
+
+	tu.tidcnt = 0;		/* for early errors */
+	if (!dd->ipath_pageshadow) {
+		ret = -ENOMEM;
+		goto done;
+	}
+	if (copy_from_user(&tu, tidu, sizeof tu)) {
+		ret = -EFAULT;
+		goto done;
+	}
+
+	if (!(cnt = tu.tidcnt)) {
+		_IPATH_DBG("After copyin, tidcnt 0, tidlist %llx\n",
+			   tu.tidlist);
+		/* or should we treat as success?  likely a bug */
+		ret = -EFAULT;
+		goto done;
+	}
+	tidcnt = dd->ipath_rcvtidcnt;
+	if (cnt >= tidcnt) {	/* make sure it all fits in port_tid_pg_list */
+		_IPATH_INFO
+		    ("Process tried to allocate %u TIDs, only trying max (%u)\n",
+		     cnt, tidcnt);
+		cnt = tidcnt;
+	}
+	pagep = (struct page **)pd->port_tid_pg_list;
+	tidlist = (uint16_t *) (&pagep[cnt]);
+
+	memset(tidmap, 0, sizeof(tidmap));
+	tid = pd->port_tidcursor;
+	/* before decrement; chip actual # */
+	porttid = pd->port_port * tidcnt;
+	ntids = tidcnt;
+	tidbase = (volatile uint64_t *)((volatile char *)
+					(devdata[pd->port_unit].
+					 ipath_kregbase) +
+					devdata[pd->port_unit].
+					ipath_rcvtidbase +
+					porttid * sizeof(*tidbase));
+
+	_IPATH_VDBG("Port%u %u tids, cursor %u, tidbase %p\n", pd->port_port,
+		    cnt, tid, tidbase);
+
+	vaddr = tu.tidvaddr;	/* virtual address of first page in transfer */
+	if (!access_ok(VERIFY_WRITE, (void *)vaddr, cnt * PAGE_SIZE)) {
+		_IPATH_DBG("Fail vaddr %llx, %u pages, !access_ok\n",
+			   vaddr, cnt);
+		ret = -EFAULT;
+		goto done;
+	}
+	if ((ret = ipath_mlock((unsigned long)vaddr, cnt, pagep))) {
+		if (ret == -EBUSY) {
+			_IPATH_DBG
+			    ("Failed to lock addr %p, %u pages (already locked)\n",
+			     (void *)vaddr, cnt);
+			/*
+			 * for now, continue, and see what happens
+			 * but with the new implementation, this should
+			 * never happen, unless perhaps the user has
+			 * mpin'ed the pages themselves (something we
+			 * need to test)
+			 */
+			ret = 0;
+		} else {
+			_IPATH_INFO
+			    ("Failed to lock addr %p, %u pages: errno %d\n",
+			     (void *)vaddr, cnt, -ret);
+			goto done;
+		}
+	}
+	for (i = 0; i < cnt; i++, vaddr += PAGE_SIZE) {
+		for (; ntids--; tid++) {
+			if (tid == tidcnt)
+				tid = 0;
+			if (!dd->ipath_pageshadow[porttid + tid])
+				break;
+		}
+		if (ntids < 0) {
+			/*
+			 * oops, wrapped all the way through their TIDs,
+			 * and didn't have enough free; see comments at
+			 * start of routine
+			 */
+			_IPATH_DBG
+			    ("Not enough free TIDs for %u pages (index %d), failing\n",
+			     cnt, i);
+			i--;	/* last tidlist[i] not filled in */
+			ret = -ENOMEM;
+			break;
+		}
+		tidlist[i] = tid;
+		_IPATH_VDBG("Updating idx %u to TID %u, vaddr %llx\n",
+			    i, tid, vaddr);
+		/* for now we "know" system pages and TID pages are same size */
+		/* for ipath_free_tid */
+		dd->ipath_pageshadow[porttid + tid] = pagep[i];
+		__set_bit(tid, tidmap);	/* don't need atomic or it's overhead */
+		physaddr = page_to_phys(pagep[i]);
+		ipath_stats.sps_pagelocks++;
+		_IPATH_VDBG("TID %u, vaddr %llx, physaddr %llx pgp %p\n",
+			    tid, vaddr, physaddr, pagep[i]);
+		/*
+		 * in words (fixed, full page).  could make less for very last
+		 * page in transfer, but for now we won't worry about it.
+		 */
+		lenvalid = PAGE_SIZE >> 2;
+		lenvalid <<= INFINIPATH_RT_BUFSIZE_SHIFT;
+		physaddr |= lenvalid | INFINIPATH_RT_VALID;
+		ipath_kput_memq(pd->port_unit, &tidbase[tid], physaddr);
+		/*
+		 * don't check this tid in ipath_portshadow, since we
+		 * just filled it in; start with the next one.
+		 */
+		tid++;
+	}
+
+	if (ret) {
+		uint32_t limit;
+		uint64_t tidval;
+		/*
+		 * chip errata bug 7358, try to work around it by
+		 * marking invalid tids as having max length
+		 */
+		tidval =
+		    (~0ULL & INFINIPATH_RT_BUFSIZE_MASK) <<
+		    INFINIPATH_RT_BUFSIZE_SHIFT;
+	      cleanup:
+		/* jump here if copy out of updated info failed... */
+		_IPATH_DBG("After failure (ret=%d), undo %d of %d entries\n",
+			   -ret, i, cnt);
+		/* same code that's in ipath_free_tid() */
+		if ((limit = sizeof(tidmap) * _BITS_PER_BYTE) > tidcnt)
+			/* just in case size changes in future */
+			limit = tidcnt;
+		tid = find_first_bit((const unsigned long *)tidmap, limit);
+		/*
+		 * chip errata bug 7358, try to work around it by
+		 * marking invalid tids as having max length
+		 */
+		tidval =
+		    (~0ULL & INFINIPATH_RT_BUFSIZE_MASK) <<
+		    INFINIPATH_RT_BUFSIZE_SHIFT;
+		for (; tid < limit; tid++) {
+			if (!test_bit(tid, tidmap))
+				continue;
+			if (dd->ipath_pageshadow[porttid + tid]) {
+				_IPATH_VDBG("Freeing TID %u\n", tid);
+				ipath_kput_memq(pd->port_unit, &tidbase[tid],
+						tidval);
+				dd->ipath_pageshadow[porttid + tid] = NULL;
+				ipath_stats.sps_pageunlocks++;
+			}
+		}
+		(void)ipath_munlock(cnt, pagep);
+	} else {
+		/*
+		 * copy the updated array, with ipath_tid's filled in,
+		 * back to user.  Since we did the copy in already, this
+		 * "should never fail"
+		 * If it does, we have to clean up...
+		 */
+		int r;
+		if ((r =
+		     copy_to_user((void *)tu.tidlist, tidlist,
+				  cnt * sizeof(*tidlist)))) {
+			_IPATH_DBG
+			    ("Failed to copy out %d TIDs (%lx bytes) to %llx (ret %x)\n",
+			     cnt, cnt * sizeof(*tidlist), tu.tidlist, r);
+			ret = -EFAULT;
+			goto cleanup;
+		}
+		if (copy_to_user((void *)tu.tidmap, tidmap, sizeof tidmap)) {
+			_IPATH_DBG("Failed to copy out TID map to %llx\n",
+				   tu.tidmap);
+			ret = -EFAULT;
+			goto cleanup;
+		}
+		if (tid == tidcnt)
+			tid = 0;
+		pd->port_tidcursor = tid;
+	}
+
+done:
+	if (ret)
+		_IPATH_DBG
+		    ("Failed to map %u TID pages, failing with %d, tidu %p\n",
+		     tu.tidcnt, -ret, tidu);
+	return ret;
+}
+
+/*
+ * right now we are unlocking one page at a time, but since
+ * the intended use of this routine is for a single group of
+ * virtually contiguous pages, that should change to improve
+ * performance.  We check that the TID is in range for this port
+ * but otherwise don't check validity; if user has an error and
+ * frees the wrong tid, it's only their own data that can thereby
+ * be corrupted.  We do check that the TID was in use, for sanity
+ * We always use our idea of the saved address, not the address that
+ * they pass in to us.
+ */
+
+static int ipath_tid_free(ipath_portdata * pd, struct _tidupd *tidu)
+{
+	int ret = 0;
+	uint32_t tid, porttid, cnt, limit, tidcnt;
+	struct _tidupd tu;
+	ipath_devdata *dd = &devdata[pd->port_unit];
+	uint64_t *tidbase;
+	uint64_t tidmap[8];
+	uint64_t tidval;
+
+	tu.tidcnt = 0;		/* for early errors */
+	if (!dd->ipath_pageshadow) {
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	if (copy_from_user(&tu, tidu, sizeof tu)) {
+		_IPATH_DBG("copy of tidupd structure failed\n");
+		ret = -EFAULT;
+		goto done;
+	}
+	if (copy_from_user(tidmap, (void *)tu.tidmap, sizeof tidmap)) {
+		_IPATH_DBG("copy of tidmap failed\n");
+		ret = -EFAULT;
+		goto done;
+	}
+
+	porttid = pd->port_port * dd->ipath_rcvtidcnt;
+	tidbase =
+	    (uint64_t *) ((char *)(devdata[pd->port_unit].ipath_kregbase) +
+			  devdata[pd->port_unit].ipath_rcvtidbase +
+			  porttid * sizeof(*tidbase));
+
+	tidcnt = dd->ipath_rcvtidcnt;
+	if ((limit = sizeof(tidmap) * _BITS_PER_BYTE) > tidcnt)
+		limit = tidcnt;	/* just in case size changes in future */
+	tid = find_first_bit((const unsigned long *)tidmap, limit);
+	_IPATH_VDBG
+	    ("Port%u free %u tids; first bit (max=%d) set is %d, porttid %u\n",
+	     pd->port_port, tu.tidcnt, limit, tid, porttid);
+	/*
+	 * chip errata bug 7358, try to work around it by marking invalid
+	 * tids as having max length
+	 */
+	tidval =
+	    (~0ULL & INFINIPATH_RT_BUFSIZE_MASK) << INFINIPATH_RT_BUFSIZE_SHIFT;
+	for (cnt = 0; tid < limit; tid++) {
+		/*
+		 * small optimization; if we detect a run of 3 or so without
+		 * any set, use find_first_bit again.  That's mainly to 
+		 * accelerate the case where we wrapped, so we have some at 
+		 * the beginning, and some at the end, and a big gap
+		 * in the middle.
+		 */
+		if (!test_bit(tid, tidmap))
+			continue;
+		cnt++;
+		if (dd->ipath_pageshadow[porttid + tid]) {
+			_IPATH_VDBG("Freeing TID %u\n", tid);
+			ipath_kput_memq(pd->port_unit, &tidbase[tid], tidval);
+			ipath_munlock(1, &dd->ipath_pageshadow[porttid + tid]);
+			dd->ipath_pageshadow[porttid + tid] = NULL;
+			ipath_stats.sps_pageunlocks++;
+		} else
+			_IPATH_DBG("Unused tid %u, ignoring\n", tid);
+	}
+	if (cnt != tu.tidcnt)
+		_IPATH_DBG("passed in tidcnt %d, only %d bits set in map\n",
+			   tu.tidcnt, cnt);
+done:
+	if (ret)
+		_IPATH_DBG("Failed to unmap %u TID pages, failing with %d\n",
+			   tu.tidcnt, -ret);
+	return ret;
+}
+
+/* called from user init code, and also layered driver init */
+int ipath_setrcvhdrsize(const ipath_type mdev, unsigned rhdrsize)
+{
+	int ret = 0;
+	if (devdata[mdev].ipath_flags & IPATH_RCVHDRSZ_SET) {
+		if (devdata[mdev].ipath_rcvhdrsize != rhdrsize) {
+			_IPATH_INFO
+			    ("Error: can't set protocol header size %u, already %u\n",
+			     rhdrsize, devdata[mdev].ipath_rcvhdrsize);
+			ret = -EAGAIN;
+		} else
+			/* OK if set already, with same value, nothing to do */
+			_IPATH_VDBG("Reuse same protocol header size %u\n",
+				    devdata[mdev].ipath_rcvhdrsize);
+	} else if (rhdrsize >
+		   (devdata[mdev].ipath_rcvhdrentsize -
+		    (sizeof(uint64_t) / sizeof(uint32_t)))) {
+		_IPATH_DBG
+		    ("Error: can't set protocol header size %u (> max %u)\n",
+		     rhdrsize,
+		     devdata[mdev].ipath_rcvhdrentsize -
+		     (uint32_t) (sizeof(uint64_t) / sizeof(uint32_t)));
+		ret = -EOVERFLOW;
+	} else {
+		devdata[mdev].ipath_flags |= IPATH_RCVHDRSZ_SET;
+		devdata[mdev].ipath_rcvhdrsize = rhdrsize;
+		ipath_kput_kreg(mdev, kr_rcvhdrsize,
+				devdata[mdev].ipath_rcvhdrsize);
+		_IPATH_VDBG("Set protocol header size to %u\n",
+			    devdata[mdev].ipath_rcvhdrsize);
+	}
+	return ret;
+}
+
+/*
+ * find an available pio buffer, and do appropriate marking as busy, etc.
+ * returns buffer number if one found (>=0), negative number is error.
+ * Used by ipath_send_smapkt and ipath_layer_send
+ */
+int ipath_getpiobuf(int mdev)
+{
+	int i, j, starti, updated = 0;
+	unsigned piobcnt, iter;
+	unsigned long flags;
+	ipath_devdata *dd = &devdata[mdev];
+	uint64_t *shadow = dd->ipath_pioavailshadow;
+
+	piobcnt = (unsigned)dd->ipath_piobcnt;
+	starti = dd->ipath_lastport_piobuf;
+	iter = piobcnt - starti;
+	if (dd->ipath_upd_pio_shadow) {
+		/*
+		 * minor optimization.	If we had no buffers on last call,
+		 * start out by doing the update; continue and do scan
+		 * even if no buffers were updated, to be paranoid
+		 */
+		ipath_update_pio_bufs(mdev);
+		/* we scanned here, don't do it at end of scan */
+		updated = 1;
+		i = starti;
+	} else
+		i = dd->ipath_lastpioindex;
+
+rescan:
+	/*
+	 * while test_and_set_bit() is atomic,
+	 * we do that and then the change_bit(), and the pair is not.
+	 * See if this is the cause of the remaining armlaunch errors.
+	 */
+	spin_lock_irqsave(&ipath_pioavail_lock, flags);
+	for (j = 0; j < iter; j++, i++) {
+		if (i >= piobcnt)
+			i = starti;
+		/*
+		 * To avoid bus lock overhead, we first find a candidate
+		 * buffer, then do the test and set, and continue if
+		 * that fails.
+		 */
+		if (test_bit((2 * i) + 1, shadow) ||
+		    test_and_set_bit((2 * i) + 1, shadow)) {
+			continue;
+		}
+		/* flip generation bit */
+		change_bit(2 * i, shadow);
+		break;
+	}
+	spin_unlock_irqrestore(&ipath_pioavail_lock, flags);
+
+	if (j == iter) {
+		/*
+		 * first time through; shadow exhausted, but may be
+		 * real buffers available, so go see; if any updated,
+		 * rescan (once)
+		 */
+		if (!updated) {
+			ipath_update_pio_bufs(mdev);
+			updated = 1;
+			i = starti;
+			goto rescan;
+		}
+		dd->ipath_upd_pio_shadow = 1;
+		/* not atomic, but if we lose one once in a while, that's OK */
+		ipath_stats.sps_nopiobufs++;
+		if (!(++dd->ipath_consec_nopiobuf % 100000)) {
+			_IPATH_DBG
+			    ("%u pio sends with no bufavail; dmacopy: %llx %llx %llx %llx; shadow:  %llx %llx %llx %llx\n",
+			     dd->ipath_consec_nopiobuf,
+			     dd->ipath_pioavailregs_dma[0],
+			     dd->ipath_pioavailregs_dma[1],
+			     dd->ipath_pioavailregs_dma[2],
+			     dd->ipath_pioavailregs_dma[3],
+			     shadow[0], shadow[1], shadow[2], shadow[3]);
+			/*
+			 * 4 buffers per byte, 4 registers above, cover
+			 * rest below
+			 */
+			if (dd->ipath_piobcnt > (sizeof(shadow[0]) * 4 * 4))
+				_IPATH_DBG
+				    ("2nd group: dmacopy: %llx %llx %llx %llx; shadow: %llx %llx %llx %llx\n",
+				     dd->ipath_pioavailregs_dma[4],
+				     dd->ipath_pioavailregs_dma[5],
+				     dd->ipath_pioavailregs_dma[6],
+				     dd->ipath_pioavailregs_dma[7],
+				     shadow[4], shadow[5], shadow[6],
+				     shadow[7]);
+		}
+		return -EBUSY;
+	}
+
+	if (updated && dd->ipath_layer.l_intr) {
+		/*
+		 * ran out of bufs, now some (at least this one we just got)
+		 * are now available, so tell the layered driver.
+		 */
+		 dd->ipath_layer.l_intr(mdev, IPATH_LAYER_INT_SEND_CONTINUE);
+	}
+
+	/*
+	 * set next starting place.  Since it's just an optimization,
+	 * it doesn't matter who wins on this, so no locking
+	 */
+	dd->ipath_lastpioindex = i + 1;
+	if(dd->ipath_upd_pio_shadow)
+		dd->ipath_upd_pio_shadow = 0;
+	if(dd->ipath_consec_nopiobuf)
+		dd->ipath_consec_nopiobuf = 0;
+	return i;
+}
+
+/*
+ * this is like ipath_getpiobuf(), except it just probes to see if a buffer
+ * is available.  If it returns that there is one, it's not allocated,
+ * and so may not be available if caller tries to send.
+ * NOTE: This can be called from interrupt context by ipath_intr()
+ * and from non-interrupt context by layer_send_getpiobuf().
+ */
+int ipath_bufavail(int mdev)
+{
+	int i;
+	unsigned piobcnt;
+	uint64_t *shadow = devdata[mdev].ipath_pioavailshadow;
+
+	piobcnt = (unsigned)devdata[mdev].ipath_piobcnt;
+
+	for (i = devdata[mdev].ipath_lastport_piobuf; i < piobcnt; i++)
+		if (!test_bit((2 * i) + 1, shadow))
+			return 1;
+
+	/* if none, check for update and rescan if we updated */
+	ipath_update_pio_bufs(mdev);
+	for (i = devdata[mdev].ipath_lastport_piobuf; i < piobcnt; i++)
+		if (!test_bit((2 * i) + 1, shadow))
+			return 1;
+	_IPATH_PDBG("No bufs avail\n");
+	return 0;
+}
+
+/*
+ * This routine is no longer on any critical paths; it is used only
+ * for sending SMA packets, but that could change in the future, so it
+ * should be kept pretty tight, with anything that
+ * increases the cache footprint, adds branches, etc. carefully
+ * examined, and if needed only for unusual cases, should, be moved out to
+ * a separate routine, or out of the main execution path.
+ * Because it's currently sma only, there are no checks to see if the
+ * link is up; sma must be able to send in the not fully initialized state
+ */
+int ipath_send_smapkt(struct ipath_sendpkt * upkt)
+{
+	int i, ret = 0, whichpb;
+	uint32_t *piobuf, plen = 0, clen;
+	uint64_t pboff;
+	struct ipath_sendpkt kpkt;
+	struct ipath_iovec *iov = kpkt.sps_iov;
+	ipath_type t;
+
+	if (unlikely((copy_from_user(&kpkt, upkt, sizeof kpkt))))
+		ret = -EFAULT;
+	if (ret) {
+		_IPATH_VDBG("Send failed: error %d\n", -ret);
+		goto done;
+	}
+	t = kpkt.sps_flags;
+	if (t >= infinipath_max || !(devdata[t].ipath_flags & IPATH_PRESENT) ||
+	    !devdata[t].ipath_kregbase) {
+		_IPATH_SMADBG("illegal unit %u for sma send\n", t);
+		return -ENODEV;
+	}
+	if (!(devdata[t].ipath_flags & IPATH_INITTED)) {
+		/* no hardware, freeze, etc. */
+		_IPATH_SMADBG("unit %u not usable\n", t);
+		return -ENODEV;
+	}
+
+	/* need total length before first word written */
+	plen = sizeof(uint32_t);	/* +1 word is for the qword padding */
+	for (i = 0; i < kpkt.sps_cnt; i++)
+		/* each must be dword multiple */
+		plen += kpkt.sps_iov[i].iov_len;
+
+	if ((plen + 4) > devdata[t].ipath_ibmaxlen) {
+		_IPATH_DBG("Pkt len 0x%x > ibmaxlen %x!\n", plen - 4,
+			   devdata[t].ipath_ibmaxlen);
+		ret = -EINVAL;
+		goto done;	/* before writing pbc */
+	}
+	plen >>= 2;		/* in words */
+
+	whichpb = ipath_getpiobuf(t);
+	if (whichpb < 0) {
+		ret = whichpb;
+		devdata[t].ipath_nosma_bufs++;
+		_IPATH_SMADBG("No PIO buffers available unit %u %u times\n",
+			t, devdata[t].ipath_nosma_bufs);
+		goto done;
+	}
+	if(devdata[t].ipath_nosma_bufs) {
+		_IPATH_SMADBG(
+			"Unit %u got SMA send buffer after %u failures, %u seconds\n",
+			t, devdata[t].ipath_nosma_bufs, devdata[t].ipath_nosma_secs);
+		devdata[t].ipath_nosma_bufs = 0;
+		devdata[t].ipath_nosma_secs = 0;
+	}
+	if((devdata[t].ipath_lastibcstat & 0x11) != 0x11 &&
+		(devdata[t].ipath_lastibcstat & 0x21) != 0x21) {
+	    /* we need to be at least at INIT for SMA packets to go out.  If we
+	     * aren't, something has gone wrong, and SMA hasn't noticed.
+	     * Therefore we'll try to go to INIT here, in hopes of fixing up the
+	     * problem.  First we verify that indeed the state is still "bad"
+	     * (that is, that lastibcstat * isn't "stale") */
+	    uint64_t val;
+	    val = ipath_kget_kreg64(t, kr_ibcstatus);
+	    if((val & 0x11) != 0x11 && (val & 0x21) != 0x21) {
+		_IPATH_SMADBG("Invalid Link state 0x%llx unit %u for send, try INIT\n",
+			val, t);
+		ipath_set_ib_lstate(t, INFINIPATH_IBCC_LINKCMD_INIT);
+		val = ipath_kget_kreg64(t, kr_ibcstatus);
+		if((val & 0x11) != 0x11 && (val & 0x21) != 0x21)
+		    _IPATH_SMADBG("Link state still not OK unit %u (0x%llx) after INIT\n",
+				t, val);
+		else
+		    _IPATH_SMADBG("Link state OK unit %u (0x%llx) after INIT\n",
+				t, val);
+	    }
+	    /* and continue, regardless */
+	}
+
+	pboff = devdata[t].ipath_piobufbase;
+	piobuf = (uint32_t *) (((char *)(devdata[t].ipath_kregbase)) + pboff
+			       + whichpb * devdata[t].ipath_palign);
+
+	if(infinipath_debug & __IPATH_PKTDBG) // SMA and PKT, both
+		_IPATH_SMADBG("unit %u 0x%x+1w pio%d, (scnt %d)\n",
+			t, plen - 1, whichpb, kpkt.sps_cnt);
+
+	ret = 0;
+	clen = 2;		/* size of the pbc */
+	{
+		/*
+		 * If this code ever gets used for anything performance
+		 * oriented, or that isn't inherently single-threaded,
+		 * then I need to implement the original idea of our
+		 * own equivalent of copy_from_user that uses only dword
+		 * or qword copies.  copy_from_user() can use byte copies,
+		 * and that is a problem for our chip.
+		 */
+		static uint32_t tmpbuf[2176 / sizeof(uint32_t)];
+		*(uint64_t *) tmpbuf = (uint64_t) plen;
+		for (i = 0; i < kpkt.sps_cnt; i++) {
+			if (unlikely
+			    (copy_from_user
+			     (tmpbuf + clen, (void *)iov->iov_base,
+			      iov->iov_len)))
+				ret = -EFAULT;	/* no break */
+			clen += iov->iov_len >> 2;
+			iov++;
+		}
+		ipath_dwordcpy(piobuf, tmpbuf, clen);
+	}
+
+	/* flush the packet out now, don't leave it waiting around */
+	mb();
+
+	if (ret) {
+		/*
+		 * Packet is bad, so we need to use the PIO abort mechanism to
+		 * abort the packet
+		 */
+		uint32_t sendctrl;
+		sendctrl = devdata[t].ipath_sendctrl | INFINIPATH_S_DISARM |
+		    (whichpb << INFINIPATH_S_DISARMPIOBUF_SHIFT);
+		_IPATH_DBG("Doing PIO abort on buffer %u after error\n",
+			   whichpb);
+		ipath_kput_kreg(t, kr_sendctrl, sendctrl);
+	}
+
+done:
+	return ret;
+}
+
+/*
+ * implemention of the ioctl to get the counter values from the chip
+ * For the time being, we get all of them when asked, no shadowing.
+ * We need to shadow the byte counters at a minimum, because otherwise
+ * they will wrap in just a few seconds at full bandwidth
+ * The second argument is the user address to which we do the copy_to_user()
+ */
+static int ipath_get_counters(ipath_type t,
+			      struct infinipath_counters * ucounters)
+{
+	int ret = 0;
+	uint64_t val;
+	uint64_t *ucreg;
+	uint16_t vcreg;
+
+	ucreg = (uint64_t *) ucounters;
+	/*
+	 * for now, let's do this one at a time.  It's not the most
+	 * optimal method, but it is simple, and has no intermediate
+	 * memory requirements.
+	 */
+	for (vcreg = 0;
+	     vcreg < (sizeof(struct infinipath_counters) / sizeof(val));
+	     vcreg++, ucreg++) {
+		ipath_creg creg = vcreg;
+		val = ipath_snap_cntr(t, creg);
+		if ((ret = copy_to_user(ucreg, &val, sizeof(val)))) {
+			_IPATH_DBG("copy_to_user error on counter %d\n", creg);
+			break;
+		}
+	}
+
+	return ret;
+}
+
+/*
+ * implemention of the ioctl to get the stats values from the driver
+ * The argument is the user address to which we do the copy_to_user()
+ */
+static int ipath_get_stats(struct infinipath_stats *ustats)
+{
+	int ret = 0;
+
+	if ((ret = copy_to_user(ustats, &ipath_stats, sizeof(ipath_stats))))
+		_IPATH_DBG("copy_to_user error on driver stats\n");
+
+	return ret;
+}
+
+/* set a partition key.  We can have up to 4 active at a time (other than
+ * the default, which is always allowed).  This is somewhat tricky, since
+ * multiple ports may set the same key, so we reference count them, and
+ * clean up at exit.  All 4 partition keys are packed into a single
+ * infinipath register.  It's an error for a process to set the same
+ * pkey multiple times.  We provide no mechanism to de-allocate a pkey
+ * at this time, we may eventually need to do that.
+ * I've used the atomic operations, and no locking, and only make a single
+ * pass through what's available.  This should be more than adequate for
+ * some time. I'll think about spinlocks or the like if and as it's necessary
+ */
+static int ipath_set_partkey(ipath_portdata *pd, uint16_t key)
+{
+	ipath_devdata *dd;
+	int i, any = 0, pidx = -1;
+	uint16_t lkey = key & 0x7FFF;
+
+	dd = &devdata[pd->port_unit];
+
+	if (lkey == (IPS_DEFAULT_P_KEY & 0x7FFF)) {
+		/* nothing to do; this key always valid */
+		return 0;
+	}
+
+	_IPATH_VDBG
+	    ("p%u try to set pkey %hx, current keys %hx:%x %hx:%x %hx:%x %hx:%x\n",
+	     pd->port_port, key, dd->ipath_pkeys[0],
+	     atomic_read(&dd->ipath_pkeyrefs[0]), dd->ipath_pkeys[1],
+	     atomic_read(&dd->ipath_pkeyrefs[1]), dd->ipath_pkeys[2],
+	     atomic_read(&dd->ipath_pkeyrefs[2]), dd->ipath_pkeys[3],
+	     atomic_read(&dd->ipath_pkeyrefs[3]));
+
+	if (!lkey) {
+		_IPATH_PRDBG("p%u tries to set key 0, not allowed\n",
+			     pd->port_port);
+		return -EINVAL;
+	}
+
+	/*
+	 * Set the full membership bit, because it has to be
+	 * set in the register or the packet, and it seems
+	 * cleaner to set in the register than to force all
+	 * callers to set it. (see bug 4331)
+	 */
+	key |= 0x8000;
+
+	for (i = 0; i < ARRAY_SIZE(pd->port_pkeys); i++) {
+		if (!pd->port_pkeys[i] && pidx == -1)
+			pidx = i;
+		if (pd->port_pkeys[i] == key) {
+			_IPATH_VDBG
+			    ("p%u tries to set same pkey (%x) more than once\n",
+			     pd->port_port, key);
+			return -EEXIST;
+		}
+	}
+	if (pidx == -1) {
+		_IPATH_DBG
+		    ("All pkeys for port %u already in use, can't set %x\n",
+		     pd->port_port, key);
+		return -EBUSY;
+	}
+	for (any = i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
+		if (!dd->ipath_pkeys[i]) {
+			any++;
+			continue;
+		}
+		if (dd->ipath_pkeys[i] == key) {
+			if (atomic_inc_return(&dd->ipath_pkeyrefs[i]) > 1) {
+				pd->port_pkeys[pidx] = key;
+				_IPATH_VDBG
+				    ("p%u set key %x matches #%d, count now %d\n",
+				     pd->port_port, key, i,
+				     atomic_read(&dd->ipath_pkeyrefs[i]));
+				return 0;
+			} else {
+				/* lost race, decrement count, catch below */
+				atomic_dec(&dd->ipath_pkeyrefs[i]);
+				_IPATH_VDBG
+				    ("Lost race, count was 0, after dec, it's %d\n",
+				     atomic_read(&dd->ipath_pkeyrefs[i]));
+				any++;
+			}
+		}
+		if ((dd->ipath_pkeys[i] & 0x7FFF) == lkey) {
+			/*
+			 * It makes no sense to have both the limited and full
+			 * membership PKEY set at the same time since the
+			 * unlimited one will disable the limited one.
+			 */
+			return -EEXIST;
+		}
+	}
+	if (!any) {
+		_IPATH_DBG
+		    ("port %u, all pkeys already in use, can't set %x\n",
+		     pd->port_port, key);
+		return -EBUSY;
+	}
+	for (any = i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
+		if (!dd->ipath_pkeys[i] &&
+		    atomic_inc_return(&dd->ipath_pkeyrefs[i]) == 1) {
+			uint64_t pkey;
+
+			/* for ipathstats, etc. */
+			ipath_stats.sps_pkeys[i] = lkey;
+			pd->port_pkeys[pidx] = dd->ipath_pkeys[i] = key;
+			pkey =
+			    (uint64_t) dd->ipath_pkeys[0] |
+			    ((uint64_t) dd->ipath_pkeys[1] << 16) |
+			    ((uint64_t) dd->ipath_pkeys[2] << 32) |
+			    ((uint64_t) dd->ipath_pkeys[3] << 48);
+			_IPATH_PRDBG
+			    ("p%u set key %x in #%d, portidx %d, new pkey reg %llx\n",
+			     pd->port_port, key, i, pidx, pkey);
+			ipath_kput_kreg(pd->port_unit, kr_partitionkey, pkey);
+
+			return 0;
+		}
+	}
+	_IPATH_DBG
+	    ("port %u, all pkeys already in use 2nd pass, can't set %x\n",
+	     pd->port_port, key);
+	return -EBUSY;
+}
+
+/*
+ * stop_start == 0 disables receive on the port, for use in queue overflow
+ * conditions.  stop_start==1 re-enables, and returns value of tail register,
+ * to be used to re-init the software copy of the head register
+ */
+
+static int ipath_manage_rcvq(ipath_portdata * pd, uint16_t start_stop)
+{
+	ipath_devdata *dd;
+	/*
+	 * This needs to be volatile, so that the compiler doesn't
+	 * optimize away the read to the device's mapped memory.
+	 */
+	volatile uint64_t tval;
+
+	dd = &devdata[pd->port_unit];
+	_IPATH_PRDBG("%sabling rcv for unit %u port %u\n",
+		     start_stop ? "en" : "dis", pd->port_unit, pd->port_port);
+	/* atomically clear receive enable port. */
+	if (start_stop) {
+		/*
+		 * on enable, force in-memory copy of the tail register
+		 * to 0, so that protocol code doesn't have to worry
+		 * about whether or not the chip has yet updated
+		 * the in-memory copy or not on return from the system
+		 * call. The chip always resets it's tail register back
+		 * to 0 on a transition from disabled to enabled.
+		 * This could cause a problem if software was broken,
+		 * and did the enable w/o the disable, but eventually
+		 * the in-memory copy will be updated and correct
+		 * itself, even in the face of software bugs.
+		 */
+		*pd->port_rcvhdrtail_kvaddr = 0;
+		atomic_set_mask(1U <<
+				(INFINIPATH_R_PORTENABLE_SHIFT + pd->port_port),
+				&dd->ipath_rcvctrl);
+	} else
+		atomic_clear_mask(1U <<
+				  (INFINIPATH_R_PORTENABLE_SHIFT +
+				   pd->port_port), &dd->ipath_rcvctrl);
+	ipath_kput_kreg(pd->port_unit, kr_rcvctrl, dd->ipath_rcvctrl);
+	/* now be sure chip saw it before we return */
+	tval = ipath_kget_kreg64(pd->port_unit, kr_scratch);
+	if (start_stop) {
+		/*
+		 * and try to be sure that tail reg update has happened
+		 * too.  This should in theory interlock with the RXE
+		 * changes to the tail register.  Don't assign it to
+		 * the tail register in memory copy, since we could
+		 * overwrite an update by the chip if we did.
+		 */
+		tval =
+		    ipath_kget_ureg32(pd->port_unit, ur_rcvhdrtail,
+				      pd->port_port);
+	}
+	/* always; new head should be equal to new tail; see above */
+	return 0;
+}
+
+/*
+ * This routine is now quite different for user and kernel, because
+ * the kernel uses skb's, for the accelerated network performance
+ * This is the user port version
+ *
+ * allocate the eager TID buffers and program them into infinipath
+ * They are no longer completely contiguous, we do multiple
+ * alloc_pages() calls.
+ */
+static int ipath_create_user_egr(ipath_portdata * pd)
+{
+	char *buf;
+	ipath_devdata *dd = &devdata[pd->port_unit];
+	uint64_t *egrbase, egroff, lenvalid;
+	unsigned e, egrcnt, alloced, order, egrperchunk, chunk;
+	unsigned long pa, pent;
+
+	egrcnt = dd->ipath_rcvegrcnt;
+	egroff =
+	    dd->ipath_rcvegrbase + pd->port_port * egrcnt * sizeof(*egrbase);
+	egrbase = (uint64_t *) ((char *)(dd->ipath_kregbase) + egroff);
+	_IPATH_VDBG("Allocating %d egr buffers, at chip offset %llx (%p)\n",
+		    egrcnt, egroff, egrbase);
+
+	/*
+	 * to avoid wasting a lot of memory, we allocate 32KB chunks of
+	 * physically contiguous memory, advance through it until used up
+	 * and then allocate more.  Of course, we need memory to store
+	 * those extra pointers, now.  Started out with 256KB, but under
+	 * heavy memory pressure (creating large files and then copying
+	 * them over NFS while doing lots of MPI jobs), we hit some
+	 * alloc_pages() failures, even though we can sleep...  (2.6.10)
+	 * Still get failures at 64K.  32K is the lowest we can go without
+	 * waiting more memory again.  It seems likely that the coalescing
+	 * in free_pages, etc. still has issues (as it has had previously
+	 * during 2.6.x development).
+	 */
+	order = get_order(0x8000);
+	alloced =
+	    round_up(dd->ipath_rcvegrbufsize * egrcnt,
+		     (1 << order) * PAGE_SIZE);
+	egrperchunk = ((1 << order) * PAGE_SIZE) / dd->ipath_rcvegrbufsize;
+	chunk = (egrcnt + egrperchunk - 1) / egrperchunk;
+	pd->port_rcvegrbuf_chunks = chunk;
+	pd->port_rcvegrbufs_perchunk = egrperchunk;
+	pd->port_rcvegrbuf_order = order;
+	pd->port_rcvegrbuf_pages =
+	    vmalloc(chunk * sizeof(pd->port_rcvegrbuf_pages[0]));
+	pd->port_rcvegrbuf_virt =
+	    vmalloc(chunk * sizeof(pd->port_rcvegrbuf_virt[0]));
+	if (!pd->port_rcvegrbuf_pages || !pd->port_rcvegrbuf_pages) {
+		_IPATH_UNIT_ERROR(pd->port_unit,
+				  "Unable to allocate %u EGR buffer array pointers\n",
+				  chunk);
+		if (pd->port_rcvegrbuf_pages) {
+			vfree(pd->port_rcvegrbuf_pages);
+			pd->port_rcvegrbuf_pages = NULL;
+		}
+		return -ENOMEM;
+	}
+	for (e = 0; e < pd->port_rcvegrbuf_chunks; e++) {
+		/*
+		 * GFP_USER, but without GFP_FS, so buffer cache can
+		 * be coalesced (we hope); otherwise, even at order 4, heavy
+		 * filesystem activity makes these fail
+		 */
+		if (!
+		    (pd->port_rcvegrbuf_pages[e] =
+		     alloc_pages(__GFP_WAIT | __GFP_IO, order))) {
+			_IPATH_UNIT_ERROR(pd->port_unit,
+					  "Unable to allocate EGR buffer array %u/%u\n",
+					  e, pd->port_rcvegrbuf_chunks);
+			vfree(pd->port_rcvegrbuf_pages);
+			pd->port_rcvegrbuf_pages = NULL;
+			vfree(pd->port_rcvegrbuf_virt);
+			pd->port_rcvegrbuf_virt = NULL;
+			return -ENOMEM;
+		}
+	}
+
+	/*
+	 * calculate physical, then phys_to_virt()
+	 * so that we get an address that fits in 64 bits, so we can use
+	 * mmap64 from 32 bit programs on the chip and kernel virtual
+	 * addresses (mmap64 for 32 bit programs on i386 and x86_64
+	 * only has 44 bits of address, because it uses mmap2())
+	 * We do this with the first chunk;  We don't need a kernel
+	 * virtually contiguous address to give the user virtually
+	 * contiguous mappings.  It just complicates the nopage routine
+	 * a little tiny bit ;)
+	 */
+	buf = page_address(pd->port_rcvegrbuf_pages[0]);
+	pa = virt_to_phys(buf);
+	pd->port_rcvegr_phys = pa;
+
+	/* in words */
+	lenvalid = (dd->ipath_rcvegrbufsize - pd->port_egrskip) >> 2;
+	_IPATH_VDBG
+	    ("port%u egrbuf vaddr %p, cpu %d, egrskip %u, len %llx words\n",
+	     pd->port_port, buf, smp_processor_id(), pd->port_egrskip,
+	     lenvalid);
+	lenvalid <<= INFINIPATH_RT_BUFSIZE_SHIFT;
+	lenvalid |= INFINIPATH_RT_VALID;
+
+	for (e = chunk = 0; chunk < pd->port_rcvegrbuf_chunks; chunk++) {
+		int i, n;
+		struct page *p;
+		p = pd->port_rcvegrbuf_pages[chunk];
+		pa = page_to_phys(p);
+		buf = page_address(p);
+		/*
+		 * stash away for later use, since page_address() lookup
+		 * is not cheap
+		 */
+		pd->port_rcvegrbuf_virt[chunk] = buf;
+		if (pa & ~INFINIPATH_RT_ADDR_MASK)
+			_IPATH_INFO
+			    ("physaddr %lx has more than 40 bits, using only 40!\n",
+			     pa);
+		n = 1 << pd->port_rcvegrbuf_order;
+		for (i = 0; i < n; i++)
+			SetPageReserved(virt_to_page(buf + (i * PAGE_SIZE)));
+
+		/* clear buffer for security, sanity, and, debugging */
+		memset(buf, 0, PAGE_SIZE * n);
+
+		for (i = 0; e < egrcnt && i < egrperchunk; e++, i++) {
+			pent =
+			    ((pa +
+			      pd->
+			      port_egrskip) & INFINIPATH_RT_ADDR_MASK) |
+			    lenvalid;
+
+			ipath_kput_memq(pd->port_unit, &egrbase[e], pent);
+			_IPATH_VDBG("egr %u phys %lx val %lx\n", e, pa, pent);
+			pa += dd->ipath_rcvegrbufsize;
+		}
+		yield();	/* don't hog the cpu */
+	}
+
+	return 0;
+}
+
+/*
+ * This routine is now quite different for user and kernel, because
+ * the kernel uses skb's, for the accelerated network performance
+ * This is the kernel (port0) version
+ *
+ * Allocate the eager TID buffers and program them into infinipath.
+ * We use the network layer alloc_skb() allocator to allocate the memory, and
+ * either use the buffers as is for things like SMA packets, or pass
+ * the buffers up to the ipath layered driver and thence the network layer,
+ * replacing them as we do so (see ipath_kreceive())
+ */
+static int ipath_create_port0_egr(ipath_portdata * pd)
+{
+	int ret = 0;
+	uint64_t *egrbase, egroff;
+	unsigned e, egrcnt;
+	ipath_devdata *dd;
+	struct sk_buff **skbs;
+
+	dd = &devdata[pd->port_unit];
+	egrcnt = dd->ipath_rcvegrcnt;
+	egroff =
+	    dd->ipath_rcvegrbase + pd->port_port * egrcnt * sizeof(*egrbase);
+	egrbase = (uint64_t *) ((char *)(dd->ipath_kregbase) + egroff);
+	_IPATH_VDBG
+	    ("unit%u Allocating %d egr buffers, at chip offset %llx (%p)\n",
+	     pd->port_unit, egrcnt, egroff, egrbase);
+
+	skbs = vmalloc(sizeof(*dd->ipath_port0_skbs) * egrcnt);
+	if (skbs == NULL)
+		ret = -ENOMEM;
+	else {
+		for (e = 0; e < egrcnt; e++) {
+			/*
+			 * This is a bit tricky in that we allocate
+			 * extra space for 2 bytes of the 14 byte
+			 * ethernet header.  These two bytes are passed
+			 * in the ipath header so the rest of the data
+			 * is word aligned.  We allocate 4 bytes so that the
+			 * data buffer stays word aligned.
+			 * See ipath_kreceive() for more details.
+			 */
+			skbs[e] =
+			    __dev_alloc_skb(dd->ipath_ibmaxlen + 4, GFP_KERNEL);
+			if (skbs[e] == NULL) {
+				_IPATH_UNIT_ERROR(pd->port_unit,
+						  "SKB allocation error for eager TID %u\n",
+						  e);
+				while (e != 0)
+					dev_kfree_skb(skbs[--e]);
+				ret = -ENOMEM;
+				break;
+			}
+			skb_reserve(skbs[e], 4);
+		}
+	}
+	/*
+	 * after loop above, so we can test non-NULL
+	 * to see if ready to use at receive, etc.  Hope this fixes some
+	 * panics.
+	 */
+	dd->ipath_port0_skbs = skbs;
+
+	/*
+	 * have to tell chip each time we init it
+	 * even if we are re-using previous memory.
+	 */
+	if (!ret) {
+		uint64_t lenvalid;	/* in words */
+
+		lenvalid = (dd->ipath_ibmaxlen - pd->port_egrskip) >> 2;
+		lenvalid <<= INFINIPATH_RT_BUFSIZE_SHIFT;
+		lenvalid |= INFINIPATH_RT_VALID;
+		for (e = 0; e < egrcnt; e++) {
+			unsigned long pa, pent;
+
+			pa = virt_to_phys(dd->ipath_port0_skbs[e]->data);
+			pa += pd->port_egrskip;
+			if (!e && (pa & ~INFINIPATH_RT_ADDR_MASK))
+				_IPATH_INFO
+				    ("phys addr %lx has more than 40 bits, using only 40!!!\n",
+				     pa);
+			pent = (pa & INFINIPATH_RT_ADDR_MASK) | lenvalid;
+			/*
+			 * don't need this except extreme debugging,
+			 * but leaving to save future typing.
+			 * _IPATH_VDBG("egr[%d] %p <- %lx\n", e, &egrbase[e], pent);
+			 */
+			ipath_kput_memq(pd->port_unit, &egrbase[e], pent);
+		}
+		yield();	/* don't hog the cpu */
+	}
+
+	return ret;
+}
+
+/*
+ * this *must* be physically contiguous memory, and for now,
+ * that limits it to what kmalloc can do.
+ */
+static int ipath_create_rcvhdrq(ipath_portdata * pd)
+{
+	int i, ret = 0, amt, order, pgs;
+	char *qt;
+	struct page *p;
+	unsigned long pa, pa0;
+
+	amt = round_up(devdata[pd->port_unit].ipath_rcvhdrcnt
+		       * devdata[pd->port_unit].ipath_rcvhdrentsize *
+		       sizeof(uint32_t), PAGE_SIZE);
+	if (!pd->port_rcvhdrq) {
+		order = get_order(amt);
+		/*
+		 * not using REPEAT isn't viable; at 128KB, we can easily fail
+		 * this.  The problem with REPEAT is we can block here
+		 * "forever".  There isn't an inbetween, unfortunately.
+		 * We could reduce the risk by never freeing the rcvhdrq
+		 * except at unload, but even then, the first time a
+		 * port is used, we could delay for some time...
+		 */
+		p = alloc_pages(GFP_USER, order);
+		if (!p) {
+			_IPATH_UNIT_ERROR(pd->port_unit,
+					  "attempt to allocate order %u memory for port %u rcvhdrq failed\n",
+					  order, pd->port_port);
+			return -ENOMEM;
+		}
+
+		/*
+		 * should use kmap (and later kunmap), even though high mem will
+		 * always be mapped on x86_64, to play it safe, but for some
+		 * bizarre reason these aren't exported symbols...
+		 */
+		pd->port_rcvhdrq = page_address(p);
+		if (!virt_addr_valid(pd->port_rcvhdrq)) {
+			_IPATH_DBG
+			    ("weird, virt_addr_valid false right after alloc_pages\n");
+			_IPATH_DBG("__pa(%p) is %lx, num_physpages %lx\n",
+				   pd->port_rcvhdrq, __pa(pd->port_rcvhdrq),
+				   num_physpages);
+		}
+		pd->port_rcvhdrq_phys = virt_to_phys(pd->port_rcvhdrq);
+		pd->port_rcvhdrq_order = order;
+
+		pa0 = pd->port_rcvhdrq_phys;
+		pgs = amt >> PAGE_SHIFT;
+		_IPATH_VDBG
+		    ("%d pages at %p (phys %lx) order=%u for port %u rcvhdr Q\n",
+		     pgs, pd->port_rcvhdrq, pa0, pd->port_rcvhdrq_order,
+		     pd->port_port);
+
+		/*
+		 * verify it's really physically contiguous, to be paranoid
+		 * also mark pages as reserved, to avoid problems when
+		 * user process with them mapped then exits.
+		 */
+		qt = pd->port_rcvhdrq;
+		SetPageReserved(virt_to_page(qt));
+		qt += PAGE_SIZE;
+		for (pa = pa0, i = 1; i < pgs; i++, qt += PAGE_SIZE) {
+			SetPageReserved(virt_to_page(qt));
+			pa = virt_to_phys(qt);
+			if (pa != (pa0 + (i * PAGE_SIZE)))
+				_IPATH_INFO
+				    ("pg %d at %p phys %lx not contiguous\n", i,
+				     qt, pa);
+			else
+				_IPATH_VDBG("pg %d at %p phys %lx\n", i, qt,
+					    pa);
+		}
+	}
+
+	/*
+	 * clear for security, sanity, and/or debugging (each time we
+	 * use/reuse)
+	 */
+	memset(pd->port_rcvhdrq, 0, amt);
+
+	/*
+	 * tell chip each time we init it, even if we are re-using previous
+	 * memory (we zero it at process close)
+	 */
+	_IPATH_VDBG("writing port %d rcvhdraddr as %lx\n", pd->port_port,
+		    pd->port_rcvhdrq_phys);
+	ipath_kput_kreg_port(pd->port_unit, kr_rcvhdraddr, pd->port_port,
+			     pd->port_rcvhdrq_phys);
+
+	return ret;
+}
+
+#ifdef _IPATH_EXTRA_DEBUG
+/*
+ * occasionally useful to dump the full set of kernel registers for debugging.
+ */
+static void ipath_dump_allregs(char *what, ipath_type t)
+{
+	uint16_t reg;
+	_IPATH_DBG("%s\n", what);
+	for (reg = 0; reg <= 0x100; reg++) {
+		uint64_t v = ipath_kget_kreg64(t, reg);
+		if (!(reg % 4))
+			printk("\n%3x: ", reg);
+		printk("%16llx  ", v);
+	}
+	printk("\n");
+}
+#endif				/* _IPATH_EXTRA_DEBUG */
+
+/*
+ * Do the actual initialization sequence on the chip.  For the real
+ * hardware, this is done from the init routine called from the PCI
+ * infrastructure.
+ */
+int ipath_init_chip(const ipath_type t)
+{
+	int ret = 0, i;
+	uint32_t val32, kpiobufs;
+	uint64_t val, atmp;
+	volatile uint32_t *piobuf;
+	uint32_t pioincr;
+	ipath_devdata *dd = &devdata[t];
+	ipath_portdata *pd;
+	struct page *vpage;
+	char boardn[32];
+
+	/* first time only, set after static version info */
+	if (!chip_driver_version) {
+		i = strlen(ipath_core_version);
+		chip_driver_version = ipath_core_version + i;
+		chip_driver_size = sizeof ipath_core_version - i;
+	}
+
+	/*
+	 * have to clear shadow copies of registers at init that are not
+	 * otherwise set here, or all kinds of bizarre things happen with
+	 * driver on chip reset
+	 */
+	dd->ipath_rcvhdrsize = 0;
+
+	/*
+	 * don't clear ipath_flags as 8bit mode was set before entering
+	 * this func.  However, we do set the linkstate to unknown
+	 */
+
+	/* so we can watch for a transition */
+	dd->ipath_flags |= IPATH_LINKUNK;
+	dd->ipath_flags &= ~(IPATH_LINKACTIVE | IPATH_LINKARMED | IPATH_LINKDOWN
+			     | IPATH_LINKINIT);
+
+	_IPATH_VDBG("Try to read spc chip revision\n");
+	dd->ipath_revision = ipath_kget_kreg64(t, kr_revision);
+
+	/*
+	 * set up fundamental info we need to use the chip;  we assume if
+	 * the revision reg and these regs are OK, we don't need to special
+	 * case the rest
+	 */
+	dd->ipath_sregbase = ipath_kget_kreg32(t, kr_sendregbase);
+	dd->ipath_cregbase = ipath_kget_kreg32(t, kr_counterregbase);
+	dd->ipath_uregbase = ipath_kget_kreg32(t, kr_userregbase);
+	_IPATH_VDBG("ipath_kregbase %p, sendbase %x usrbase %x, cntrbase %x\n",
+		    dd->ipath_kregbase, dd->ipath_sregbase, dd->ipath_uregbase,
+		    dd->ipath_cregbase);
+	if ((dd->ipath_revision & 0xffffffff) == 0xffffffff ||
+	    (dd->ipath_sregbase & 0xffffffff) == 0xffffffff ||
+	    (dd->ipath_cregbase & 0xffffffff) == 0xffffffff ||
+	    (dd->ipath_uregbase & 0xffffffff) == 0xffffffff) {
+		_IPATH_UNIT_ERROR(t,
+				  "Register read failures from chip, giving up initialization\n");
+		ret = -ENODEV;
+		goto done;
+	}
+
+	/* clear the initial reset flag, in case first driver load */
+	ipath_kput_kreg(t, kr_errorclear, INFINIPATH_E_RESET);
+
+	dd->ipath_portcnt = ipath_kget_kreg32(t, kr_portcnt);
+	if (!infinipath_cfgports)
+		dd->ipath_cfgports = dd->ipath_portcnt;
+	else if (infinipath_cfgports <= dd->ipath_portcnt) {
+		dd->ipath_cfgports = infinipath_cfgports;
+		_IPATH_DBG("Configured to use %u ports out of %u in chip\n",
+			   dd->ipath_cfgports, dd->ipath_portcnt);
+	} else {
+		dd->ipath_cfgports = dd->ipath_portcnt;
+		_IPATH_DBG
+		    ("Tried to configured to use %u ports; chip only supports %u\n",
+		     infinipath_cfgports, dd->ipath_portcnt);
+	}
+	dd->ipath_pd = kmalloc(sizeof(*dd->ipath_pd) * dd->ipath_cfgports,
+			       GFP_KERNEL);
+	if (!dd->ipath_pd) {
+		_IPATH_UNIT_ERROR(t,
+				  "Unable to allocate portdata array, failing\n");
+		ret = -ENOMEM;
+		goto done;
+	}
+	memset(dd->ipath_pd, 0, sizeof(*dd->ipath_pd) * dd->ipath_cfgports);
+
+	dd->ipath_lastegrheads = kmalloc(sizeof(*dd->ipath_lastegrheads)
+					 * dd->ipath_cfgports, GFP_KERNEL);
+	dd->ipath_lastrcvhdrqtails = kmalloc(sizeof(*dd->ipath_lastrcvhdrqtails)
+					     * dd->ipath_cfgports, GFP_KERNEL);
+	if (!dd->ipath_lastegrheads || !dd->ipath_lastrcvhdrqtails) {
+		_IPATH_UNIT_ERROR(t,
+				  "Unable to allocate head arrays, failing\n");
+		ret = -ENOMEM;
+		goto done;
+	}
+	memset(dd->ipath_lastrcvhdrqtails, 0,
+	       sizeof(*dd->ipath_lastrcvhdrqtails)
+	       * dd->ipath_cfgports);
+	memset(dd->ipath_lastegrheads, 0, sizeof(*dd->ipath_lastegrheads)
+	       * dd->ipath_cfgports);
+
+	dd->ipath_pd[0] = kmalloc(sizeof(ipath_portdata), GFP_KERNEL);
+	if (!dd->ipath_pd[0]) {
+		_IPATH_UNIT_ERROR(t,
+				  "Unable to allocate portdata for port 0, failing\n");
+		ret = -ENOMEM;
+		goto done;
+	}
+	memset(dd->ipath_pd[0], 0, sizeof(ipath_portdata));
+
+	pd = dd->ipath_pd[0];
+	pd->port_unit = t;
+	pd->port_port = 0;
+	pd->port_cnt = 1;
+	/* The port 0 pkey table is used by the layer interface. */
+	pd->port_pkeys[0] = IPS_DEFAULT_P_KEY;
+
+	dd->ipath_rcvtidcnt = ipath_kget_kreg32(t, kr_rcvtidcnt);
+	dd->ipath_rcvtidbase = ipath_kget_kreg32(t, kr_rcvtidbase);
+	dd->ipath_rcvegrcnt = ipath_kget_kreg32(t, kr_rcvegrcnt);
+	dd->ipath_rcvegrbase = ipath_kget_kreg32(t, kr_rcvegrbase);
+	dd->ipath_palign = ipath_kget_kreg32(t, kr_pagealign);
+	dd->ipath_piobufbase = ipath_kget_kreg32(t, kr_sendpiobufbase);
+	dd->ipath_piosize = ipath_kget_kreg32(t, kr_sendpiosize);
+	dd->ipath_ibmtu = 4096;	/* default to largest legal MTU */
+	dd->ipath_piobcnt = ipath_kget_kreg32(t, kr_sendpiobufcnt);
+
+	_IPATH_VDBG
+	    ("Revision %llx (PCI %x), %u ports, %u tids, %u egrtids, %u piobufs\n",
+	     dd->ipath_revision, dd->ipath_pcirev, dd->ipath_portcnt,
+	     dd->ipath_rcvtidcnt, dd->ipath_rcvegrcnt, dd->ipath_piobcnt);
+
+	if (((dd->ipath_revision >> INFINIPATH_R_SOFTWARE_SHIFT) & INFINIPATH_R_SOFTWARE_MASK) != IPATH_CHIP_SWVERSION) {	/* >= maybe, someday */
+		_IPATH_UNIT_ERROR(t,
+				  "Driver only handles version %d, chip swversion is %d (%llx), failng\n",
+				  IPATH_CHIP_SWVERSION,
+				  (int)(dd->
+					ipath_revision >>
+					INFINIPATH_R_SOFTWARE_SHIFT) &
+				  INFINIPATH_R_SOFTWARE_MASK,
+				  dd->ipath_revision);
+		ret = -ENOSYS;
+		goto done;
+	}
+	dd->ipath_majrev = (uint8_t) ((dd->ipath_revision >>
+				       INFINIPATH_R_CHIPREVMAJOR_SHIFT) &
+				      INFINIPATH_R_CHIPREVMAJOR_MASK);
+	dd->ipath_minrev =
+	    (uint8_t) ((dd->
+			ipath_revision >> INFINIPATH_R_CHIPREVMINOR_SHIFT) &
+		       INFINIPATH_R_CHIPREVMINOR_MASK);
+	dd->ipath_boardrev =
+	    (uint8_t) ((dd->
+			ipath_revision >> INFINIPATH_R_BOARDID_SHIFT) &
+		       INFINIPATH_R_BOARDID_MASK);
+
+	ipath_get_boardname(t, boardn, sizeof boardn);
+
+	{
+		snprintf(chip_driver_version, chip_driver_size,
+			 "Driver %u.%u, %s, InfiniPath%u %u.%u, PCI %u, SW Compat %u\n",
+			 IPATH_CHIP_VERS_MAJ, IPATH_CHIP_VERS_MIN, boardn,
+			 (unsigned)(dd->
+				    ipath_revision >> INFINIPATH_R_ARCH_SHIFT) &
+			 INFINIPATH_R_ARCH_MASK, dd->ipath_majrev,
+			 dd->ipath_minrev, dd->ipath_pcirev,
+			 (unsigned)(dd->
+				    ipath_revision >>
+				    INFINIPATH_R_SOFTWARE_SHIFT) &
+			 INFINIPATH_R_SOFTWARE_MASK);
+
+	}
+
+	_IPATH_DBG("%s", chip_driver_version);
+
+	/*
+	 * we ignore most issues after reporting them, but have to specially
+	 * handle hardware-disabled chips.
+	 */
+	if(ipath_validate_rev(dd) == 2) {
+		ret = -EPERM; /* unique error, known to infinipath_init_one() */
+		goto done;
+	}
+
+	/*
+	 * zero all the TID entries at startup.  We do this for sanity,
+	 * in case of a previous driver crash of some kind, and also
+	 * because the chip powers up with these memories in an unknown
+	 * state.  Use portcnt, not cfgports, since this is for the full chip,
+	 * not for current (possibly different) configuration value
+	 * Chip Errata bug 6447
+	 */
+	for (val32 = 0; val32 < dd->ipath_portcnt; val32++)
+		ipath_clear_tids(t, val32);
+
+	dd->ipath_rcvhdrentsize = IPATH_RCVHDRENTSIZE;
+	/* we could bump this
+	 * to allow for full rcvegrcnt + rcvtidcnt, but then it no
+	 * longer nicely fits power of two, and since we now use
+	 * alloc_pages, the rest would be wasted.
+	 */
+	dd->ipath_rcvhdrcnt = dd->ipath_rcvegrcnt;
+	/*
+	 * setup offset of last valid entry in rcvhdrq, for various tests, to
+	 * avoid calculating each time we need it
+	 */
+	dd->ipath_hdrqlast =
+	    dd->ipath_rcvhdrentsize * (dd->ipath_rcvhdrcnt - 1);
+	ipath_kput_kreg(t, kr_rcvhdrentsize, dd->ipath_rcvhdrentsize);
+	ipath_kput_kreg(t, kr_rcvhdrcnt, dd->ipath_rcvhdrcnt);
+	/*
+	 * not in ipath_rcvhdrsize, so user programs can set differently, but
+	 * so any early packets see the default size.
+	 */
+	ipath_kput_kreg(t, kr_rcvhdrsize, IPATH_DFLT_RCVHDRSIZE);
+
+	/*
+	 * we "know" that this works
+	 * out OK.  It's actually a bit more than we need, but 2048+64 isn't
+	 * quite enough for full size, and we want the +N to be a power of 2
+	 * to give us reasonable alignment and fit within page_alloc()'ed
+	 * memory
+	 */
+	dd->ipath_rcvegrbufsize = dd->ipath_piosize;
+
+	/*
+	 * the min() check here is currently a nop, but it may not always be,
+	 * depending on just how we do ipath_rcvegrbufsize
+	 */
+	dd->ipath_ibmaxlen = min(dd->ipath_piosize, dd->ipath_rcvegrbufsize);
+	dd->ipath_init_ibmaxlen = dd->ipath_ibmaxlen;
+
+	/*
+	 * set up the shadow copies of the piobufavail registers, which
+	 * we compare against the chip registers for now, and the in
+	 * memory DMA'ed copies of the registers.  This has to be done
+	 * early, before we calculate lastport, etc.
+	 */
+	val = dd->ipath_piobcnt;
+	/*
+	 * calc number of pioavail registers, and save it; we have 2 bits
+	 * per buffer
+	 */
+	dd->ipath_pioavregs =
+	    round_up(val, sizeof(uint64_t) * _BITS_PER_BYTE / 2) /
+	    (sizeof(uint64_t) * _BITS_PER_BYTE / 2);
+	if (dd->ipath_pioavregs >
+	    (sizeof(dd->ipath_pioavailshadow) /
+	     sizeof(dd->ipath_pioavailshadow[0]))) {
+		dd->ipath_pioavregs =
+		    sizeof(dd->ipath_pioavailshadow) /
+		    sizeof(dd->ipath_pioavailshadow[0]);
+		dd->ipath_piobcnt = dd->ipath_pioavregs * sizeof(uint64_t) * _BITS_PER_BYTE >> 1;	/* 2 bits/reg */
+		_IPATH_INFO
+		    ("Warning: %lld piobufs is too many to fit in shadow, only using %d\n",
+		     val, dd->ipath_piobcnt);
+	}
+
+	if (!infinipath_kpiobufs) {
+		/* have to have at least one, for SMA */
+		kpiobufs = infinipath_kpiobufs = 1;
+	} else if (dd->ipath_piobcnt <
+		   (dd->ipath_cfgports * IPATH_MIN_USER_PORT_BUFCNT)) {
+		_IPATH_INFO
+		    ("Too few PIO buffers (%u) for %u ports to have %u each!\n",
+		     dd->ipath_piobcnt, dd->ipath_cfgports,
+		     IPATH_MIN_USER_PORT_BUFCNT);
+		kpiobufs = 1;	/* reserve just the minimum for SMA/ether */
+	} else
+		kpiobufs = infinipath_kpiobufs;
+
+	if (kpiobufs >
+	    (dd->ipath_piobcnt -
+	     (dd->ipath_cfgports * IPATH_MIN_USER_PORT_BUFCNT))) {
+		i = dd->ipath_piobcnt -
+		    (dd->ipath_cfgports * IPATH_MIN_USER_PORT_BUFCNT);
+		if (i < 0)
+			i = 0;
+		_IPATH_INFO
+		    ("Allocating %d PIO bufs for kernel leaves too few for %d user ports (%d each); using %u\n",
+		     kpiobufs, dd->ipath_cfgports - 1,
+		     IPATH_MIN_USER_PORT_BUFCNT, i);
+		/*
+		 * shouldn't change infinipath_kpiobufs, because could be
+		 * different for different devices...
+		 */
+		kpiobufs = i;
+	}
+	dd->ipath_lastport_piobuf = dd->ipath_piobcnt - kpiobufs;
+	dd->ipath_pbufsport = dd->ipath_cfgports > 1 ?
+	    dd->ipath_lastport_piobuf / (dd->ipath_cfgports - 1) : 0;
+	val32 = dd->ipath_lastport_piobuf -
+	    (dd->ipath_pbufsport * (dd->ipath_cfgports - 1));
+	if (val32 > 0) {
+		_IPATH_DBG
+		    ("allocating %u pbufs/port leaves %u unused, add to kernel\n",
+		     dd->ipath_pbufsport, val32);
+		dd->ipath_lastport_piobuf -= val32;
+		_IPATH_DBG("%u pbufs/port leaves %u unused, add to kernel\n",
+			   dd->ipath_pbufsport, val32);
+	}
+	dd->ipath_lastpioindex = dd->ipath_lastport_piobuf;
+	_IPATH_VDBG
+	    ("%d PIO bufs %u - %u, %u each for %u user ports\n",
+	     kpiobufs, dd->ipath_lastport_piobuf, dd->ipath_piobcnt, dd->ipath_pbufsport,
+	     dd->ipath_cfgports - 1);
+
+	/*
+	 * this has to be page aligned, and on a page of it's own, so we
+	 * can map it into user space.  We also use it to give processes
+	 * a copy of ipath_statusp, on a separate cacheline, followed by
+	 * a copy of the freeze error string, if it's happened.  Might also
+	 * use that space for other things.
+	 */
+	val = round_up(2 * L1_CACHE_BYTES + sizeof(*dd->ipath_statusp) +
+		       dd->ipath_pioavregs * sizeof(uint64_t), 2 * PAGE_SIZE);
+	if (!(dd->ipath_pioavailregs_dma = kmalloc(val * sizeof(uint64_t),
+						   GFP_KERNEL))) {
+		_IPATH_UNIT_ERROR(t,
+				  "failed to allocate PIOavail reg area in memory\n");
+		ret = -ENOMEM;
+		goto done;
+	}
+	if ((PAGE_SIZE - 1) & (uint64_t) dd->ipath_pioavailregs_dma) {
+		dd->__ipath_pioavailregs_base = dd->ipath_pioavailregs_dma;
+		dd->ipath_pioavailregs_dma = (uint64_t *)
+		    round_up((uint64_t) dd->ipath_pioavailregs_dma, PAGE_SIZE);
+	} else
+		dd->__ipath_pioavailregs_base = dd->ipath_pioavailregs_dma;
+	/*
+	 * zero initial, since whole thing mapped
+	 * into user space, and don't want info leak, or confusing garbage
+	 */
+	memset((void *)dd->ipath_pioavailregs_dma, 0, PAGE_SIZE);
+
+	/*
+	 * we really want L2 cache aligned, but for current CPUs of interest,
+	 * they are the same.
+	 */
+	dd->ipath_statusp = (uint64_t *) ((char *)dd->ipath_pioavailregs_dma +
+					  ((2 * L1_CACHE_BYTES +
+					    dd->ipath_pioavregs *
+					    sizeof(uint64_t)) &
+					   ~L1_CACHE_BYTES));
+	/* copy the current value now that it's really allocated */
+	*dd->ipath_statusp = dd->_ipath_status;
+	/*
+	 * setup buffer to hold freeze msg, accessible to apps, following
+	 * statusp
+	 */
+	dd->ipath_freezemsg = (char *)&dd->ipath_statusp[1];
+	/* and it's length */
+	dd->ipath_freezelen = L1_CACHE_BYTES - sizeof(dd->ipath_statusp[0]);
+
+	atmp = virt_to_phys(dd->ipath_pioavailregs_dma);
+	/* stash physical address for user progs */
+	dd->ipath_pioavailregs_phys = atmp;
+	(void)ipath_kput_kreg(t, kr_sendpioavailaddr, atmp);
+	/*
+	 * this is to detect s/w errors, which the h/w works around by
+	 * ignoring the low 6 bits of address, if it wasn't aligned.
+	 */
+	val = ipath_kget_kreg64(t, kr_sendpioavailaddr);
+	if (val != atmp) {
+		_IPATH_UNIT_ERROR(t,
+				  "Catastrophic software error, SendPIOAvailAddr written as %llx, read back as %llx\n",
+				  atmp, val);
+		ret = -EINVAL;
+		goto done;
+	}
+
+	if (t * 64 > (sizeof(ipath_port0_rcvhdrtail) - 64)) {
+		_IPATH_UNIT_ERROR(t,
+				  "unit %u too large for port 0 rcvhdrtail buffer size\n",
+				  t);
+		ret = -ENODEV;
+	}
+
+	/*
+	 * kernel modules loaded into vmalloc'ed memory,
+	 * verify that when we assume that, map to phys, and back to virt,
+	 * that we get the right contents, so we did the mapping right.
+	 */
+	vpage = vmalloc_to_page((void *)ipath_port0_rcvhdrtail);
+	if (vpage == NOPAGE_SIGBUS || vpage == NOPAGE_OOM) {
+		_IPATH_UNIT_ERROR(t, "vmalloc_to_page for rcvhdrtail fails!\n");
+		ret = -ENOMEM;
+		goto done;
+	}
+
+	/*
+	 * 64 is driven by cache line size, and also by chip requirement
+	 * that low 6 bits be 0
+	 */
+	val = page_to_phys(vpage) + t * 64;
+
+	/* verify that the alignment requirement was met */
+	ipath_kput_kreg_port(t, kr_rcvhdrtailaddr, 0, val);
+	atmp = ipath_kget_kreg64_port(t, kr_rcvhdrtailaddr, 0);
+	if (val != atmp) {
+		_IPATH_UNIT_ERROR(t,
+				  "Catastrophic software error, RcvHdrTailAddr0 written as %llx, read back as %llx from %x\n",
+				  val, atmp, kr_rcvhdrtailaddr);
+		ret = -EINVAL;
+		goto done;
+	}
+	/* so we can get current tail in ipath_kreceive(), per chip */
+	dd->ipath_hdrqtailptr =
+	    &ipath_port0_rcvhdrtail[t *
+				    (64 / sizeof(ipath_port0_rcvhdrtail[0]))];
+
+	ipath_kput_kreg(t, kr_rcvbthqp, IPATH_KD_QP);
+
+	/*
+	 * make sure we are not in freeze, and PIO send enabled, so
+	 * writes to pbc happen
+	 */
+	ipath_kput_kreg(t, kr_hwerrmask, 0ULL);
+	ipath_kput_kreg(t, kr_hwerrclear, ~0ULL);
+	ipath_kput_kreg(t, kr_control, 0ULL);
+	ipath_kput_kreg(t, kr_sendctrl, INFINIPATH_S_PIOENABLE);
+
+	/*
+	 * write the pbc of each buffer, to be sure it's initialized, then
+	 * cancel all the buffers, and also abort any packets that might
+	 * have been in flight for some reason (the latter is for driver
+	 * unload/reload, but isn't a bad idea at first init).
+	 * PIO send isn't enabled at this point, so there is no danger
+	 * of sending these out on the wire.
+	 * Chip Errata bug 6610
+	 */
+	piobuf = (uint32_t *) (((char *)(dd->ipath_kregbase)) +
+			       dd->ipath_piobufbase);
+	pioincr = devdata[t].ipath_palign / sizeof(*piobuf);
+	for (i = 0; i < dd->ipath_piobcnt; i++) {
+		*piobuf = 16;	/* reasonable word count, just to init pbc */
+		piobuf += pioincr;
+	}
+	/* self-clearing */
+	ipath_kput_kreg(t, kr_sendctrl, INFINIPATH_S_ABORT);
+
+	/*
+	 * before error clears, since we expect serdes pll errors during
+	 * this, the first time after reset
+	 */
+	if (ipath_bringup_link(t)) {
+		_IPATH_INFO("Failed to bringup IB link\n");
+		ret = -ENETDOWN;
+		goto done;
+	}
+
+	/*
+	 * clear any "expected" hwerrs from reset and/or initialization
+	 * clear any that aren't enabled (at least this once), and then
+	 * set the enable mask
+	 */
+	ipath_clear_init_hwerrs(t);
+	ipath_kput_kreg(t, kr_hwerrclear, ~0ULL);
+	ipath_kput_kreg(t, kr_hwerrmask, dd->ipath_hwerrmask);
+
+	dd->ipath_maskederrs = dd->ipath_ignorederrs;
+	ipath_kput_kreg(t, kr_errorclear, ~0ULL);	/* clear all */
+	/* enable errors that are masked, at least this first time. */
+	ipath_kput_kreg(t, kr_errormask, ~dd->ipath_maskederrs);
+	/* clear any interrups up to this point (ints still not enabled) */
+	ipath_kput_kreg(t, kr_intclear, ~0ULL);
+
+	ipath_stats.sps_lid[t] = dd->ipath_lid;
+
+	/*
+	 * allocate the shadow TID array, so we can ipath_munlock
+	 * previous entries. It make make more sense to move the pageshadow
+	 * to the port data structure, so we only allocate memory for ports
+	 * actually in use, since we at 8k per port, now
+	 */
+	dd->ipath_pageshadow = (struct page **)
+	    vmalloc(dd->ipath_cfgports * dd->ipath_rcvtidcnt *
+		    sizeof(struct page *));
+	if (!dd->ipath_pageshadow)
+		_IPATH_UNIT_ERROR(t,
+				  "failed to allocate shadow page * array, no expected sends!\n");
+	else
+		memset(dd->ipath_pageshadow, 0,
+		       dd->ipath_cfgports * dd->ipath_rcvtidcnt *
+		       sizeof(struct page *));
+
+	/* set up the port 0 (kernel) rcvhdr q and egr TIDs */
+	if (!(ret = ipath_create_rcvhdrq(dd->ipath_pd[0])))
+		ret = ipath_create_port0_egr(dd->ipath_pd[0]);
+	if (ret)
+		_IPATH_UNIT_ERROR(t,
+				  "failed to allocate port 0 (kernel) rcvhdrq and/or egr bufs\n");
+	else {
+		init_waitqueue_head(&ipath_sma_wait);
+		init_waitqueue_head(&ipath_sma_state_wait);
+
+		ipath_kput_kreg(pd->port_unit, kr_rcvctrl, dd->ipath_rcvctrl);
+
+		ipath_kput_kreg(t, kr_rcvbthqp, IPATH_KD_QP);
+
+		/* Enable PIO send, and update of PIOavail regs to memory. */
+		dd->ipath_sendctrl = INFINIPATH_S_PIOENABLE
+		    | INFINIPATH_S_PIOBUFAVAILUPD;
+		ipath_kput_kreg(t, kr_sendctrl, dd->ipath_sendctrl);
+
+		/*
+		 * enable port 0 receive, and receive interrupt
+		 * other ports done as user opens and inits them
+		 */
+		dd->ipath_rcvctrl = INFINIPATH_R_TAILUPD |
+		    (1ULL << INFINIPATH_R_PORTENABLE_SHIFT) |
+		    (1ULL << INFINIPATH_R_INTRAVAIL_SHIFT);
+		ipath_kput_kreg(t, kr_rcvctrl, dd->ipath_rcvctrl);
+
+		/*
+		 * now ready for use
+		 * this should be cleared whenever we detect a reset, or
+		 * initiate one.
+		 */
+		dd->ipath_flags |= IPATH_INITTED;
+
+		/*
+		 * init our shadow copies of head from tail values, and write
+		 * head values to match
+		 */
+		val32 = ipath_kget_ureg32(t, ur_rcvegrindextail, 0);
+		(void)ipath_kput_ureg(t, ur_rcvegrindexhead, val32, 0);
+		dd->ipath_port0head = ipath_kget_ureg32(t, ur_rcvhdrtail, 0);
+		(void)ipath_kput_ureg(t, ur_rcvhdrhead, dd->ipath_port0head, 0);
+
+		/*
+		 * by now pioavail updates to memory should have occurred,
+		 * so copy them into our working/shadow registers; this is
+		 * in case something went wrong with abort, but mostly to
+		 * get the initial values of the generation bit correct
+		 */
+		for (i = 0; i < dd->ipath_pioavregs; i++) {
+			/*
+			 * Chip Errata bug 6641; even and odd qwords>3
+			 * are swapped
+			 */
+			if (i > 3) {
+				if (i & 1)
+					dd->ipath_pioavailshadow[i] =
+					    dd->ipath_pioavailregs_dma[i - 1];
+				else
+					dd->ipath_pioavailshadow[i] =
+					    dd->ipath_pioavailregs_dma[i + 1];
+			} else
+				dd->ipath_pioavailshadow[i] =
+				    dd->ipath_pioavailregs_dma[i];
+		}
+		/* can get counters, stats, etc. */
+		dd->ipath_flags |= IPATH_PRESENT;
+	}
+
+	/*
+	 * cause retrigger of pending interrupts ignored during init, even if
+	 * we had errors
+	 */
+	ipath_kput_kreg(t, kr_intclear, 0ULL);
+
+	/*
+	 * set up stats retrieval timer, even if we had errors in last
+	 * portion of setup
+	 */
+	init_timer(&dd->ipath_stats_timer);
+	dd->ipath_stats_timer.function = ipath_get_faststats;
+	dd->ipath_stats_timer.data = (unsigned long)t;
+	/* every 5 seconds; */
+	dd->ipath_stats_timer.expires = jiffies + 5 * HZ;
+	/* takes ~16 seconds to overflow at full IB 4x bandwdith */
+	add_timer(&dd->ipath_stats_timer);
+
+	dd->ipath_stats_timer_active = 1;
+
+done:
+	if (!ret) {
+		ipath_get_guid(t);
+		*dd->ipath_statusp |= IPATH_STATUS_CHIP_PRESENT;
+		if (!ipath_sma_data_spare) {
+			/* first init, setup SMA data structs */
+			ipath_sma_data_spare =
+			    ipath_sma_data_bufs[IPATH_NUM_SMAPKTS];
+			for (i = 0; i < IPATH_NUM_SMAPKTS; i++)
+				ipath_sma_data[i].buf = ipath_sma_data_bufs[i];
+		}
+		/*
+		 * sps_nports is a global, so, we set it to the highest
+		 * number of ports of any of the chips we find; we never
+		 * decrement it, at least for now.
+		 */
+		if (dd->ipath_cfgports > ipath_stats.sps_nports)
+			ipath_stats.sps_nports = dd->ipath_cfgports;
+	}
+	/* if ret is non-zero, we probably should do some cleanup here... */
+	return ret;
+}
+
+int ipath_waitfor_complete(const ipath_type t, ipath_kreg reg_id,
+			   uint64_t bits_to_wait_for, uint64_t * valp)
+{
+	uint64_t timeout, lastval, val;
+
+	lastval = ipath_kget_kreg64(t, reg_id);
+	timeout = get_cycles() + 0x10000000ULL;	/* <- ridiculously long time */
+	do {
+		val = ipath_kget_kreg64(t, reg_id);
+		*valp = val;	/* so they have something, even on failures. */
+		if ((val & bits_to_wait_for) == bits_to_wait_for)
+			return 0;
+		if (val != lastval)
+			_IPATH_VDBG
+			    ("Changed from %llx to %llx, waiting for %llx bits\n",
+			     lastval, val, bits_to_wait_for);
+		yield();
+		if (get_cycles() > timeout) {
+			_IPATH_DBG
+			    ("Didn't get bits %llx in register 0x%x, got %llx\n",
+			     bits_to_wait_for, reg_id, *valp);
+			return ENODEV;
+		}
+	} while (1);
+}
+
+/*
+ * like ipath_waitfor_complete(), but we wait for the CMDVALID bit to go away
+ * indicating the last command has completed.  It doesn't return data
+ */
+int ipath_waitfor_mdio_cmdready(const ipath_type t)
+{
+	uint64_t timeout;
+	uint64_t val;
+
+	timeout = get_cycles() + 0x10000000ULL;	/* <- ridiculously long time */
+	do {
+		val = ipath_kget_kreg64(t, kr_mdio);
+		if (!(val & IPATH_MDIO_CMDVALID))
+			return 0;
+		yield();
+		if (get_cycles() > timeout) {
+			_IPATH_DBG("CMDVALID stuck in mdio reg? (%llx)\n", val);
+			return ENODEV;
+		}
+	} while (1);
+}
+
+void ipath_set_ib_lstate(const ipath_type t, int which)
+{
+	ipath_devdata *dd = &devdata[t];
+	char *what;
+
+	/*
+	 * For all cases, we'll either be setting a new value of linkcmd, or
+	 * we want it to be NOP, so clear it here.
+	 * Similarly, we want the linkinitcmd to be NOP for everything
+	 * other than explictly than explictly changing linkinitcmd,
+	 * and for that case, we want to first clear any existing bits
+	 */
+	dd->ipath_ibcctrl &= ~((INFINIPATH_IBCC_LINKCMD_MASK <<
+				INFINIPATH_IBCC_LINKCMD_SHIFT) |
+			       (INFINIPATH_IBCC_LINKINITCMD_MASK <<
+				INFINIPATH_IBCC_LINKINITCMD_SHIFT));
+
+	if (which == INFINIPATH_IBCC_LINKCMD_INIT) {
+		dd->ipath_flags &= ~(IPATH_LINK_TOARMED | IPATH_LINK_TOACTIVE
+				     | IPATH_LINK_SLEEPING);
+		/* so we can watch for a transition */
+		dd->ipath_flags |= IPATH_LINKDOWN;
+		what = "INIT";
+	} else if (which == INFINIPATH_IBCC_LINKCMD_ARMED) {
+		dd->ipath_flags |= IPATH_LINK_TOARMED;
+		dd->ipath_flags &= ~(IPATH_LINK_TOACTIVE | IPATH_LINK_SLEEPING);
+		/*
+		 * this is mainly for loopback testing.  If INITCMD is
+		 * NOP or SLEEP, the link won't ever come up in loopback...
+		 */
+		if (!
+		    (dd->
+		     ipath_flags & (IPATH_LINKINIT | IPATH_LINKARMED |
+				    IPATH_LINKACTIVE))) {
+			_IPATH_SMADBG
+			    ("going to armed, but link not  yet up, set POLL\n");
+			dd->ipath_ibcctrl |=
+			    INFINIPATH_IBCC_LINKINITCMD_POLL <<
+			    INFINIPATH_IBCC_LINKINITCMD_SHIFT;
+		}
+		what = "ARMED";
+	} else if (which == INFINIPATH_IBCC_LINKCMD_ACTIVE) {
+		dd->ipath_flags |= IPATH_LINK_TOACTIVE;
+		dd->ipath_flags &= ~(IPATH_LINK_TOARMED | IPATH_LINK_SLEEPING);
+		what = "ACTIVE";
+	} else if (which & (INFINIPATH_IBCC_LINKINITCMD_MASK << INFINIPATH_IBCC_LINKINITCMD_SHIFT)) {	/* down, disable, etc. */
+		dd->ipath_flags &= ~(IPATH_LINK_TOARMED | IPATH_LINK_TOACTIVE);
+		if (((which & INFINIPATH_IBCC_LINKINITCMD_MASK) >>
+		     INFINIPATH_IBCC_LINKINITCMD_SHIFT) ==
+		    INFINIPATH_IBCC_LINKINITCMD_SLEEP) {
+			dd->ipath_flags |= IPATH_LINK_SLEEPING | IPATH_LINKDOWN;
+		} else
+			dd->ipath_flags |= IPATH_LINKDOWN;
+		dd->ipath_ibcctrl |=
+		    which & (INFINIPATH_IBCC_LINKINITCMD_MASK <<
+			     INFINIPATH_IBCC_LINKINITCMD_SHIFT);
+		what = "DOWN";
+	} else {
+		what = "UNKNOWN";
+		_IPATH_INFO("Unknown link transition requested (which=0x%x)\n",
+			    which);
+	}
+
+	dd->ipath_ibcctrl |= ((uint64_t) which & INFINIPATH_IBCC_LINKCMD_MASK)
+	    << INFINIPATH_IBCC_LINKCMD_SHIFT;
+
+	_IPATH_SMADBG("Trying to move unit %u to %s, current ltstate is %s\n",
+			t, what, ipath_ibcstatus_str[(ipath_kget_kreg64(t, kr_ibcstatus)
+					   >> INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT)
+					  & INFINIPATH_IBCS_LINKTRAININGSTATE_MASK]);
+	ipath_kput_kreg(t, kr_ibcctrl, dd->ipath_ibcctrl);
+}
+
+static int ipath_bringup_link(const ipath_type t)
+{
+	ipath_devdata *dd = &devdata[t];
+	uint64_t val, ibc;
+	int ret = 0;
+
+	dd->ipath_control &= ~INFINIPATH_C_LINKENABLE;	/* hold IBC in reset */
+	ipath_kput_kreg(t, kr_control, dd->ipath_control);
+
+	/*
+	 * Note that prior to try 14 or 15 of IB, the credit scaling
+	 * wasn't working, because it was swapped for writes with the
+	 * 1 bit default linkstate field
+	 */
+
+	/* ignore pbc and align word */
+	val = dd->ipath_piosize - 2 * sizeof(uint32_t);
+	/*
+	 * for ICRC, which we only send in diag test pkt mode, and we don't
+	 * need to worry about that for mtu
+	 */
+	val += 1;
+	/*
+	 * set the IBC maxpktlength to the size of our pio buffers
+	 * the maxpktlength is in words.  This is *not* the IB data MTU
+	 */
+	ibc = (val / sizeof(uint32_t)) << INFINIPATH_IBCC_MAXPKTLEN_SHIFT;
+	/* in KB */
+	ibc |= 0x5ULL << INFINIPATH_IBCC_FLOWCTRLWATERMARK_SHIFT;
+	/* how often flowctrl sent
+	 * more or less in usecs; balance against watermark value, so that
+	 * in theory senders always get a flow control update in time to not
+	 * let the IB link go idle.
+	 */
+	ibc |= 0x3ULL << INFINIPATH_IBCC_FLOWCTRLPERIOD_SHIFT;
+	/* max error tolerance */
+	ibc |= 0xfULL << INFINIPATH_IBCC_PHYERRTHRESHOLD_SHIFT;
+	/* use "real" buffer space for */
+	ibc |= 4ULL << INFINIPATH_IBCC_CREDITSCALE_SHIFT;
+	/* IB credit flow control. */
+	ibc |= 0xfULL << INFINIPATH_IBCC_OVERRUNTHRESHOLD_SHIFT;
+	/* initially come up waiting for TS1, without sending anything. */
+	dd->ipath_ibcctrl = ibc;
+	/* don't put linkinitcmd in ipath_ibcctrl, want that to stay a NOP */
+	ibc |=
+	    INFINIPATH_IBCC_LINKINITCMD_SLEEP <<
+	    INFINIPATH_IBCC_LINKINITCMD_SHIFT;
+	dd->ipath_flags |= IPATH_LINK_SLEEPING;
+	ipath_kput_kreg(t, kr_ibcctrl, ibc);
+
+	ret = ipath_bringup_serdes(t);
+
+	if (ret)
+		_IPATH_INFO("Could not initialize SerDes, not usable\n");
+	else {
+		dd->ipath_control |= INFINIPATH_C_LINKENABLE;	/* enable IBC */
+		ipath_kput_kreg(t, kr_control, dd->ipath_control);
+	}
+
+	return ret;
+}
+
+/*
+ * called from ipath_shutdown_link(), and from sma doing a LINKDOWN
+ * Left as a separate function for historical reasons, and may want
+ * it to do more than just call ipath_set_ib_lstate() again sometime
+ * in the future.
+ */
+void ipath_down_link(const ipath_type t)
+{
+	ipath_set_ib_lstate(t, INFINIPATH_IBCC_LINKINITCMD_SLEEP <<
+			    INFINIPATH_IBCC_LINKINITCMD_SHIFT);
+}
+
+/*
+ * do this when driver is being unloaded, or perhaps for diags, and
+ * maybe when we get an interrupt of a fatal link error that requires
+ * bringing the linkd down and back up
+ */
+static int ipath_shutdown_link(const ipath_type t)
+{
+	uint64_t val;
+	ipath_devdata *dd = &devdata[t];
+	int ret = 0;
+
+	_IPATH_DBG("Shutting down the link\n");
+	ipath_down_link(t);
+
+	/*
+	 * we are shutting down, so tell the layered driver.  We don't
+	 * do this on just a link state change, much like ethernet, 
+	 * a cable unplug, etc. doesn't change driver state
+	 */
+	if (dd->ipath_layer.l_intr)
+		dd->ipath_layer.l_intr(t, IPATH_LAYER_INT_IF_DOWN);
+
+	dd->ipath_control &= ~INFINIPATH_C_LINKENABLE;	/* disable IBC */
+	ipath_kput_kreg(t, kr_control, dd->ipath_control);
+
+	*dd->ipath_statusp &= ~(IPATH_STATUS_IB_CONF | IPATH_STATUS_IB_READY);
+
+	/*
+	 * clear SerdesEnable and turn the leds off; do this here because
+	 * we are unloading, so don't count on interrupts to move along
+	 */
+
+	ipath_quiet_serdes(t);
+	val = dd->ipath_extctrl &
+	    ~(INFINIPATH_EXTC_LEDPRIPORTGREENON |
+	      INFINIPATH_EXTC_LEDPRIPORTYELLOWON);
+	dd->ipath_extctrl = val;
+	ipath_kput_kreg(t, kr_extctrl, val);
+
+	if (dd->ipath_stats_timer_active) {
+		del_timer_sync(&dd->ipath_stats_timer);
+		dd->ipath_stats_timer_active = 0;
+	}
+	if (*dd->ipath_statusp & IPATH_STATUS_CHIP_PRESENT) {
+		/* can't do anything more with chip */
+		/* needs re-init */
+		*dd->ipath_statusp &= ~IPATH_STATUS_CHIP_PRESENT;
+		if (dd->ipath_kregbase) {
+			/*
+			 * if we haven't already cleaned up before these
+			 * are to ensure any register reads/writes "fail"
+			 * until re-init
+			 */
+			dd->ipath_kregbase = NULL;
+			dd->ipath_kregvirt = NULL;
+			dd->ipath_uregbase = 0ULL;
+			dd->ipath_sregbase = 0ULL;
+			dd->ipath_cregbase = 0ULL;
+			dd->ipath_kregsize = 0;
+		}
+#ifdef CONFIG_MTRR
+		if (dd->ipath_mtrr) {
+			_IPATH_VDBG("undoing WCCOMB on pio buffers\n");
+			mtrr_del(dd->ipath_mtrr, 0, 0);
+			dd->ipath_mtrr = 0;
+		}
+#endif
+	}
+
+	return ret;
+}
+
+/*
+ * when closing, free up any allocated data for a port, if the
+ * reference count goes to zero
+ * Note: this also frees the portdata itself!
+ */
+void ipath_free_pddata(ipath_devdata * dd, uint32_t port, int freehdrq)
+{
+	ipath_portdata *pd = dd->ipath_pd[port];
+
+	if (!pd)
+		return;
+	if (freehdrq)
+		/*
+		 * only clear and free portdata if we are going to
+		 * also release the hdrq, otherwise we leak the hdrq on each
+		 * open/close cycle
+		 */
+		dd->ipath_pd[port] = NULL;
+	/* cleanup locked pages private data structures */
+	ipath_mlock_cleanup(pd);
+	if (freehdrq && pd->port_rcvhdrq) {
+		int i, n = 1 << pd->port_rcvhdrq_order;
+		_IPATH_VDBG("free closed port %d rcvhdrq @ %p (order=%u)\n",
+			    pd->port_port, pd->port_rcvhdrq,
+			    pd->port_rcvhdrq_order);
+		for (i = 0; i < n; i++)
+			ClearPageReserved(virt_to_page
+					  (pd->port_rcvhdrq + (i * PAGE_SIZE)));
+		free_pages((unsigned long)pd->port_rcvhdrq,
+			   pd->port_rcvhdrq_order);
+		pd->port_rcvhdrq = NULL;
+	}
+	if (port && pd->port_rcvegrbuf_pages) {	/* always free this, however */
+		void *virt;
+		unsigned e, i, n = 1 << pd->port_rcvegrbuf_order;
+		if (pd->port_rcvegrbuf_virt) {
+			for (e = 0; e < pd->port_rcvegrbuf_chunks; e++) {
+				virt = pd->port_rcvegrbuf_virt[e];
+				for (i = 0; i < n; i++)
+					ClearPageReserved(virt_to_page
+							  (virt +
+							   (i * PAGE_SIZE)));
+				_IPATH_VDBG
+				    ("egrbuf free_pages(%p, %x), chunk %u/%u\n",
+				     virt, pd->port_rcvegrbuf_order, e,
+				     pd->port_rcvegrbuf_chunks);
+				free_pages((unsigned long)virt,
+					   pd->port_rcvegrbuf_order);
+			}
+			vfree(pd->port_rcvegrbuf_virt);
+			pd->port_rcvegrbuf_virt = NULL;
+		}
+		pd->port_rcvegrbuf_chunks = 0;
+		_IPATH_VDBG("free closed port %d rcvegrbufs ptr array\n",
+			    pd->port_port);
+		/* now the pointer array. */
+		vfree(pd->port_rcvegrbuf_pages);
+		pd->port_rcvegrbuf_pages = NULL;
+	} else if (port == 0 && dd->ipath_port0_skbs) {
+		unsigned e;
+		struct sk_buff **skbs = dd->ipath_port0_skbs;
+
+		dd->ipath_port0_skbs = NULL;
+		_IPATH_VDBG("free closed port %d ipath_port0_skbs @ %p\n",
+			    pd->port_port, skbs);
+		for (e = 0; e < dd->ipath_rcvegrcnt; e++)
+			if (skbs[e])
+				dev_kfree_skb(skbs[e]);
+		vfree(skbs);
+	}
+	if (freehdrq) {
+		kfree(pd->port_tid_pg_list);
+		kfree(pd);
+	}
+}
+
+int __init infinipath_init(void)
+{
+	int r = 0, i;
+
+	_IPATH_DBG(KERN_INFO DRIVER_LOAD_MSG "%s", ipath_core_version);
+
+	ipath_init_picotime();	/* init cycles -> pico conversion */
+
+	if (!ipath_ctl_header) {	/* should be always */
+		if (!(ipath_ctl_header = register_sysctl_table(ipath_ctl, 1)))
+			_IPATH_INFO("Couldn't register sysctl interface\n");
+	}
+
+	/*
+	 * initialize the statusp to temporary storage so we can use it
+	 * everywhere without first checking.  When we "really" assign it,
+	 * we copy from _ipath_status
+	 */
+	for (i = 0; i < infinipath_max; i++)
+		devdata[i].ipath_statusp = &devdata[i]._ipath_status;
+
+	/*
+	 * init these early, in case we take an interrupt as soon as the irq
+	 * is setup.  Saw a spinlock panic once that appeared to be due to that
+	 * problem, when they were initted later on.
+	 */
+	spin_lock_init(&ipath_pioavail_lock);
+	spin_lock_init(&ipath_sma_lock);
+
+	pci_register_driver(&infinipath_driver);
+
+	driver_create_file(&(infinipath_driver.driver), &driver_attr_version);
+
+	if ((r = register_chrdev(ipath_major, MODNAME, &ipath_fops)))
+		_IPATH_ERROR("Unable to register %s device\n", MODNAME);
+
+
+	/*
+	 * never return an error, since we could have stuff registered,
+	 * resources used, etc., even if no hardware found.  This way we
+	 * can clean up through unload.
+	 */
+	return 0;
+}
+
+/*
+ * note: if for some reason the unload fails after this routine, and leaves
+ * the driver enterable by user code, we'll almost certainly crash and burn...
+ */
+static void __exit infinipath_cleanup(void)
+{
+	int r, m, port;
+
+	driver_remove_file(&(infinipath_driver.driver), &driver_attr_version);
+	if (ipath_ctl_header) {
+		unregister_sysctl_table(ipath_ctl_header);
+		ipath_ctl_header = NULL;
+	} else
+		_IPATH_DBG("No sysctl unregister, not registered OK\n");
+	if ((r = unregister_chrdev(ipath_major, MODNAME)))
+		_IPATH_DBG("unregister of device failed: %d\n", r);
+
+
+	/*
+	 * turn off rcv, send, and interrupts for all ports, all drivers
+	 * should also hard reset the chip here?
+	 * free up port 0 (kernel) rcvhdr, egr bufs, and eventually tid bufs
+	 * for all versions of the driver, if they were allocated
+	 */
+	for (m = 0; m < infinipath_max; m++) {
+		uint64_t val;
+		ipath_devdata *dd = &devdata[m];
+		if (dd->ipath_kregbase) {
+			/* in case unload fails, be consistent */
+			dd->ipath_rcvctrl = 0U;
+			ipath_kput_kreg(m, kr_rcvctrl, dd->ipath_rcvctrl);
+
+			/*
+			 * gracefully stop all sends allowing any in
+			 * progress to trickle out first.
+			 */
+			ipath_kput_kreg(m, kr_sendctrl, 0ULL);
+			val = ipath_kget_kreg64(m, kr_scratch);	/* flush it */
+			/*
+			 * enough for anything that's going to trickle
+			 * out to have actually done so.
+			 */
+			udelay(5);
+
+			/*
+			 * abort any armed or launched PIO buffers that
+			 * didn't go. (self clearing).	Will cause any
+			 * packet currently being transmitted to go out
+			 * with an EBP, and may also cause a short packet
+			 * error on the receiver.
+			 */
+			ipath_kput_kreg(m, kr_sendctrl, INFINIPATH_S_ABORT);
+
+			/* mask interrupts, but not errors */
+			ipath_kput_kreg(m, kr_intmask, 0ULL);
+			ipath_shutdown_link(m);
+
+			/*
+			 * clear all interrupts and errors.  Next time
+			 * driver is loaded, we know that whatever is
+			 * set happened while we were unloaded
+			 */
+			ipath_kput_kreg(m, kr_hwerrclear, ~0ULL);
+			ipath_kput_kreg(m, kr_errorclear, ~0ULL);
+			ipath_kput_kreg(m, kr_intclear, ~0ULL);
+			if (dd->__ipath_pioavailregs_base) {
+				kfree((void *)dd->__ipath_pioavailregs_base);
+				dd->__ipath_pioavailregs_base =
+				    dd->ipath_pioavailregs_dma = 0;
+			}
+
+			if (dd->ipath_pageshadow) {
+				struct page **tmpp = dd->ipath_pageshadow;
+				int i, cnt = 0;
+
+				_IPATH_VDBG
+				    ("Unlocking any expTID pages still locked\n");
+				for (port = 0; port < dd->ipath_cfgports;
+				     port++) {
+					int port_tidbase =
+					    port * dd->ipath_rcvtidcnt;
+					int maxtid =
+					    port_tidbase + dd->ipath_rcvtidcnt;
+					for (i = port_tidbase; i < maxtid; i++) {
+						if (tmpp[i]) {
+							ipath_munlock(1,
+								      &tmpp[i]);
+							tmpp[i] = 0;
+							cnt++;
+						}
+					}
+				}
+				if (cnt) {
+					ipath_stats.sps_pageunlocks += cnt;
+					_IPATH_VDBG
+					    ("There were still %u expTID entries locked\n",
+					     cnt);
+				}
+				if (ipath_stats.sps_pagelocks
+				    || ipath_stats.sps_pageunlocks)
+					_IPATH_VDBG
+					    ("%llu pages locked, %llu unlocked via ipath_m{un}lock\n",
+					     ipath_stats.sps_pagelocks,
+					     ipath_stats.sps_pageunlocks);
+
+				_IPATH_VDBG
+				    ("Free shadow page tid array at %p\n",
+				     dd->ipath_pageshadow);
+				vfree(dd->ipath_pageshadow);
+				dd->ipath_pageshadow = NULL;
+			}
+
+			/*
+			 * free any resources still in use (usually just
+			 * kernel ports) at unload
+			 */
+			for (port = 0; port < dd->ipath_cfgports; port++)
+				ipath_free_pddata(dd, port, 1);
+			kfree(dd->ipath_pd);
+			/*
+			 * debuggability, in case some cleanup path
+			 * tries to use it after this
+			 */
+			dd->ipath_pd = NULL;
+		}
+
+		if (dd->pcidev) {
+			if (dd->pcidev->irq) {
+				_IPATH_VDBG("unit %u free_irq of irq %x\n", m,
+					    dd->pcidev->irq);
+				free_irq(dd->pcidev->irq, dd);
+			} else
+				_IPATH_DBG
+				    ("irq is 0, not doing free_irq for unit %u\n",
+				     m);
+			dd->pcidev = NULL;
+		}
+		if (dd->pci_registered) {
+			_IPATH_VDBG
+			    ("Unregistering pci infrastructure unit %u\n", m);
+			pci_unregister_driver(&infinipath_driver);
+			dd->pci_registered = 0;
+		} else
+			_IPATH_VDBG
+			    ("unit %u: no pci unreg, wasn't registered\n", m);
+		ipath_chip_cleanup(dd);	/* clean up any per-chip chip-specific stuff */
+	}
+	/*
+	 * clean up any chip-specific stuff for now, only one type of chip
+	 * for any given driver
+	 */
+	ipath_chip_done();
+
+	/* cleanup all our locked pages private data structures */
+	ipath_mlock_cleanup(NULL);
+}
+
+/* This is a generic function here, so it can return device-specific
+ * info.  This allows keeping in sync with the version that supports
+ * multiple chip types.
+*/
+void ipath_get_boardname(const ipath_type t, char *name, size_t namelen)
+{
+	ipath_ht_get_boardname(t, name, namelen);
+}
+
+module_init(infinipath_init);
+module_exit(infinipath_cleanup);
+
+EXPORT_SYMBOL(infinipath_debug);
+EXPORT_SYMBOL(ipath_get_boardname);
Signature · 1 line
0.99.9n
#8 lore raw

RFC PATCH 7/13 ipath core misc files

Misc ipath LLD files

---

 drivers/infiniband/hw/ipath/ipath_ht400.c | 1164 +++++++++++++++++++++++++++++
 drivers/infiniband/hw/ipath/ipath_i2c.c   |  472 ++++++++++++
 drivers/infiniband/hw/ipath/ipath_lib.c   |   92 ++
 drivers/infiniband/hw/ipath/ipath_mlock.c |  139 +++
 4 files changed, 1867 insertions(+), 0 deletions(-)
 create mode 100644 drivers/infiniband/hw/ipath/ipath_ht400.c
 create mode 100644 drivers/infiniband/hw/ipath/ipath_i2c.c
 create mode 100644 drivers/infiniband/hw/ipath/ipath_lib.c
 create mode 100644 drivers/infiniband/hw/ipath/ipath_mlock.c

8d0f07cdb6b7e4f243e31f5ac9fddb4225908062

drivers/infiniband/hw/ipath/ipath_ht400.c new file +1164 -0
new file mode 100644
index 0000000..2d5b795
@@ -0,0 +1,1164 @@
+/*
+ * Copyright (c) 2003, 2004, 2005. PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Patent licenses, if any, provided herein do not apply to
+ * combinations of this program with other software, or any other
+ * product whatsoever.
+ *
+ * $Id: ipath_ht400.c 4491 2005-12-15 22:20:31Z rjwalsh $
+ */
+
+/*
+ * The first part of this file is shared with the diags, the second
+ * part is used only in the kernel.
+ */
+
+#include <stddef.h>		/* for offsetof */
+
+#ifdef __KERNEL__
+#include <linux/types.h>
+#include <linux/time.h>
+#include <linux/timer.h>
+#include <linux/wait.h>
+#include "ipath_kernel.h"
+#else
+#include <sys/types.h>
+#include <stdint.h>
+#endif
+
+#include "ipath_registers.h"
+#include "ipath_common.h"
+
+/*
+ * This lists the InfiniPath registers, in the actual chip layout.  This
+ * structure should never be directly accessed.  It is included by the
+ * user mode diags, and so must be able to be compiled in both user
+ * and kernel mode.
+ */
+struct _infinipath_do_not_use_kernel_regs {
+	unsigned long long Revision;
+	unsigned long long Control;
+	unsigned long long PageAlign;
+	unsigned long long PortCnt;
+	unsigned long long DebugPortSelect;
+	unsigned long long DebugPort;
+	unsigned long long SendRegBase;
+	unsigned long long UserRegBase;
+	unsigned long long CounterRegBase;
+	unsigned long long Scratch;
+	unsigned long long ReservedMisc1;
+	unsigned long long InterruptConfig;
+	unsigned long long IntBlocked;
+	unsigned long long IntMask;
+	unsigned long long IntStatus;
+	unsigned long long IntClear;
+	unsigned long long ErrorMask;
+	unsigned long long ErrorStatus;
+	unsigned long long ErrorClear;
+	unsigned long long HwErrMask;
+	unsigned long long HwErrStatus;
+	unsigned long long HwErrClear;
+	unsigned long long HwDiagCtrl;
+	unsigned long long MDIO;
+	unsigned long long IBCStatus;
+	unsigned long long IBCCtrl;
+	unsigned long long ExtStatus;
+	unsigned long long ExtCtrl;
+	unsigned long long GPIOOut;
+	unsigned long long GPIOMask;
+	unsigned long long GPIOStatus;
+	unsigned long long GPIOClear;
+	unsigned long long RcvCtrl;
+	unsigned long long RcvBTHQP;
+	unsigned long long RcvHdrSize;
+	unsigned long long RcvHdrCnt;
+	unsigned long long RcvHdrEntSize;
+	unsigned long long RcvTIDBase;
+	unsigned long long RcvTIDCnt;
+	unsigned long long RcvEgrBase;
+	unsigned long long RcvEgrCnt;
+	unsigned long long RcvBufBase;
+	unsigned long long RcvBufSize;
+	unsigned long long RxIntMemBase;
+	unsigned long long RxIntMemSize;
+	unsigned long long RcvPartitionKey;
+	unsigned long long ReservedRcv[10];
+	unsigned long long SendCtrl;
+	unsigned long long SendPIOBufBase;
+	unsigned long long SendPIOSize;
+	unsigned long long SendPIOBufCnt;
+	unsigned long long SendPIOAvailAddr;
+	unsigned long long TxIntMemBase;
+	unsigned long long TxIntMemSize;
+	unsigned long long ReservedSend[9];
+	unsigned long long SendBufferError;
+	unsigned long long SendBufferErrorCONT1;
+	unsigned long long SendBufferErrorCONT2;
+	unsigned long long SendBufferErrorCONT3;
+	unsigned long long ReservedSBE[4];
+	unsigned long long RcvHdrAddr0;
+	unsigned long long RcvHdrAddr1;
+	unsigned long long RcvHdrAddr2;
+	unsigned long long RcvHdrAddr3;
+	unsigned long long RcvHdrAddr4;
+	unsigned long long RcvHdrAddr5;
+	unsigned long long RcvHdrAddr6;
+	unsigned long long RcvHdrAddr7;
+	unsigned long long RcvHdrAddr8;
+	unsigned long long ReservedRHA[7];
+	unsigned long long RcvHdrTailAddr0;
+	unsigned long long RcvHdrTailAddr1;
+	unsigned long long RcvHdrTailAddr2;
+	unsigned long long RcvHdrTailAddr3;
+	unsigned long long RcvHdrTailAddr4;
+	unsigned long long RcvHdrTailAddr5;
+	unsigned long long RcvHdrTailAddr6;
+	unsigned long long RcvHdrTailAddr7;
+	unsigned long long RcvHdrTailAddr8;
+	unsigned long long ReservedRHTA[7];
+	unsigned long long Sync;	/* Software only */
+	unsigned long long Dump;	/* Software only */
+	unsigned long long SimVer;	/* Software only */
+	unsigned long long ReservedSW[5];
+	unsigned long long SerdesConfig0;
+	unsigned long long SerdesConfig1;
+	unsigned long long SerdesStatus;
+	unsigned long long XGXSConfig;
+	unsigned long long ReservedSW2[4];
+};
+
+#ifdef __KERNEL__ /* kernel uses reg#; diags use offset in bytes, not reg # */
+#define IPATH_KREG_OFFSET(field) (offsetof(struct \
+    _infinipath_do_not_use_kernel_regs, field) / sizeof(uint64_t))
+#define IPATH_CREG_OFFSET(field) (offsetof( \
+    struct infinipath_counters, field) / sizeof(uint64_t))
+#else				/* diags */
+#define IPATH_KREG_OFFSET(field) (offsetof(struct \
+    _infinipath_do_not_use_kernel_regs, field))
+#define IPATH_CREG_OFFSET(field) (offsetof( \
+    struct infinipath_counters, field))
+#endif /* __KERNEL__ */
+
+ipath_kreg
+	kr_control = IPATH_KREG_OFFSET(Control),
+	kr_counterregbase = IPATH_KREG_OFFSET(CounterRegBase),
+	kr_debugport = IPATH_KREG_OFFSET(DebugPort),
+	kr_debugportselect = IPATH_KREG_OFFSET(DebugPortSelect),
+	kr_errorclear = IPATH_KREG_OFFSET(ErrorClear),
+	kr_errormask = IPATH_KREG_OFFSET(ErrorMask),
+	kr_errorstatus = IPATH_KREG_OFFSET(ErrorStatus),
+	kr_extctrl = IPATH_KREG_OFFSET(ExtCtrl),
+	kr_extstatus = IPATH_KREG_OFFSET(ExtStatus),
+	kr_gpio_clear = IPATH_KREG_OFFSET(GPIOClear),
+	kr_gpio_mask = IPATH_KREG_OFFSET(GPIOMask),
+	kr_gpio_out = IPATH_KREG_OFFSET(GPIOOut),
+	kr_gpio_status = IPATH_KREG_OFFSET(GPIOStatus),
+	kr_hwdiagctrl = IPATH_KREG_OFFSET(HwDiagCtrl),
+	kr_hwerrclear = IPATH_KREG_OFFSET(HwErrClear),
+	kr_hwerrmask = IPATH_KREG_OFFSET(HwErrMask),
+	kr_hwerrstatus = IPATH_KREG_OFFSET(HwErrStatus),
+	kr_ibcctrl = IPATH_KREG_OFFSET(IBCCtrl),
+	kr_ibcstatus = IPATH_KREG_OFFSET(IBCStatus),
+	kr_intblocked = IPATH_KREG_OFFSET(IntBlocked),
+	kr_intclear = IPATH_KREG_OFFSET(IntClear),
+	kr_interruptconfig = IPATH_KREG_OFFSET(InterruptConfig),
+	kr_intmask = IPATH_KREG_OFFSET(IntMask),
+	kr_intstatus = IPATH_KREG_OFFSET(IntStatus),
+	kr_mdio = IPATH_KREG_OFFSET(MDIO),
+	kr_pagealign = IPATH_KREG_OFFSET(PageAlign),
+	kr_partitionkey = IPATH_KREG_OFFSET(RcvPartitionKey),
+	kr_portcnt = IPATH_KREG_OFFSET(PortCnt),
+	kr_rcvbthqp = IPATH_KREG_OFFSET(RcvBTHQP),
+	kr_rcvbufbase = IPATH_KREG_OFFSET(RcvBufBase),
+	kr_rcvbufsize = IPATH_KREG_OFFSET(RcvBufSize),
+	kr_rcvctrl = IPATH_KREG_OFFSET(RcvCtrl),
+	kr_rcvegrbase = IPATH_KREG_OFFSET(RcvEgrBase),
+	kr_rcvegrcnt = IPATH_KREG_OFFSET(RcvEgrCnt),
+	kr_rcvhdrcnt = IPATH_KREG_OFFSET(RcvHdrCnt),
+	kr_rcvhdrentsize = IPATH_KREG_OFFSET(RcvHdrEntSize),
+	kr_rcvhdrsize = IPATH_KREG_OFFSET(RcvHdrSize),
+	kr_rcvintmembase = IPATH_KREG_OFFSET(RxIntMemBase),
+	kr_rcvintmemsize = IPATH_KREG_OFFSET(RxIntMemSize),
+	kr_rcvtidbase = IPATH_KREG_OFFSET(RcvTIDBase),
+	kr_rcvtidcnt = IPATH_KREG_OFFSET(RcvTIDCnt),
+	kr_revision = IPATH_KREG_OFFSET(Revision),
+	kr_scratch = IPATH_KREG_OFFSET(Scratch),
+	kr_sendbuffererror = IPATH_KREG_OFFSET(SendBufferError),
+	kr_sendbuffererror1 = IPATH_KREG_OFFSET(SendBufferErrorCONT1),
+	kr_sendbuffererror2 = IPATH_KREG_OFFSET(SendBufferErrorCONT2),
+	kr_sendbuffererror3 = IPATH_KREG_OFFSET(SendBufferErrorCONT3),
+	kr_sendctrl = IPATH_KREG_OFFSET(SendCtrl),
+	kr_sendpioavailaddr = IPATH_KREG_OFFSET(SendPIOAvailAddr),
+	kr_sendpiobufbase = IPATH_KREG_OFFSET(SendPIOBufBase),
+	kr_sendpiobufcnt = IPATH_KREG_OFFSET(SendPIOBufCnt),
+	kr_sendpiosize = IPATH_KREG_OFFSET(SendPIOSize),
+	kr_sendregbase = IPATH_KREG_OFFSET(SendRegBase),
+	kr_txintmembase = IPATH_KREG_OFFSET(TxIntMemBase),
+	kr_txintmemsize = IPATH_KREG_OFFSET(TxIntMemSize),
+	kr_userregbase = IPATH_KREG_OFFSET(UserRegBase),
+	/* no simulator, register not used */
+	kr_sync = IPATH_KREG_OFFSET(Scratch),
+	/* no simulator, register not used */
+	kr_dump = IPATH_KREG_OFFSET(Scratch),
+	/* no simulator, register not used */
+	kr_simver = IPATH_KREG_OFFSET(Scratch),
+	/* onchip serdes */
+	kr_serdesconfig0 = IPATH_KREG_OFFSET(SerdesConfig0),
+	/* onchip serdes */
+	kr_serdesconfig1 = IPATH_KREG_OFFSET(SerdesConfig1),
+	/* onchip serdes */
+	kr_serdesstatus = IPATH_KREG_OFFSET(SerdesStatus),
+	/* onchip serdes */
+	kr_xgxsconfig = IPATH_KREG_OFFSET(XGXSConfig),
+	/*
+	 * last valid direct use register other than diag-only registers
+	 */
+	__kr_lastvaliddirect = IPATH_KREG_OFFSET(ReservedSW2[0]),
+	/* always invalid for initializing */
+	__kr_invalid = IPATH_KREG_OFFSET(ReservedSW2[0]) + 1,
+	/*
+	 * These should not be used directly via ipath_kget_kreg64(),
+	 * use them with ipath_kget_kreg64_port()
+	 */
+	kr_rcvhdraddr = IPATH_KREG_OFFSET(RcvHdrAddr0),	/* not for direct use */
+	/* not for direct use */
+	kr_rcvhdrtailaddr = IPATH_KREG_OFFSET(RcvHdrTailAddr0),
+	/* we define the full set for the diags, the kernel doesn't use them */
+	kr_rcvhdraddr1 = IPATH_KREG_OFFSET(RcvHdrAddr1),
+	kr_rcvhdraddr2 = IPATH_KREG_OFFSET(RcvHdrAddr2),
+	kr_rcvhdraddr3 = IPATH_KREG_OFFSET(RcvHdrAddr3),
+	kr_rcvhdraddr4 = IPATH_KREG_OFFSET(RcvHdrAddr4),
+	kr_rcvhdrtailaddr1 = IPATH_KREG_OFFSET(RcvHdrTailAddr1),
+	kr_rcvhdrtailaddr2 = IPATH_KREG_OFFSET(RcvHdrTailAddr2),
+	kr_rcvhdrtailaddr3 = IPATH_KREG_OFFSET(RcvHdrTailAddr3),
+	kr_rcvhdrtailaddr4 = IPATH_KREG_OFFSET(RcvHdrTailAddr4),
+	kr_rcvhdraddr5 = IPATH_KREG_OFFSET(RcvHdrAddr5),
+	kr_rcvhdraddr6 = IPATH_KREG_OFFSET(RcvHdrAddr6),
+	kr_rcvhdraddr7 = IPATH_KREG_OFFSET(RcvHdrAddr7),
+	kr_rcvhdraddr8 = IPATH_KREG_OFFSET(RcvHdrAddr8),
+	kr_rcvhdrtailaddr5 = IPATH_KREG_OFFSET(RcvHdrTailAddr5),
+	kr_rcvhdrtailaddr6 = IPATH_KREG_OFFSET(RcvHdrTailAddr6),
+	kr_rcvhdrtailaddr7 = IPATH_KREG_OFFSET(RcvHdrTailAddr7),
+	kr_rcvhdrtailaddr8 = IPATH_KREG_OFFSET(RcvHdrTailAddr8);
+
+/*
+ * first of the pioavail registers, the total number is
+ * (kr_sendpiobufcnt / 32); each buffer uses 2 bits
+ * More properly, it's:
+ *   (kr_sendpiobufcnt / ((sizeof(uint64_t)*BITS_PER_BYTE)/2))
+ */
+ipath_sreg sr_sendpioavail = 0;
+
+ipath_creg
+	cr_badformatcnt = IPATH_CREG_OFFSET(RxBadFormatCnt),
+	cr_erricrccnt = IPATH_CREG_OFFSET(RxICRCErrCnt),
+	cr_errlinkcnt = IPATH_CREG_OFFSET(RxLinkProblemCnt),
+	cr_errlpcrccnt = IPATH_CREG_OFFSET(RxLPCRCErrCnt),
+	cr_errpkey = IPATH_CREG_OFFSET(RxPKeyMismatchCnt),
+	cr_errrcvflowctrlcnt = IPATH_CREG_OFFSET(RxFlowCtrlErrCnt),
+	cr_err_rlencnt = IPATH_CREG_OFFSET(RxLenErrCnt),
+	cr_errslencnt = IPATH_CREG_OFFSET(TxLenErrCnt),
+	cr_errtidfull = IPATH_CREG_OFFSET(RxTIDFullErrCnt),
+	cr_errtidvalid = IPATH_CREG_OFFSET(RxTIDValidErrCnt),
+	cr_errvcrccnt = IPATH_CREG_OFFSET(RxVCRCErrCnt),
+	cr_ibstatuschange = IPATH_CREG_OFFSET(IBStatusChangeCnt),
+	/* calc from Reg_CounterRegBase + offset */
+	cr_intcnt = IPATH_CREG_OFFSET(LBIntCnt),
+	cr_invalidrlencnt = IPATH_CREG_OFFSET(RxMaxMinLenErrCnt),
+	cr_invalidslencnt = IPATH_CREG_OFFSET(TxMaxMinLenErrCnt),
+	cr_lbflowstallcnt = IPATH_CREG_OFFSET(LBFlowStallCnt),
+	cr_pktrcvcnt = IPATH_CREG_OFFSET(RxDataPktCnt),
+	cr_pktrcvflowctrlcnt = IPATH_CREG_OFFSET(RxFlowPktCnt),
+	cr_pktsendcnt = IPATH_CREG_OFFSET(TxDataPktCnt),
+	cr_pktsendflowcnt = IPATH_CREG_OFFSET(TxFlowPktCnt),
+	cr_portovflcnt = IPATH_CREG_OFFSET(RxP0HdrEgrOvflCnt),
+	cr_portovflcnt1 = IPATH_CREG_OFFSET(RxP1HdrEgrOvflCnt),
+	cr_portovflcnt2 = IPATH_CREG_OFFSET(RxP2HdrEgrOvflCnt),
+	cr_portovflcnt3 = IPATH_CREG_OFFSET(RxP3HdrEgrOvflCnt),
+	cr_portovflcnt4 = IPATH_CREG_OFFSET(RxP4HdrEgrOvflCnt),
+	cr_portovflcnt5 = IPATH_CREG_OFFSET(RxP5HdrEgrOvflCnt),
+	cr_portovflcnt6 = IPATH_CREG_OFFSET(RxP6HdrEgrOvflCnt),
+	cr_portovflcnt7 = IPATH_CREG_OFFSET(RxP7HdrEgrOvflCnt),
+	cr_portovflcnt8 = IPATH_CREG_OFFSET(RxP8HdrEgrOvflCnt),
+	cr_rcvebpcnt = IPATH_CREG_OFFSET(RxEBPCnt),
+	cr_rcvovflcnt = IPATH_CREG_OFFSET(RxBufOvflCnt),
+	cr_senddropped = IPATH_CREG_OFFSET(TxDroppedPktCnt),
+	cr_sendstallcnt = IPATH_CREG_OFFSET(TxFlowStallCnt),
+	cr_sendunderruncnt = IPATH_CREG_OFFSET(TxUnderrunCnt),
+	cr_wordrcvcnt = IPATH_CREG_OFFSET(RxDwordCnt),
+	cr_wordsendcnt = IPATH_CREG_OFFSET(TxDwordCnt),
+	cr_unsupvlcnt = IPATH_CREG_OFFSET(TxUnsupVLErrCnt),
+	cr_rxdroppktcnt = IPATH_CREG_OFFSET(RxDroppedPktCnt),
+	cr_iblinkerrrecovcnt = IPATH_CREG_OFFSET(IBLinkErrRecoveryCnt),
+	cr_iblinkdowncnt = IPATH_CREG_OFFSET(IBLinkDownedCnt),
+	cr_ibsymbolerrcnt = IPATH_CREG_OFFSET(IBSymbolErrCnt);
+
+/* kr_sendctrl bits */
+#define INFINIPATH_S_DISARMPIOBUF_MASK 0xFF
+
+/* kr_rcvctrl bits */
+#define INFINIPATH_R_PORTENABLE_MASK 0x1FF
+#define INFINIPATH_R_INTRAVAIL_MASK 0x1FF
+
+/* kr_intstatus, kr_intclear, kr_intmask bits */
+#define INFINIPATH_I_RCVURG_MASK 0x1FF
+#define INFINIPATH_I_RCVAVAIL_MASK 0x1FF
+
+/* kr_hwerrclear, kr_hwerrmask, kr_hwerrstatus, bits */
+#define INFINIPATH_HWE_HTCMEMPARITYERR_MASK 0x3FFFFFULL
+#define INFINIPATH_HWE_HTCLNKABYTE0CRCERR   0x0000000000800000ULL
+#define INFINIPATH_HWE_HTCLNKABYTE1CRCERR   0x0000000001000000ULL
+#define INFINIPATH_HWE_HTCLNKBBYTE0CRCERR   0x0000000002000000ULL
+#define INFINIPATH_HWE_HTCLNKBBYTE1CRCERR   0x0000000004000000ULL
+#define INFINIPATH_HWE_HTCMISCERR4        0x0000000008000000ULL
+#define INFINIPATH_HWE_HTCMISCERR5          0x0000000010000000ULL
+#define INFINIPATH_HWE_HTCMISCERR6          0x0000000020000000ULL
+#define INFINIPATH_HWE_HTCMISCERR7          0x0000000040000000ULL
+#define INFINIPATH_HWE_MEMBISTFAILED        0x0040000000000000ULL
+#define INFINIPATH_HWE_COREPLL_FBSLIP       0x0080000000000000ULL
+#define INFINIPATH_HWE_COREPLL_RFSLIP       0x0100000000000000ULL
+#define INFINIPATH_HWE_HTBPLL_FBSLIP        0x0200000000000000ULL
+#define INFINIPATH_HWE_HTBPLL_RFSLIP        0x0400000000000000ULL
+#define INFINIPATH_HWE_HTAPLL_FBSLIP        0x0800000000000000ULL
+#define INFINIPATH_HWE_HTAPLL_RFSLIP        0x1000000000000000ULL
+#define INFINIPATH_HWE_EXTSERDESPLLFAILED   0x2000000000000000ULL
+
+/* kr_hwdiagctrl bits */
+#define INFINIPATH_DC_NUMHTMEMS 22
+
+/* kr_extstatus bits */
+#define INFINIPATH_EXTS_FREQSEL 0x2
+#define INFINIPATH_EXTS_SERDESSEL 0x4
+#define INFINIPATH_EXTS_MEMBIST_ENDTEST     0x0000000000004000
+#define INFINIPATH_EXTS_MEMBIST_CORRECT     0x0000000000008000
+
+/* kr_extctrl bits */
+
+/*
+ * masks and bits that are different in different chips, or present only
+ * in one
+ */
+const uint32_t infinipath_i_rcvavail_mask = INFINIPATH_I_RCVAVAIL_MASK;
+const uint32_t infinipath_i_rcvurg_mask = INFINIPATH_I_RCVURG_MASK;
+const uint64_t infinipath_hwe_htcmemparityerr_mask =
+	INFINIPATH_HWE_HTCMEMPARITYERR_MASK;
+
+const uint64_t infinipath_hwe_spibdcmlockfailed_mask = 0ULL;
+const uint64_t infinipath_hwe_sphtdcmlockfailed_mask = 0ULL;
+const uint64_t infinipath_hwe_htcdcmlockfailed_mask = 0ULL;
+const uint64_t infinipath_hwe_htcdcmlockfailed_shift = 0ULL;
+const uint64_t infinipath_hwe_sphtdcmlockfailed_shift = 0ULL;
+const uint64_t infinipath_hwe_spibdcmlockfailed_shift = 0ULL;
+
+const uint64_t infinipath_hwe_htclnkabyte0crcerr =
+	INFINIPATH_HWE_HTCLNKABYTE0CRCERR;
+const uint64_t infinipath_hwe_htclnkabyte1crcerr =
+	INFINIPATH_HWE_HTCLNKABYTE1CRCERR;
+const uint64_t infinipath_hwe_htclnkbbyte0crcerr =
+	INFINIPATH_HWE_HTCLNKBBYTE0CRCERR;
+const uint64_t infinipath_hwe_htclnkbbyte1crcerr =
+	INFINIPATH_HWE_HTCLNKBBYTE1CRCERR;
+
+const uint64_t infinipath_c_bitsextant =
+	(INFINIPATH_C_FREEZEMODE | INFINIPATH_C_LINKENABLE);
+
+const uint64_t infinipath_s_bitsextant =
+	(INFINIPATH_S_ABORT | INFINIPATH_S_PIOINTBUFAVAIL |
+	 INFINIPATH_S_PIOBUFAVAILUPD | INFINIPATH_S_PIOENABLE |
+	 INFINIPATH_S_DISARM |
+	 (INFINIPATH_S_DISARMPIOBUF_MASK << INFINIPATH_S_DISARMPIOBUF_SHIFT));
+
+const uint64_t infinipath_r_bitsextant =
+	((INFINIPATH_R_PORTENABLE_MASK << INFINIPATH_R_PORTENABLE_SHIFT) |
+	 (INFINIPATH_R_INTRAVAIL_MASK << INFINIPATH_R_INTRAVAIL_SHIFT) |
+	 INFINIPATH_R_TAILUPD);
+
+const uint64_t infinipath_i_bitsextant =
+	((INFINIPATH_I_RCVURG_MASK << INFINIPATH_I_RCVURG_SHIFT) |
+	 (INFINIPATH_I_RCVAVAIL_MASK << INFINIPATH_I_RCVAVAIL_SHIFT) |
+	 INFINIPATH_I_ERROR | INFINIPATH_I_SPIOSENT |
+	 INFINIPATH_I_SPIOBUFAVAIL | INFINIPATH_I_GPIO);
+
+const uint64_t infinipath_e_bitsextant =
+	(INFINIPATH_E_RFORMATERR | INFINIPATH_E_RVCRC | INFINIPATH_E_RICRC |
+	 INFINIPATH_E_RMINPKTLEN | INFINIPATH_E_RMAXPKTLEN |
+	 INFINIPATH_E_RLONGPKTLEN | INFINIPATH_E_RSHORTPKTLEN |
+	 INFINIPATH_E_RUNEXPCHAR | INFINIPATH_E_RUNSUPVL | INFINIPATH_E_REBP |
+	 INFINIPATH_E_RIBFLOW | INFINIPATH_E_RBADVERSION |
+	 INFINIPATH_E_RRCVEGRFULL | INFINIPATH_E_RRCVHDRFULL |
+	 INFINIPATH_E_RBADTID | INFINIPATH_E_RHDRLEN |
+	 INFINIPATH_E_RHDR | INFINIPATH_E_RIBLOSTLINK |
+	 INFINIPATH_E_SMINPKTLEN | INFINIPATH_E_SMAXPKTLEN |
+	 INFINIPATH_E_SUNDERRUN | INFINIPATH_E_SPKTLEN |
+	 INFINIPATH_E_SDROPPEDSMPPKT | INFINIPATH_E_SDROPPEDDATAPKT |
+	 INFINIPATH_E_SPIOARMLAUNCH | INFINIPATH_E_SUNEXPERRPKTNUM |
+	 INFINIPATH_E_SUNSUPVL | INFINIPATH_E_IBSTATUSCHANGED |
+	 INFINIPATH_E_INVALIDADDR | INFINIPATH_E_RESET | INFINIPATH_E_HARDWARE);
+
+const uint64_t infinipath_hwe_bitsextant =
+	(INFINIPATH_HWE_HTCMEMPARITYERR_MASK <<
+	 INFINIPATH_HWE_HTCMEMPARITYERR_SHIFT) |
+	(INFINIPATH_HWE_TXEMEMPARITYERR_MASK <<
+	 INFINIPATH_HWE_TXEMEMPARITYERR_SHIFT) |
+	(INFINIPATH_HWE_RXEMEMPARITYERR_MASK <<
+	 INFINIPATH_HWE_RXEMEMPARITYERR_SHIFT) |
+	INFINIPATH_HWE_HTCLNKABYTE0CRCERR |
+	INFINIPATH_HWE_HTCLNKABYTE1CRCERR | INFINIPATH_HWE_HTCLNKBBYTE0CRCERR |
+	INFINIPATH_HWE_HTCLNKBBYTE1CRCERR | INFINIPATH_HWE_HTCMISCERR4 |
+	INFINIPATH_HWE_HTCMISCERR5 | INFINIPATH_HWE_HTCMISCERR6 |
+	INFINIPATH_HWE_HTCMISCERR7 | INFINIPATH_HWE_HTCBUSTREQPARITYERR |
+	INFINIPATH_HWE_HTCBUSTRESPPARITYERR |
+	INFINIPATH_HWE_HTCBUSIREQPARITYERR |
+	INFINIPATH_HWE_RXDSYNCMEMPARITYERR | INFINIPATH_HWE_MEMBISTFAILED |
+	INFINIPATH_HWE_COREPLL_FBSLIP | INFINIPATH_HWE_COREPLL_RFSLIP |
+	INFINIPATH_HWE_HTBPLL_FBSLIP | INFINIPATH_HWE_HTBPLL_RFSLIP |
+	INFINIPATH_HWE_HTAPLL_FBSLIP | INFINIPATH_HWE_HTAPLL_RFSLIP |
+	INFINIPATH_HWE_EXTSERDESPLLFAILED |
+	INFINIPATH_HWE_IBCBUSTOSPCPARITYERR |
+	INFINIPATH_HWE_IBCBUSFRSPCPARITYERR;
+
+const uint64_t infinipath_dc_bitsextant =
+	(INFINIPATH_DC_FORCEHTCMEMPARITYERR_MASK <<
+	 INFINIPATH_DC_FORCEHTCMEMPARITYERR_SHIFT) |
+	(INFINIPATH_DC_FORCETXEMEMPARITYERR_MASK <<
+	 INFINIPATH_DC_FORCETXEMEMPARITYERR_SHIFT) |
+	(INFINIPATH_DC_FORCERXEMEMPARITYERR_MASK <<
+	 INFINIPATH_DC_FORCERXEMEMPARITYERR_SHIFT) |
+	INFINIPATH_DC_FORCEHTCBUSTREQPARITYERR |
+	INFINIPATH_DC_FORCEHTCBUSTRESPPARITYERR |
+	INFINIPATH_DC_FORCEHTCBUSIREQPARITYERR |
+	INFINIPATH_DC_FORCERXDSYNCMEMPARITYERR |
+	INFINIPATH_DC_COUNTERDISABLE | INFINIPATH_DC_COUNTERWREN |
+	INFINIPATH_DC_FORCEIBCBUSTOSPCPARITYERR |
+	INFINIPATH_DC_FORCEIBCBUSFRSPCPARITYERR;
+
+const uint64_t infinipath_ibcc_bitsextant =
+	(INFINIPATH_IBCC_FLOWCTRLPERIOD_MASK <<
+	 INFINIPATH_IBCC_FLOWCTRLPERIOD_SHIFT) |
+	(INFINIPATH_IBCC_FLOWCTRLWATERMARK_MASK <<
+	 INFINIPATH_IBCC_FLOWCTRLWATERMARK_SHIFT) |
+	(INFINIPATH_IBCC_LINKINITCMD_MASK <<
+	 INFINIPATH_IBCC_LINKINITCMD_SHIFT) |
+	(INFINIPATH_IBCC_LINKCMD_MASK << INFINIPATH_IBCC_LINKCMD_SHIFT) |
+	(INFINIPATH_IBCC_MAXPKTLEN_MASK << INFINIPATH_IBCC_MAXPKTLEN_SHIFT) |
+	(INFINIPATH_IBCC_PHYERRTHRESHOLD_MASK <<
+	 INFINIPATH_IBCC_PHYERRTHRESHOLD_SHIFT) |
+	(INFINIPATH_IBCC_OVERRUNTHRESHOLD_MASK <<
+	 INFINIPATH_IBCC_OVERRUNTHRESHOLD_SHIFT) |
+	(INFINIPATH_IBCC_CREDITSCALE_MASK <<
+	 INFINIPATH_IBCC_CREDITSCALE_SHIFT) |
+	INFINIPATH_IBCC_LOOPBACK | INFINIPATH_IBCC_LINKDOWNDEFAULTSTATE;
+
+const uint64_t infinipath_mdio_bitsextant =
+	(INFINIPATH_MDIO_CLKDIV_MASK << INFINIPATH_MDIO_CLKDIV_SHIFT) |
+	(INFINIPATH_MDIO_COMMAND_MASK << INFINIPATH_MDIO_COMMAND_SHIFT) |
+	(INFINIPATH_MDIO_DEVADDR_MASK << INFINIPATH_MDIO_DEVADDR_SHIFT) |
+	(INFINIPATH_MDIO_REGADDR_MASK << INFINIPATH_MDIO_REGADDR_SHIFT) |
+	(INFINIPATH_MDIO_DATA_MASK << INFINIPATH_MDIO_DATA_SHIFT) |
+	INFINIPATH_MDIO_CMDVALID | INFINIPATH_MDIO_RDDATAVALID;
+
+const uint64_t infinipath_ibcs_bitsextant =
+	(INFINIPATH_IBCS_LINKTRAININGSTATE_MASK <<
+	 INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT) |
+	(INFINIPATH_IBCS_LINKSTATE_MASK << INFINIPATH_IBCS_LINKSTATE_SHIFT) |
+	INFINIPATH_IBCS_TXREADY | INFINIPATH_IBCS_TXCREDITOK;
+
+const uint64_t infinipath_extc_bitsextant =
+	(INFINIPATH_EXTC_GPIOINVERT_MASK << INFINIPATH_EXTC_GPIOINVERT_SHIFT) |
+	(INFINIPATH_EXTC_GPIOOE_MASK << INFINIPATH_EXTC_GPIOOE_SHIFT) |
+	INFINIPATH_EXTC_SERDESENABLE | INFINIPATH_EXTC_SERDESCONNECT |
+	INFINIPATH_EXTC_SERDESENTRUNKING | INFINIPATH_EXTC_SERDESDISRXFIFO |
+	INFINIPATH_EXTC_SERDESENPLPBK1 | INFINIPATH_EXTC_SERDESENPLPBK2 |
+	INFINIPATH_EXTC_SERDESENENCDEC | INFINIPATH_EXTC_LEDSECPORTGREENON |
+	INFINIPATH_EXTC_LEDSECPORTYELLOWON | INFINIPATH_EXTC_LEDPRIPORTGREENON |
+	INFINIPATH_EXTC_LEDPRIPORTYELLOWON | INFINIPATH_EXTC_LEDGBLOKGREENON |
+	INFINIPATH_EXTC_LEDGBLERRREDOFF;
+
+/*  Start of Documentation block for SerDes registers
+ *  serdes and xgxs register bits; not all have defines,
+ *  since I haven't yet needed them all, and I'm lazy.  Those that I needed
+ *  are in ipath_registers.h
+
+serdesConfig0Out  (R/W)
+                                        Default Value
+bit[3:0]        - ResetA/B/C/D          (4'b1111)
+bit[7:4]         -L1PwrdnA/B/C/D         (4'b0000)
+bit[11:8]       - RxIdleEnX             (4'b0000)
+bit[15:12]      - TxIdleEnX             (4'b0000)
+bit[19:16]      - RxDetectEnX           (4'b0000)
+bit[23:20]      - BeaconTxEnX           (4'b0000)
+bit[27:24]      - RxTermEnX             (4'b0000)
+bit[28]         - ResetPLL              (1'b0)
+bit[29]         -L2Pwrdn                (1'b0)
+bit[37:30]      - Offset[7:0]           (8'b00000000)
+bit[38]         -OffsetEn               (1'b0)
+bit[39]         -ParLBPK                (1'b0)
+bit[40]         -ParReset               (1'b0)
+bit[42:41]      - RefSel                (2'b10)
+bit[43]         - PW                    (1'b0)
+bit[47:44]      - LPBKA/B/C/D           (4'b0000)
+bit[49:48]      - ClkBufTermAdj         (2'b0)
+bit[51:50]      - RxTermAdj             (2'b0)
+bit[53:52]      - TxTermAdj             (2'b0)
+bit[55:54]      - RxEqCtl               (2'b0)
+bit[63:56]      - Reserved
+
+cce_wip_serdesConfig1Out[63:0]     (R/W)
+bit[3:0]        - HiDrvX                (4'b0000)
+bit[7:4]        - LoDrvX                (4'b0000)
+bit[12:11]      - DtxA[3:0]             (4'b0000)
+bit[15:12]      - DtxB[3:0]             (4'b0000)
+bit[19:16]      - DtxC[3:0]             (4'b0000)
+bit[23:20]      - DtxD[3:0]             (4'b0000)
+bit[27:24]      - DeqA[3:0]             (4'b0000)
+bit[31:28]      - DeqB[3:0]             (4'b0000)
+bit[35:32]      - DeqC[3:0]             (4'b0000)
+bit[39:36]      - DeqD[3:0]             (4'b0000)
+Framer interface, bits 40-59, not used
+bit[44:40]      - FmOffsetA[4:0]        (5'b00000)
+bit[49:45]      - FmOffsetB[4:0]        (5'b00000)
+bit[54:50]      - FmOffsetC[4:0]        (5'b00000)
+bit[59:55]      - FmOffsetD[4:0]        (5'b00000)
+bit[63:60]      - FmOffsetEnA/B/C/D     (4'b0000)
+
+SerdesStatus[63:0]      (RO)
+bit[3:0]        - TxIdleDetectA/B/C/D
+bit[7:4]        - RxDetectA/B/C/D
+bit[11:8]       - BeaconDetectA/B/C/D
+bit[63:12]      - Reserved
+
+XGXSConfigOut[63:0]
+bit[2:0]        - Resets, init to 1; bit 0 unused?
+bit[3]          - MDIO, select register bank for vendor specific register
+        (0x1e if set, else 0x1f); vendor-specific status in register 8
+        bits 0-3 lanes0-3 signal detect, 1 if detected
+        bits 4-7 lanes0-3 CTC fifo errors, 1 if detected (latched until read)
+bit[8:4]        - MDIO port address
+bit[18:9]       - lnk_sync_mask
+bit[22:19]      - polarity inv
+
+Documentation end */
+
+/*
+ *
+ * General specs:
+ *  ExtCtrl[63:48]   = EXTC_GPIOOE[15:0]
+ *  ExtCtrl[47:32]   = EXTC_GPIOInvert[15:0]
+ *  ExtStatus[63:48] = GpioIn[15:0]
+ *
+ * GPIO[1] = EEPROM_SDA
+ * GPIO[0] = EEPROM_SCL
+ */
+
+#define _IPATH_GPIO_SDA_NUM 1
+#define _IPATH_GPIO_SCL_NUM 0
+
+#define IPATH_GPIO_SDA \
+	(1UL << (_IPATH_GPIO_SDA_NUM+INFINIPATH_EXTC_GPIOOE_SHIFT))
+#define IPATH_GPIO_SCL \
+	(1UL << (_IPATH_GPIO_SCL_NUM+INFINIPATH_EXTC_GPIOOE_SHIFT))
+
+/*
+ * register bits for selecting i2c direction and values, used for I2C serial
+ * flash
+ */
+const uint16_t ipath_gpio_sda_num = _IPATH_GPIO_SDA_NUM;
+const uint16_t ipath_gpio_scl_num = _IPATH_GPIO_SCL_NUM;
+const uint64_t ipath_gpio_sda = IPATH_GPIO_SDA;
+const uint64_t ipath_gpio_scl = IPATH_GPIO_SCL;
+
+/* The remaining portion of this file is used only for the kernel */
+#ifdef __KERNEL__
+
+#include <linux/config.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/init.h>
+#include <linux/string.h>
+#include <linux/delay.h>
+
+/*
+ * This file contains all of the code that is specific to the InfiniPath
+ * HT-400 chip.
+ */
+
+/* we support up to 4 chips per system */
+const uint32_t infinipath_max = 4;
+ipath_devdata devdata[4];
+static const char *ipath_unit_names[4] = {
+	"infinipath0", "infinipath1", "infinipath2", "infinipath3"
+};
+
+const char *ipath_get_unit_name(int unit)
+{
+	return ipath_unit_names[unit];
+}
+
+static void ipath_check_htlink(ipath_type t);
+
+/*
+ * display hardware errors.  Use same msg buffer as regular errors to avoid
+ * excessive stack use.  Most hardware errors are catastrophic, but for 
+ * right now, we'll print them and continue.
+ * We reuse the same message buffer as ipath_handle_errors() to avoid
+ * excessive stack usage.
+ */
+void ipath_handle_hwerrors(const ipath_type t, char *msg, int msgl)
+{
+	uint64_t hwerrs = ipath_kget_kreg64(t, kr_hwerrstatus);
+	uint32_t bits, ctrl;
+	int isfatal = 0;
+	char bitsmsg[64];
+
+	if (!hwerrs) {
+		_IPATH_VDBG("Called but no hardware errors set\n");
+		/*
+		 * better than printing cofusing messages
+		 * This seems to be related to clearing the crc error, or
+		 * the pll error during init.
+		 */
+		return;
+	} else if (hwerrs == ~0ULL) {
+		_IPATH_UNIT_ERROR(t,
+				  "Read of hardware error status failed (all bits set); ignoring\n");
+		return;
+	}
+	ipath_stats.sps_hwerrs++;
+
+	/*
+	 * clear the error, regardless of whether we continue or stop using
+	 * the chip.
+	 */
+	ipath_kput_kreg(t, kr_hwerrclear, hwerrs);
+
+	hwerrs &= devdata[t].ipath_hwerrmask;
+
+	/*
+	 * make sure we get this much out, unless told to be quiet,
+	 * or it's occurred within the last 5 seconds
+	 */
+	if ((hwerrs & ~devdata[t].ipath_lasthwerror) ||
+	    (infinipath_debug & __IPATH_VERBDBG))
+		_IPATH_INFO("Hardware error: hwerr=0x%llx (cleared)\n", hwerrs);
+	devdata[t].ipath_lasthwerror |= hwerrs;
+
+	if (hwerrs & ~infinipath_hwe_bitsextant)
+		_IPATH_UNIT_ERROR(t,
+				  "hwerror interrupt with unknown errors %llx set\n",
+				  hwerrs & ~infinipath_hwe_bitsextant);
+
+	ctrl = ipath_kget_kreg32(t, kr_control);
+	if (ctrl & INFINIPATH_C_FREEZEMODE) {
+		if (hwerrs) {
+			/*
+			 * if any set that we aren't ignoring
+			 * only make the complaint once, in case it's stuck or
+			 * recurring, and we get here multiple times
+			 */
+			if (devdata[t].ipath_flags & IPATH_INITTED) {
+				_IPATH_UNIT_ERROR(t,
+						  "Fatal Error (freezemode), no longer usable\n");
+				isfatal = 1;
+			}
+			*devdata[t].ipath_statusp &= ~IPATH_STATUS_IB_READY;
+			/* mark as having had error */
+			*devdata[t].ipath_statusp |= IPATH_STATUS_HWERROR;
+			/*
+			 * mark as not usable, at a minimum until driver
+			 * is reloaded, probably until reboot, since no
+			 * other reset is possible.
+			 */
+			devdata[t].ipath_flags &= ~IPATH_INITTED;
+		} else {
+			_IPATH_DBG
+			    ("Clearing freezemode on ignored hardware error\n");
+			ctrl &= ~INFINIPATH_C_FREEZEMODE;
+			ipath_kput_kreg(t, kr_control, ctrl);
+		}
+	}
+
+	*msg = '\0';
+
+	/*
+	 * may someday want to decode into which bits are which
+	 * functional area for parity errors, etc.
+	 */
+	if (hwerrs & (infinipath_hwe_htcmemparityerr_mask
+		      << INFINIPATH_HWE_HTCMEMPARITYERR_SHIFT)) {
+		bits = (uint32_t) ((hwerrs >>
+				    INFINIPATH_HWE_HTCMEMPARITYERR_SHIFT) &
+				   INFINIPATH_HWE_HTCMEMPARITYERR_MASK);
+		snprintf(bitsmsg, sizeof bitsmsg, "[HTC Parity Errs %x] ",
+			 bits);
+		strlcat(msg, bitsmsg, msgl);
+	}
+	if (hwerrs & (INFINIPATH_HWE_RXEMEMPARITYERR_MASK
+		      << INFINIPATH_HWE_RXEMEMPARITYERR_SHIFT)) {
+		bits = (uint32_t) ((hwerrs >>
+				    INFINIPATH_HWE_RXEMEMPARITYERR_SHIFT) &
+				   INFINIPATH_HWE_RXEMEMPARITYERR_MASK);
+		snprintf(bitsmsg, sizeof bitsmsg, "[RXE Parity Errs %x] ",
+			 bits);
+		strlcat(msg, bitsmsg, msgl);
+	}
+	if (hwerrs & (INFINIPATH_HWE_TXEMEMPARITYERR_MASK
+		      << INFINIPATH_HWE_TXEMEMPARITYERR_SHIFT)) {
+		bits = (uint32_t) ((hwerrs >>
+				    INFINIPATH_HWE_TXEMEMPARITYERR_SHIFT) &
+				   INFINIPATH_HWE_TXEMEMPARITYERR_MASK);
+		snprintf(bitsmsg, sizeof bitsmsg, "[TXE Parity Errs %x] ",
+			 bits);
+		strlcat(msg, bitsmsg, msgl);
+	}
+	if (hwerrs & INFINIPATH_HWE_IBCBUSTOSPCPARITYERR)
+		strlcat(msg, "[IB2IPATH Parity]", msgl);
+	if (hwerrs & INFINIPATH_HWE_IBCBUSFRSPCPARITYERR)
+		strlcat(msg, "[IPATH2IB Parity]", msgl);
+	if (hwerrs & INFINIPATH_HWE_HTCBUSIREQPARITYERR)
+		strlcat(msg, "[HTC Ireq Parity]", msgl);
+	if (hwerrs & INFINIPATH_HWE_HTCBUSTREQPARITYERR)
+		strlcat(msg, "[HTC Treq Parity]", msgl);
+	if (hwerrs & INFINIPATH_HWE_HTCBUSTRESPPARITYERR)
+		strlcat(msg, "[HTC Tresp Parity]", msgl);
+
+/* keep the code below somewhat more readonable; not used elsewhere */
+#define _IPATH_HTLINK0_CRCBITS (infinipath_hwe_htclnkabyte0crcerr | \
+        infinipath_hwe_htclnkabyte1crcerr)
+#define _IPATH_HTLINK1_CRCBITS (infinipath_hwe_htclnkbbyte0crcerr | \
+        infinipath_hwe_htclnkbbyte1crcerr)
+#define _IPATH_HTLANE0_CRCBITS (infinipath_hwe_htclnkabyte0crcerr | \
+        infinipath_hwe_htclnkbbyte0crcerr)
+#define _IPATH_HTLANE1_CRCBITS (infinipath_hwe_htclnkabyte1crcerr | \
+        infinipath_hwe_htclnkbbyte1crcerr)
+	if (hwerrs & (_IPATH_HTLINK0_CRCBITS | _IPATH_HTLINK1_CRCBITS)) {
+		char bitsmsg[64];
+		uint64_t crcbits = hwerrs &
+		    (_IPATH_HTLINK0_CRCBITS | _IPATH_HTLINK1_CRCBITS);
+		/* don't check if 8bit HT */
+		if (devdata[t].ipath_flags & IPATH_8BIT_IN_HT0)
+			crcbits &= ~infinipath_hwe_htclnkabyte1crcerr;
+		/* don't check if 8bit HT */
+		if (devdata[t].ipath_flags & IPATH_8BIT_IN_HT1)
+			crcbits &= ~infinipath_hwe_htclnkbbyte1crcerr;
+		/*
+		 * we'll want to ignore link errors on link that is
+		 * not in use, if any.  For now, complain about both
+		 */
+		if (crcbits) {
+			uint16_t ctrl0, ctrl1;
+			snprintf(bitsmsg, sizeof bitsmsg,
+				 "[HT%s lane %s CRC (%llx); ignore till reload]",
+				 !(crcbits & _IPATH_HTLINK1_CRCBITS) ?
+				 "0 (A)" : (!(crcbits & _IPATH_HTLINK0_CRCBITS)
+					    ? "1 (B)" : "0+1 (A+B)"),
+				 !(crcbits & _IPATH_HTLANE1_CRCBITS) ? "0"
+				 : (!(crcbits & _IPATH_HTLANE0_CRCBITS) ? "1" :
+				    "0+1"), crcbits);
+			strlcat(msg, bitsmsg, msgl);
+
+			/*
+			 * print extra info for debugging.
+			 * slave/primary config word 4, 8 (link control 0, 1)
+			 */
+
+			if (pci_read_config_word(devdata[t].pcidev,
+						 devdata[t].ipath_ht_slave_off +
+						 0x4, &ctrl0))
+				_IPATH_INFO
+				    ("Couldn't read linkctrl0 of slave/primary config block\n");
+			else if (!(ctrl0 & 1 << 6))	/* not if EOC bit set */
+				_IPATH_DBG("HT linkctrl0 0x%x%s%s\n", ctrl0,
+					   ((ctrl0 >> 8) & 7) ? " CRC" : "",
+					   ((ctrl0 >> 4) & 1) ? "linkfail" :
+					   "");
+			if (pci_read_config_word
+			    (devdata[t].pcidev,
+			     devdata[t].ipath_ht_slave_off + 0x8, &ctrl1))
+				_IPATH_INFO
+				    ("Couldn't read linkctrl1 of slave/primary config block\n");
+			else if (!(ctrl1 & 1 << 6))	/* not if EOC bit set */
+				_IPATH_DBG("HT linkctrl1 0x%x%s%s\n", ctrl1,
+					   ((ctrl1 >> 8) & 7) ? " CRC" : "",
+					   ((ctrl1 >> 4) & 1) ? "linkfail" :
+					   "");
+
+			/* disable until driver reloaded */
+			devdata[t].ipath_hwerrmask &= ~crcbits;
+			ipath_kput_kreg(t, kr_hwerrmask,
+					devdata[t].ipath_hwerrmask);
+			_IPATH_DBG("HT crc errs: %s\n", msg);
+		} else
+			_IPATH_DBG
+			    ("ignoring HT crc errors 0x%llx, not in use\n",
+			     hwerrs & (_IPATH_HTLINK0_CRCBITS |
+				       _IPATH_HTLINK1_CRCBITS));
+	}
+
+	if (hwerrs & INFINIPATH_HWE_HTCMISCERR5)
+		strlcat(msg, "[HT core Misc5]", msgl);
+	if (hwerrs & INFINIPATH_HWE_HTCMISCERR6)
+		strlcat(msg, "[HT core Misc6]", msgl);
+	if (hwerrs & INFINIPATH_HWE_HTCMISCERR7)
+		strlcat(msg, "[HT core Misc7]", msgl);
+	if (hwerrs & INFINIPATH_HWE_MEMBISTFAILED) {
+		strlcat(msg, "[Memory BIST test failed, HT-400 unusable]",
+			msgl);
+		/* ignore from now on, so disable until driver reloaded */
+		devdata[t].ipath_hwerrmask &= ~INFINIPATH_HWE_MEMBISTFAILED;
+		ipath_kput_kreg(t, kr_hwerrmask, devdata[t].ipath_hwerrmask);
+	}
+#define _IPATH_PLL_FAIL (INFINIPATH_HWE_COREPLL_FBSLIP | \
+    INFINIPATH_HWE_COREPLL_RFSLIP | \
+    INFINIPATH_HWE_HTBPLL_FBSLIP | \
+    INFINIPATH_HWE_HTBPLL_RFSLIP | \
+    INFINIPATH_HWE_HTAPLL_FBSLIP | \
+    INFINIPATH_HWE_HTAPLL_RFSLIP)
+
+	if (hwerrs & _IPATH_PLL_FAIL) {
+		snprintf(bitsmsg, sizeof bitsmsg,
+			 "[PLL failed (%llx), HT-400 unusable]",
+			 hwerrs & _IPATH_PLL_FAIL);
+		strlcat(msg, bitsmsg, msgl);
+		/* ignore from now on, so disable until driver reloaded */
+		devdata[t].ipath_hwerrmask &= ~(hwerrs & _IPATH_PLL_FAIL);
+		ipath_kput_kreg(t, kr_hwerrmask, devdata[t].ipath_hwerrmask);
+	}
+
+	if (hwerrs & INFINIPATH_HWE_EXTSERDESPLLFAILED) {
+		/*
+		 * If it occurs, it is left masked since the eternal interface
+		 * is unused
+		 */
+		devdata[t].ipath_hwerrmask &=
+		    ~INFINIPATH_HWE_EXTSERDESPLLFAILED;
+		ipath_kput_kreg(t, kr_hwerrmask, devdata[t].ipath_hwerrmask);
+	}
+
+	if (hwerrs & INFINIPATH_HWE_RXDSYNCMEMPARITYERR)
+		strlcat(msg, "[Rx Dsync]", msgl);
+	if (hwerrs & INFINIPATH_HWE_SERDESPLLFAILED)
+		strlcat(msg, "[SerDes PLL]", msgl);
+
+	_IPATH_UNIT_ERROR(t, "%s hardware error\n", msg);
+	if (isfatal && (!ipath_diags_enabled)) {
+		if (devdata[t].ipath_freezemsg) {
+			/*
+			 * for proc status file ; if no trailing } is copied, we'll know
+			 * it was truncated.
+			 */
+			snprintf(devdata[t].ipath_freezemsg,
+				 devdata[t].ipath_freezelen, "{%s}", msg);
+		}
+	}
+}
+
+/* fill in the board name, based on the board revision register */
+void ipath_ht_get_boardname(const ipath_type t, char *name, size_t namelen)
+{
+	char *n = NULL;
+	uint8_t boardrev = devdata[t].ipath_boardrev;
+
+	switch (boardrev) {
+	case 4:		/* Ponderosa is one of the bringup boards */
+		n = "Ponderosa";
+		break;
+	case 5:		/* HT-460 original production board */
+		n = "InfiniPath_HT-460";
+		break;
+	case 7:		/* HT-460 small form factor production board */
+		n = "InfiniPath_HT-460-2";
+		break;
+	case 6:
+		n = "OEM_Board_3";
+		break;
+	case 8:
+		n = "LS/X-1";
+		break;
+	case 9:		/* Comstock bringup test board */
+		n = "Comstock";
+		break;
+	case 10:
+		n = "OEM_Board_2";
+		break;
+	case 11:
+		n = "OEM_Board_4";
+		break;
+	default:		/* don't know, just print the number */
+		_IPATH_ERROR("Don't yet know about board with ID %u\n",
+			     boardrev);
+		snprintf(name, namelen, "UnknownBoardRev%u", boardrev);
+		break;
+	}
+	if (n)
+		snprintf(name, namelen, "%s", n);
+}
+
+int ipath_validate_rev(ipath_devdata * dd)
+{
+	if (dd->ipath_majrev != 3 || dd->ipath_minrev != 2) {
+		/*
+		 * This version of the driver only supports the HT-400
+		 * Rev 3.2
+		 */
+		_IPATH_UNIT_ERROR(IPATH_UNIT(dd),
+				  "Unsupported HT-400 revision %u.%u!\n",
+				  dd->ipath_majrev, dd->ipath_minrev);
+		return 1;
+	}
+	if (dd->ipath_htspeed != 800)
+		_IPATH_UNIT_ERROR(IPATH_UNIT(dd),
+				  "Incorrectly configured for HT @ %uMHz\n",
+				  dd->ipath_htspeed);
+	if(dd->ipath_boardrev == 7 || dd->ipath_boardrev == 11 ||
+		dd->ipath_boardrev == 6)
+		dd->ipath_flags |= IPATH_GPIO_INTR;
+	else if (dd->ipath_boardrev == 8)  { /* LS/X-1 */
+		uint64_t val;
+		val = ipath_kget_kreg64(dd->ipath_pd[0]->port_unit, kr_extstatus);
+		if(val & INFINIPATH_EXTS_SERDESSEL) {  /* hardware disabled */
+			/* This means that the chip is hardware disabled, and will
+			 * not be able to bring up the link, in any case.  We special
+			 * case this and abort early, to avoid later messages.  We
+			 * also set the DISABLED status bit
+			 */
+			_IPATH_DBG("Unit %u is hardware-disabled\n", 
+				dd->ipath_pd[0]->port_unit);
+			*dd->ipath_statusp |= IPATH_STATUS_DISABLED;
+			return 2; /* this value is handled differently */
+		}
+	}
+	return 0;
+}
+
+static void ipath_check_htlink(ipath_type t)
+{
+	uint8_t linkerr, link_off, i;
+
+	for (i = 0; i < 2; i++) {
+		link_off = devdata[t].ipath_ht_slave_off + i * 4 + 0xd;
+		if (pci_read_config_byte(devdata[t].pcidev, link_off, &linkerr))
+			_IPATH_INFO
+			    ("Couldn't read linkerror%d of HT slave/primary block\n",
+			     i);
+		else if (linkerr & 0xf0) {
+			_IPATH_VDBG("HT linkerr%d bits 0x%x set, clearing\n",
+				    linkerr >> 4, i);
+			/*
+			 * writing the linkerr bits that are set should
+			 * clear them
+			 */
+			if (pci_write_config_byte
+			    (devdata[t].pcidev, link_off, linkerr))
+				_IPATH_DBG
+				    ("Failed write to clear HT linkerror%d\n",
+				     i);
+			if (pci_read_config_byte
+			    (devdata[t].pcidev, link_off, &linkerr))
+				_IPATH_INFO
+				    ("Couldn't reread linkerror%d of HT slave/primary block\n",
+				     i);
+			else if (linkerr & 0xf0)
+				_IPATH_INFO
+				    ("HT linkerror%d bits 0x%x couldn't be cleared\n",
+				     i, linkerr >> 4);
+		}
+	}
+}
+
+/*
+ * now that we have finished initializing everything that might reasonably
+ * cause a hardware error, and cleared those errors bits as they occur,
+ * we can enable hardware errors in the mask (potentially enabling
+ * freeze mode), and enable hardware errors as errors (along with
+ * everything else) in errormask
+ */
+void ipath_clear_init_hwerrs(ipath_type t)
+{
+	uint64_t val, extsval;
+
+	extsval = ipath_kget_kreg64(t, kr_extstatus);
+
+	if (!(extsval & INFINIPATH_EXTS_MEMBIST_ENDTEST))
+		_IPATH_UNIT_ERROR(t, "MemBIST did not complete!\n");
+
+	ipath_check_htlink(t);
+
+	/* barring bugs, all hwerrors become interrupts, which can */
+	val = ~0ULL;
+	/* don't look at crc lane1 if 8 bit */
+	if (devdata[t].ipath_flags & IPATH_8BIT_IN_HT0)
+		val &= ~infinipath_hwe_htclnkabyte1crcerr;
+	/* don't look at crc lane1 if 8 bit */
+	if (devdata[t].ipath_flags & IPATH_8BIT_IN_HT1)
+		val &= ~infinipath_hwe_htclnkbbyte1crcerr;
+
+	/*
+	 * disable RXDSYNCMEMPARITY because external serdes is unused,
+	 * and therefore the logic will never be used or initialized,
+	 * and uninitialized state will normally result in this error
+	 * being asserted.  Similarly for the external serdess pll
+	 * lock signal.
+	 */
+	val &=
+	    ~(INFINIPATH_HWE_EXTSERDESPLLFAILED |
+	      INFINIPATH_HWE_RXDSYNCMEMPARITYERR);
+
+	/*
+	 * Disable MISCERR4 because of an inversion in the HT core
+	 * logic checking for errors that cause this bit to be set.
+	 * The errata can also cause the protocol error bit to be set
+	 * in the HT config space linkerror register(s).
+	 */
+	val &= ~INFINIPATH_HWE_HTCMISCERR4;
+
+	/*
+	 * PLL ignored because MDIO interface has a logic problem for reads,
+	 * on Comstock and Ponderosa.  BRINGUP
+	 */
+	if (devdata[t].ipath_boardrev == 4 || devdata[t].ipath_boardrev == 9)
+		val &= ~INFINIPATH_HWE_EXTSERDESPLLFAILED;	/* BRINGUP */
+	devdata[t].ipath_hwerrmask = val;
+}
+
+/* bring up the serdes */
+int ipath_bringup_serdes(ipath_type t)
+{
+	uint64_t val, config1;
+	int ret = 0, change = 0;
+
+	_IPATH_DBG("Trying to bringup serdes\n");
+
+	if (ipath_kget_kreg64(t, kr_hwerrstatus) &
+	    INFINIPATH_HWE_SERDESPLLFAILED) {
+		_IPATH_DBG
+		    ("At start, serdes PLL failed bit set in hwerrstatus, clearing and continuing\n");
+		ipath_kput_kreg(t, kr_hwerrclear,
+				INFINIPATH_HWE_SERDESPLLFAILED);
+	}
+
+	val = ipath_kget_kreg64(t, kr_serdesconfig0);
+	config1 = ipath_kget_kreg64(t, kr_serdesconfig1);
+
+	_IPATH_VDBG
+	    ("Initial serdes status is config0=%llx config1=%llx, sstatus=%llx xgxs %llx\n",
+	     val, config1, ipath_kget_kreg64(t, kr_serdesstatus),
+	     ipath_kget_kreg64(t, kr_xgxsconfig));
+
+	/* force reset on */
+	val |=
+	    INFINIPATH_SERDC0_RESET_PLL /* | INFINIPATH_SERDC0_RESET_MASK */ ;
+	ipath_kput_kreg(t, kr_serdesconfig0, val);
+	udelay(15);		/* need pll reset set at least for a bit */
+
+	if (val & INFINIPATH_SERDC0_RESET_PLL) {
+		uint64_t val2 = val &= ~INFINIPATH_SERDC0_RESET_PLL;
+		/* set lane resets, and tx idle, during pll reset */
+		val2 |= INFINIPATH_SERDC0_RESET_MASK | INFINIPATH_SERDC0_TXIDLE;
+		_IPATH_VDBG("Clearing serdes PLL reset (writing %llx)\n", val2);
+		ipath_kput_kreg(t, kr_serdesconfig0, val2);
+		/* be sure chip saw it */
+		val = ipath_kget_kreg64(t, kr_scratch);
+		/*
+		 * need pll reset clear at least 11 usec before lane resets
+		 * cleared; give it a few more
+		 */
+		udelay(15);
+		val = val2;	/* for check below */
+	}
+
+	if (val & (INFINIPATH_SERDC0_RESET_PLL | INFINIPATH_SERDC0_RESET_MASK
+		   | INFINIPATH_SERDC0_TXIDLE)) {
+		val &=
+		    ~(INFINIPATH_SERDC0_RESET_PLL | INFINIPATH_SERDC0_RESET_MASK
+		      | INFINIPATH_SERDC0_TXIDLE);
+		ipath_kput_kreg(t, kr_serdesconfig0, val);	/* clear them */
+	}
+
+	val = ipath_kget_kreg64(t, kr_xgxsconfig);
+	if (((val >> INFINIPATH_XGXS_MDIOADDR_SHIFT) &
+	     INFINIPATH_XGXS_MDIOADDR_MASK) != 3) {
+		val &=
+		    ~(INFINIPATH_XGXS_MDIOADDR_MASK <<
+		      INFINIPATH_XGXS_MDIOADDR_SHIFT);
+		/* we use address 3 */
+		val |= 3ULL << INFINIPATH_XGXS_MDIOADDR_SHIFT;
+		change = 1;
+	}
+	if (val & INFINIPATH_XGXS_RESET) {	/* normally true after boot */
+		val &= ~INFINIPATH_XGXS_RESET;
+		change = 1;
+	}
+	if (change)
+		ipath_kput_kreg(t, kr_xgxsconfig, val);
+
+	val = ipath_kget_kreg64(t, kr_serdesconfig0);
+
+	config1 &= ~0x0ffffffff00ULL;	/* clear current and de-emphasis bits */
+	config1 |= 0x00000000000ULL;	/* set current to 20ma */
+	config1 |= 0x0cccc000000ULL;	/* set de-emphasis to -5.68dB */
+	ipath_kput_kreg(t, kr_serdesconfig1, config1);
+
+	_IPATH_VDBG
+	    ("After setup: serdes status is config0=%llx config1=%llx, sstatus=%llx xgxs %llx\n",
+	     val, config1, ipath_kget_kreg64(t, kr_serdesstatus),
+	     ipath_kget_kreg64(t, kr_xgxsconfig));
+
+	if ((!ipath_waitfor_mdio_cmdready(t))) {
+		ipath_kput_kreg(t, kr_mdio, IPATH_MDIO_REQ(IPATH_MDIO_CMD_READ,
+							   31,
+							   IPATH_MDIO_CTRL_XGXS_REG_8,
+							   0));
+		if (ipath_waitfor_complete
+		    (t, kr_mdio, IPATH_MDIO_DATAVALID, &val))
+			_IPATH_DBG
+			    ("Never got MDIO data for XGXS status read\n");
+		else
+			_IPATH_VDBG("MDIO Read reg8, 'bank' 31 %x\n",
+				    (uint32_t) val);
+	} else
+		_IPATH_DBG("Never got MDIO cmdready for XGXS status read\n");
+
+	return ret;		/* for now, say we always succeeded */
+}
+
+/* set serdes to txidle; driver is being unloaded */
+void ipath_quiet_serdes(const ipath_type t)
+{
+	uint64_t val = ipath_kget_kreg64(t, kr_serdesconfig0);
+
+	val |= INFINIPATH_SERDC0_TXIDLE;
+	_IPATH_DBG("Setting TxIdleEn on serdes (config0 = %llx)\n", val);
+	ipath_kput_kreg(t, kr_serdesconfig0, val);
+}
+
+EXPORT_SYMBOL(ipath_get_unit_name);
+
+#endif	/* __KERNEL__ */
drivers/infiniband/hw/ipath/ipath_i2c.c new file +472 -0

472 lines not on this page · raw message

drivers/infiniband/hw/ipath/ipath_lib.c new file +92 -0

92 lines not on this page · raw message

drivers/infiniband/hw/ipath/ipath_mlock.c new file +139 -0

139 lines not on this page · raw message

Signature · 1 line
0.99.9n
#9 lore raw

RFC PATCH 8/13 ipath core last bit

Last piece of ipath LLD

---

 drivers/infiniband/hw/ipath/ipath_layer.c | 1155 +++++++++++++++++++++++++++++
 1 files changed, 1155 insertions(+), 0 deletions(-)
 create mode 100644 drivers/infiniband/hw/ipath/ipath_layer.c

978ded82c9b5a4bca4e55f36d20ef4a585c50f38

drivers/infiniband/hw/ipath/ipath_layer.c new file +1155 -0
new file mode 100644
index 0000000..6a60851
@@ -0,0 +1,1155 @@
+/*
+ * Copyright (c) 2003, 2004, 2005. PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Patent licenses, if any, provided herein do not apply to
+ * combinations of this program with other software, or any other
+ * product whatsoever.
+ *
+ * $Id: ipath_layer.c 4365 2005-12-10 00:04:16Z rjwalsh $
+ */
+
+/*
+ * These are the routines used by layered drivers, currently just the
+ * layered ethernet driver and verbs layer.
+ */
+
+#include <linux/pci.h>
+
+#include "ipath_kernel.h"
+#include "ips_common.h"
+#include "ipath_layer.h"
+
+/* unit number is already validated in ipath_ioctl() */
+int ipath_kset_linkstate(uint32_t arg)
+{
+	ipath_type unit = 0xffff & (arg >> 16);
+	uint32_t lstate;
+	ipath_devdata *dd;
+
+	if (unit >= infinipath_max ||
+	    !(devdata[unit].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", unit);
+		return -ENODEV;
+	}
+
+	dd = &devdata[unit];
+	arg &= 0xffff;
+	if (arg != IPATH_IB_LINKDOWN && arg != IPATH_IB_LINKARM &&
+	    arg != IPATH_IB_LINKACTIVE) {
+		_IPATH_DBG("Unknown linkstate 0x%x requested\n", arg);
+		return -EINVAL;
+	}
+	if (arg == IPATH_IB_LINKDOWN) {
+		ipath_down_link(unit);	/* really moving it to idle */
+		lstate = IPATH_LINKDOWN | IPATH_LINK_SLEEPING;
+	} else if (arg == IPATH_IB_LINKARM) {
+		if (!(dd->ipath_flags &
+		      (IPATH_LINKINIT | IPATH_LINKARMED | IPATH_LINKDOWN |
+		       IPATH_LINK_SLEEPING | IPATH_LINKACTIVE)))
+			_IPATH_DBG
+			    ("don't know current state (flags 0x%x), try anyway\n",
+			     dd->ipath_flags);
+		ipath_set_ib_lstate(unit, INFINIPATH_IBCC_LINKCMD_ARMED);
+		lstate = IPATH_LINKARMED;
+	} else {
+		int tryarmed = 0;
+		/*
+		 * because we sometimes go to ARMED, but then back to 0x11
+		 * (initialized) before the SMA asks us to move to ACTIVE,
+		 * we will try to advance state to ARMED here, if necessary
+		 */
+		if (!(dd->ipath_flags &
+		      (IPATH_LINKINIT | IPATH_LINKARMED | IPATH_LINKDOWN |
+		       IPATH_LINK_SLEEPING | IPATH_LINKACTIVE))) {
+			/* this one is just paranoia */
+			_IPATH_DBG
+			    ("don't know current state (flags 0x%x), try anyway\n",
+			     dd->ipath_flags);
+			tryarmed = 1;
+
+		}
+		if (!(dd->ipath_flags & (IPATH_LINKARMED | IPATH_LINKACTIVE)))
+			tryarmed = 1;
+		if (tryarmed) {
+			ipath_set_ib_lstate(unit,
+					    INFINIPATH_IBCC_LINKCMD_ARMED);
+			/*
+			 * give it up to 2 seconds to get to ARMED or
+			 * ACTIVE; continue afterwards even if we fail
+			 */
+			if (ipath_wait_linkstate
+			    (unit, IPATH_LINKARMED | IPATH_LINKACTIVE, 2000))
+				_IPATH_VDBG
+				    ("try for active, even though didn't get to ARMED\n");
+		}
+
+		ipath_set_ib_lstate(unit, INFINIPATH_IBCC_LINKCMD_ACTIVE);
+		lstate = IPATH_LINKACTIVE;
+	}
+	return ipath_wait_linkstate(unit, lstate, 5000);
+}
+
+/*
+ * we can handle "any" incoming size, the issue here is whether we
+ * need to restrict our outgoing size.   For now, we don't do any
+ * sanity checking on this, and we don't deal with what happens to
+ * programs that are already running when the size changes.
+ * unit number is already validated in ipath_ioctl()
+ * NOTE: changing the MTU will usually cause the IBC to go back to
+ * link initialize (0x11) state...
+ */
+int ipath_kset_mtu(uint32_t arg)
+{
+	unsigned unit = (arg >> 16) & 0xffff;
+	uint32_t piosize;
+	int changed = 0;
+
+	if (unit >= infinipath_max ||
+	    !(devdata[unit].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", unit);
+		return -ENODEV;
+	}
+
+	arg &= 0xffff;
+	/*
+	 * mtu is IB data payload max.  It's the largest power of 2 less
+	 * than piosize (or even larger, since it only really controls the
+	 * largest we can receive; we can send the max of the mtu and piosize).
+	 * We check that it's one of the valid IB sizes.
+	 */
+	if (arg != 256 && arg != 512 && arg != 1024 && arg != 2048 &&
+	    arg != 4096) {
+		_IPATH_DBG("Trying to set invalid mtu %u, failing\n", arg);
+		return -EINVAL;
+	}
+	if (devdata[unit].ipath_ibmtu == arg) {
+		return 0;	/* same as current */
+	}
+
+	piosize = devdata[unit].ipath_ibmaxlen;
+	devdata[unit].ipath_ibmtu = arg;
+
+	/*
+	 * the 128 is the max IB header size allowed for in our pio send buffers
+	 * If we are reducing the MTU below that, this doesn't completely make
+	 * sense, but it's OK.
+	 */
+	if (arg >= (piosize - 128)) {
+		/* hasn't been changed */
+		if (piosize == devdata[unit].ipath_init_ibmaxlen)
+			_IPATH_VDBG
+			    ("mtu 0x%x >= ibmaxlen hardware max, nothing to do\n",
+			     arg);
+		else {
+			_IPATH_VDBG
+			    ("mtu 0x%x restores ibmaxlen to full amount 0x%x\n",
+			     arg, piosize);
+			devdata[unit].ipath_ibmaxlen = piosize;
+			changed = 1;
+		}
+	} else if ((arg + 128) == devdata[unit].ipath_ibmaxlen)
+		_IPATH_VDBG("ibmaxlen %x same as current, no change\n", arg);
+	else {
+		piosize = arg + 128;
+		_IPATH_VDBG("ibmaxlen was 0x%x, setting to 0x%x (mtu 0x%x)\n",
+			    devdata[unit].ipath_ibmaxlen, piosize, arg);
+		devdata[unit].ipath_ibmaxlen = piosize;
+		changed = 1;
+	}
+
+	if (changed) {
+		/*
+		 * set the IBC maxpktlength to the size of our pio
+		 * buffers in words
+		 */
+		uint64_t ibc = devdata[unit].ipath_ibcctrl;
+		ibc &= ~(INFINIPATH_IBCC_MAXPKTLEN_MASK <<
+			 INFINIPATH_IBCC_MAXPKTLEN_SHIFT);
+
+		piosize = piosize - 2 * sizeof(uint32_t);	/* ignore pbc */
+		devdata[unit].ipath_ibmaxlen = piosize;
+		piosize /= sizeof(uint32_t);	/* in words */
+		/*
+		 * for ICRC, which we only send in diag test pkt mode, and we
+		 * don't need to worry about that for mtu
+		 */
+		piosize += 1;
+
+		ibc |= piosize << INFINIPATH_IBCC_MAXPKTLEN_SHIFT;
+		devdata[unit].ipath_ibcctrl = ibc;
+		ipath_kput_kreg(unit, kr_ibcctrl, devdata[unit].ipath_ibcctrl);
+	}
+	return 0;
+}
+
+void ipath_set_sps_lid(const ipath_type unit, uint32_t arg)
+{
+	if (unit >= infinipath_max ||
+	    !(devdata[unit].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", unit);
+		return;
+	}
+
+	ipath_stats.sps_lid[unit] = devdata[unit].ipath_lid = arg;
+	if (devdata[unit].ipath_layer.l_intr)
+		devdata[unit].ipath_layer.l_intr(unit, IPATH_LAYER_INT_LID);
+}
+
+/* XXX - need to inform anyone who cares this just happened. */
+int ipath_layer_set_guid(const ipath_type device, uint64_t guid)
+{
+	if (device >= infinipath_max ||
+	    !(devdata[device].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return -ENODEV;
+	}
+	devdata[device].ipath_guid = guid;
+	return 0;
+}
+
+uint64_t ipath_layer_get_guid(const ipath_type device)
+{
+	if (device >= infinipath_max ||
+	    !(devdata[device].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return 0;
+	}
+	return devdata[device].ipath_guid;
+}
+
+uint32_t ipath_layer_get_nguid(const ipath_type device)
+{
+	if (device >= infinipath_max ||
+	    !(devdata[device].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return 0;
+	}
+	return devdata[device].ipath_nguid;
+}
+
+int ipath_layer_query_device(const ipath_type device, uint32_t * vendor,
+			     uint32_t * boardrev, uint32_t * majrev,
+			     uint32_t * minrev)
+{
+	if (device >= infinipath_max ||
+	    !(devdata[device].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return -ENODEV;
+	}
+
+	*vendor = devdata[device].ipath_vendorid;
+	*boardrev = devdata[device].ipath_boardrev;
+	*majrev = devdata[device].ipath_majrev;
+	*minrev = devdata[device].ipath_minrev;
+
+	return 0;
+}
+
+uint32_t ipath_layer_get_flags(const ipath_type device)
+{
+	if (device >= infinipath_max ||
+	    !(devdata[device].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return 0;
+	}
+
+	return devdata[device].ipath_flags;
+}
+
+struct device *ipath_layer_get_pcidev(const ipath_type device)
+{
+	if (device >= infinipath_max ||
+	    !(devdata[device].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return NULL;
+	}
+
+	return &(devdata[device].pcidev->dev);
+}
+
+uint16_t ipath_layer_get_deviceid(const ipath_type device)
+{
+	if (device >= infinipath_max ||
+	    !(devdata[device].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return 0;
+	}
+
+	return devdata[device].ipath_deviceid;
+}
+
+uint64_t ipath_layer_get_lastibcstat(const ipath_type device)
+{
+	if (device >= infinipath_max ||
+	    !(devdata[device].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return 0;
+	}
+
+	return devdata[device].ipath_lastibcstat;
+}
+
+uint32_t ipath_layer_get_ibmtu(const ipath_type device)
+{
+	if (device >= infinipath_max ||
+	    !(devdata[device].ipath_flags & IPATH_INITTED)) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return 0;
+	}
+
+	return devdata[device].ipath_ibmtu;
+}
+
+int ipath_layer_register(const ipath_type device,
+			 int (*l_intr) (const ipath_type, uint32_t),
+			 int (*l_rcv) (const ipath_type, void *,
+				       struct sk_buff *), uint16_t l_rcv_opcode,
+			 int (*l_rcv_lid) (const ipath_type, void *),
+			 uint16_t l_rcv_lid_opcode)
+{
+	int ret = 0;
+
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return 1;
+	}
+	if (!(devdata[device].ipath_flags & IPATH_INITTED)) {
+		_IPATH_VDBG("%s not yet initialized, failing\n",
+			    ipath_get_unit_name(device));
+		return 1;
+	}
+
+	_IPATH_VDBG("intr %p rx %p, rx_lid %p\n", l_intr, l_rcv, l_rcv_lid);
+	if (devdata[device].ipath_layer.l_intr
+	    || devdata[device].ipath_layer.l_rcv) {
+		_IPATH_DBG
+		    ("Layered device already registered on unit %u, failing\n",
+		     device);
+		return 1;
+	}
+
+	if(!(*devdata[device].ipath_statusp & IPATH_STATUS_SMA))
+		*devdata[device].ipath_statusp |= IPATH_STATUS_OIB_SMA;
+	devdata[device].ipath_layer.l_intr = l_intr;
+	devdata[device].ipath_layer.l_rcv = l_rcv;
+	devdata[device].ipath_layer.l_rcv_lid = l_rcv_lid;
+	devdata[device].ipath_layer.l_rcv_opcode = l_rcv_opcode;
+	devdata[device].ipath_layer.l_rcv_lid_opcode = l_rcv_lid_opcode;
+
+	return ret;
+}
+
+static void ipath_verbs_timer(unsigned long t)
+{
+	/*
+	 * If port 0 receive packet interrupts are not availabile,
+	 * check the receive queue.
+	 */
+	if (!(devdata[t].ipath_flags & IPATH_GPIO_INTR))
+		ipath_kreceive(t);
+
+	/* Handle verbs layer timeouts. */
+	if (devdata[t].verbs_layer.l_timer_cb)
+		devdata[t].verbs_layer.l_timer_cb(t);
+
+	mod_timer(&devdata[t].verbs_layer.l_timer, jiffies + 1);
+}
+
+/* Verbs layer registration. */
+int ipath_verbs_register(const ipath_type device,
+			  int (*l_piobufavail) (const ipath_type device),
+			  void (*l_rcv) (const ipath_type device, void *rhdr,
+					 void *data, u32 tlen),
+			  void (*l_timer_cb) (const ipath_type device))
+{
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return 0;
+	}
+	if (!(devdata[device].ipath_flags & IPATH_INITTED)) {
+		_IPATH_VDBG("%s not yet initialized, failing\n",
+			    ipath_get_unit_name(device));
+		return 0;
+	}
+
+	_IPATH_VDBG("piobufavail %p rx %p\n", l_piobufavail, l_rcv);
+	if (devdata[device].verbs_layer.l_piobufavail ||
+	    devdata[device].verbs_layer.l_rcv) {
+		_IPATH_DBG("Verbs layer already registered on unit %u, "
+			   "failing\n", device);
+		return 0;
+	}
+
+	devdata[device].verbs_layer.l_piobufavail = l_piobufavail;
+	devdata[device].verbs_layer.l_rcv = l_rcv;
+	devdata[device].verbs_layer.l_timer_cb = l_timer_cb;
+	devdata[device].verbs_layer.l_flags = 0;
+
+	return 1;
+}
+
+void ipath_verbs_unregister(const ipath_type device)
+{
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return;
+	}
+	if (!(devdata[device].ipath_flags & IPATH_INITTED)) {
+		_IPATH_VDBG("%s not yet initialized, failing\n",
+			    ipath_get_unit_name(device));
+		return;
+	}
+
+	*devdata[device].ipath_statusp &= ~IPATH_STATUS_OIB_SMA;
+	devdata[device].verbs_layer.l_piobufavail = NULL;
+	devdata[device].verbs_layer.l_rcv = NULL;
+	devdata[device].verbs_layer.l_timer_cb = NULL;
+	devdata[device].verbs_layer.l_flags = 0;
+}
+
+int ipath_layer_open(const ipath_type device, uint32_t * pktmax)
+{
+	int ret = 0;
+	uint32_t intval = 0;
+
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return 1;
+	}
+	if (!devdata[device].ipath_layer.l_intr
+	    || !devdata[device].ipath_layer.l_rcv) {
+		_IPATH_DBG("layer not registered, failing\n");
+		return 1;
+	}
+
+	if ((ret =
+	     ipath_setrcvhdrsize(device, NUM_OF_EKSTRA_WORDS_IN_HEADER_QUEUE)))
+		return ret;
+
+	*pktmax = devdata[device].ipath_ibmaxlen;
+
+	if (*devdata[device].ipath_statusp & IPATH_STATUS_IB_READY)
+		intval |= IPATH_LAYER_INT_IF_UP;
+	if (ipath_stats.sps_lid[device])
+		intval |= IPATH_LAYER_INT_LID;
+	if (ipath_stats.sps_mlid[device])
+		intval |= IPATH_LAYER_INT_BCAST;
+	/*
+	 * do this on open, in case low level is already up and
+	 * just layered driver was reloaded, etc.
+	 */
+	if (intval)
+		devdata[device].ipath_layer.l_intr(device, intval);
+
+	return ret;
+}
+
+int16_t ipath_layer_get_lid(const ipath_type device)
+{
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return 0;
+	}
+
+	_IPATH_VDBG("returning mylid 0x%x for layered dev %d\n",
+		    devdata[device].ipath_lid, device);
+	return devdata[device].ipath_lid;
+}
+
+/*
+ * get the MAC address.  This is the EUID-64 OUI octets (top 3), then
+ * skip the next 2 (which should both be zero or 0xff).
+ * The returned MAC is in network order
+ * mac points to at least 6 bytes of buffer
+ * returns 0 on error (to be consistent with get_lid and get_bcast
+ * return 1 on success
+ * We assume that by the time the LID is set, that the GUID is as valid
+ * as it's ever going to be, rather than adding yet another status bit.
+ */
+
+int ipath_layer_get_mac(const ipath_type device, uint8_t * mac)
+{
+	uint8_t *guid;
+
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u, failing\n", device);
+		return 0;
+	}
+	guid = (uint8_t *) & devdata[device].ipath_guid;
+
+	mac[0] = guid[0];
+	mac[1] = guid[1];
+	mac[2] = guid[2];
+	mac[3] = guid[5];
+	mac[4] = guid[6];
+	mac[5] = guid[7];
+	if((guid[3] || guid[4]) && !(guid[3] == 0xff && guid[4] == 0xff))
+		_IPATH_DBG("Warning, guid bytes 3 and 4 not 0 or 0xffff: %x %x\n",
+			guid[3], guid[4]);
+	_IPATH_VDBG("Returning %x:%x:%x:%x:%x:%x\n",
+		    mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
+	return 1;
+}
+
+int16_t ipath_layer_get_bcast(const ipath_type device)
+{
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u, failing\n", device);
+		return 0;
+	}
+
+	_IPATH_VDBG("returning broadcast LID 0x%x for unit %u\n",
+		    devdata[device].ipath_mlid, device);
+	return devdata[device].ipath_mlid;
+}
+
+int ipath_layer_get_num_of_dev(void)
+{
+	return infinipath_max;
+}
+
+int ipath_layer_get_cr_errpkey(const ipath_type device)
+{
+	return ipath_kget_creg32(device, cr_errpkey);
+}
+
+void ipath_layer_close(const ipath_type device)
+{
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return;
+	}
+	if (!devdata[device].ipath_layer.l_intr
+	    || !devdata[device].ipath_layer.l_rcv) {
+		/* normal if not all chips are present */
+		_IPATH_VDBG("layer close without open\n");
+	} else {
+		devdata[device].ipath_layer.l_intr = NULL;
+		devdata[device].ipath_layer.l_rcv = NULL;
+		devdata[device].ipath_layer.l_rcv_lid = NULL;
+		devdata[device].ipath_layer.l_rcv_opcode = 0;
+		devdata[device].ipath_layer.l_rcv_lid_opcode = 0;
+	}
+}
+
+static inline void copy_aligned(uint32_t *piobuf, struct ipath_sge_state *ss,
+				uint32_t length)
+{
+	struct ipath_sge *sge = &ss->sge;
+
+	while (length) {
+		u32 len = sge->length;
+		u32 w;
+
+		BUG_ON(len == 0);
+		if (len > length)
+			len = length;
+		/* Need to round up for the last dword in the packet. */
+		w = (len + 3) >> 2;
+		ipath_dwordcpy(piobuf, sge->vaddr, w);
+		piobuf += w;
+		sge->vaddr += len;
+		sge->length -= len;
+		sge->sge_length -= len;
+		if (sge->sge_length == 0) {
+			if (--ss->num_sge)
+				*sge = *ss->sg_list++;
+		} else if (sge->length == 0 && sge->mr != NULL) {
+			if (++sge->n >= IPATH_SEGSZ) {
+				if (++sge->m >= sge->mr->mapsz)
+					break;
+				sge->n = 0;
+			}
+			sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
+			sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
+		}
+		length -= len;
+	}
+}
+
+static inline void copy_unaligned(uint32_t *piobuf, struct ipath_sge_state *ss,
+				  uint32_t length)
+{
+	struct ipath_sge *sge = &ss->sge;
+	union {
+		u8 wbuf[4];
+		u32 w;
+	} u;
+	int extra = 0;
+
+	while (length) {
+		u32 len = sge->length;
+
+		BUG_ON(len == 0);
+		if (len > length)
+			len = length;
+		length -= len;
+		while (len) {
+			u.wbuf[extra++] = *(u8 *) sge->vaddr;
+			sge->vaddr++;
+			sge->length--;
+			sge->sge_length--;
+			if (extra >= 4) {
+				*piobuf++ = u.w;
+				extra = 0;
+			}
+			len--;
+		}
+		if (sge->sge_length == 0) {
+			if (--ss->num_sge)
+				*sge = *ss->sg_list++;
+		} else if (sge->length == 0) {
+			if (++sge->n >= IPATH_SEGSZ) {
+				if (++sge->m >= sge->mr->mapsz)
+					break;
+				sge->n = 0;
+			}
+			sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
+			sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
+		}
+	}
+	if (extra) {
+		while (extra < 4)
+			u.wbuf[extra++] = 0;
+		*piobuf = u.w;
+	}
+}
+
+/*
+ * This is like ipath_send_smapkt() in that we need to be able to send
+ * packets after the chip is initialized (MADs) but also like
+ * ipath_layer_send() since its used by the verbs layer.
+ */
+int ipath_verbs_send(const ipath_type device, uint32_t hdrwords,
+		     uint32_t *hdr, uint32_t len, struct ipath_sge_state *ss)
+{
+	ipath_devdata *dd = &devdata[device];
+	int whichpb;
+	uint32_t *piobuf, plen;
+	uint64_t pboff;
+
+	if (device >= infinipath_max ||
+	    !(dd->ipath_flags & IPATH_PRESENT) || !dd->ipath_kregbase) {
+		_IPATH_DBG("illegal unit %u\n", device);
+		return -ENODEV;
+	}
+	if (!(dd->ipath_flags & IPATH_INITTED)) {
+		/* no hardware, freeze, etc. */
+		_IPATH_DBG("unit %u not usable\n", device);
+		return -ENODEV;
+	}
+	/* +1 is for the qword padding of pbc */
+	plen = hdrwords + ((len + 3) >> 2) + 1;
+	if ((plen << 2) > dd->ipath_ibmaxlen) {
+		_IPATH_DBG("packet len 0x%x too long, failing\n", plen);
+		return -EINVAL;
+	}
+
+	/* Get a PIO buffer to use. */
+	if ((whichpb = ipath_getpiobuf(device)) < 0)
+		return whichpb;
+
+	pboff = dd->ipath_piobufbase;
+	piobuf = (uint32_t *) (((char *)(dd->ipath_kregbase)) + pboff
+			       + whichpb * dd->ipath_palign);
+	_IPATH_EPDBG("0x%x+1w pio%d\n", plen - 1, whichpb);
+
+	/* Write len to control qword, no flags. */
+	*((uint64_t *) piobuf) = (uint64_t) plen;
+	piobuf += 2;
+	ipath_dwordcpy(piobuf, hdr, hdrwords);
+	if (len == 0)
+		return 0;
+	piobuf += hdrwords;
+	/*
+	 * If we really wanted to check everything, we would have to
+	 * check that each segment starts on a dword boundary and is
+	 * a dword multiple in length.
+	 * Since there can be lots of segments, we only check for a simple
+	 * common case where the amount to copy is contained in one segment.
+	 */
+	if (ss->sge.length == len)
+		copy_aligned(piobuf, ss, len);
+	else
+		copy_unaligned(piobuf, ss, len);
+	return 0;
+}
+
+void ipath_layer_snapshot_counters(const ipath_type device, u64 * swords,
+				   u64 * rwords, u64 * spkts, u64 * rpkts)
+{
+	if (device >= infinipath_max ||
+	    !(devdata[device].ipath_flags & IPATH_PRESENT)) {
+		_IPATH_DBG("illegal unit %u\n", device);
+		return;
+	}
+	if (!(devdata[device].ipath_flags & IPATH_INITTED)) {
+		/* no hardware, freeze, etc. */
+		_IPATH_DBG("unit %u not usable\n", device);
+		return;
+	}
+	*swords = ipath_snap_cntr(device, cr_wordsendcnt);
+	*rwords = ipath_snap_cntr(device, cr_wordrcvcnt);
+	*spkts = ipath_snap_cntr(device, cr_pktsendcnt);
+	*rpkts = ipath_snap_cntr(device, cr_pktrcvcnt);
+}
+
+/*
+ * Return the counters needed by recv_pma_get_portcounters().
+ */
+void ipath_layer_get_counters(const ipath_type device,
+			      struct ipath_layer_counters *cntrs)
+{
+	if (device >= infinipath_max ||
+	    !(devdata[device].ipath_flags & IPATH_PRESENT)) {
+		_IPATH_DBG("illegal unit %u\n", device);
+		return;
+	}
+	if (!(devdata[device].ipath_flags & IPATH_INITTED)) {
+		/* no hardware, freeze, etc. */
+		_IPATH_DBG("unit %u not usable\n", device);
+		return;
+	}
+	cntrs->symbol_error_counter =
+		ipath_snap_cntr(device, cr_ibsymbolerrcnt);
+	cntrs->link_error_recovery_counter =
+		ipath_snap_cntr(device, cr_iblinkerrrecovcnt);
+	cntrs->link_downed_counter = ipath_snap_cntr(device, cr_iblinkdowncnt);
+	cntrs->port_rcv_errors = ipath_snap_cntr(device, cr_err_rlencnt) +
+		ipath_snap_cntr(device, cr_invalidrlencnt) +
+		ipath_snap_cntr(device, cr_erricrccnt) +
+		ipath_snap_cntr(device, cr_errvcrccnt) +
+		ipath_snap_cntr(device, cr_badformatcnt);
+	cntrs->port_rcv_remphys_errors = ipath_snap_cntr(device, cr_rcvebpcnt);
+	cntrs->port_xmit_discards = ipath_snap_cntr(device, cr_unsupvlcnt);
+	cntrs->port_xmit_data = ipath_snap_cntr(device, cr_wordsendcnt);
+	cntrs->port_rcv_data = ipath_snap_cntr(device, cr_wordrcvcnt);
+	cntrs->port_xmit_packets = ipath_snap_cntr(device, cr_pktsendcnt);
+	cntrs->port_rcv_packets = ipath_snap_cntr(device, cr_pktrcvcnt);
+}
+
+void ipath_layer_want_buffer(const ipath_type device)
+{
+	atomic_set_mask(INFINIPATH_S_PIOINTBUFAVAIL,
+			&devdata[device].ipath_sendctrl);
+	ipath_kput_kreg(device, kr_sendctrl, devdata[device].ipath_sendctrl);
+}
+
+int ipath_layer_send(const ipath_type device, void *hdr, void *data,
+		     uint32_t datawords)
+{
+	int ret = 0, whichpb;
+	uint32_t *piobuf, plen;
+	uint16_t vlsllnh;
+	uint64_t pboff;
+
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u, failing\n", device);
+		return -EINVAL;
+	}
+	if (!(devdata[device].ipath_flags & IPATH_RCVHDRSZ_SET)) {
+		_IPATH_DBG("send while not open\n");
+		ret = -EINVAL;
+	} else
+	    if ((devdata[device].ipath_flags & (IPATH_LINKUNK | IPATH_LINKDOWN))
+		|| devdata[device].ipath_lid == 0) {
+		/* lid check is for when sma hasn't yet configured */
+		ret = -ENETDOWN;
+		_IPATH_VDBG("send while not ready, mylid=%u, flags=0x%x\n",
+			    devdata[device].ipath_lid,
+			    devdata[device].ipath_flags);
+	}
+	/* +1 is for the qword padding of pbc */
+	plen = (sizeof(ips_message_header_typ) >> 2) + datawords + 1;
+	if (plen > (devdata[device].ipath_ibmaxlen >> 2)) {
+		_IPATH_DBG("packet len 0x%x too long, failing\n", plen);
+		ret = -EINVAL;
+	}
+	vlsllnh = *((uint16_t *) hdr);
+	if (vlsllnh != htons(IPS_LRH_BTH)) {
+		_IPATH_DBG("Warning: lrh[0] wrong (%x, not %x); not sending\n",
+			   vlsllnh, htons(IPS_LRH_BTH));
+		ret = -EINVAL;
+	}
+	if (ret)
+		goto done;
+
+	/* Get a PIO buffer to use. */
+	if ((whichpb = ipath_getpiobuf(device)) < 0) {
+		ret = whichpb;
+		goto done;
+	}
+
+	pboff = devdata[device].ipath_piobufbase;
+	piobuf =
+	    (uint32_t *) (((char *)(devdata[device].ipath_kregbase)) + pboff +
+			  whichpb * devdata[device].ipath_palign);
+	_IPATH_EPDBG("0x%x+1w pio%d\n", plen - 1, whichpb);
+
+	/* len to control qword, no flags */
+	*((uint64_t *) piobuf) = (uint64_t) plen;
+	piobuf += 2;
+	ipath_dwordcpy(piobuf, hdr, (sizeof(ips_message_header_typ) >> 2));
+	piobuf += (sizeof(ips_message_header_typ) >> 2);
+	ipath_dwordcpy(piobuf, data, datawords);
+
+	ipath_stats.sps_ether_spkts++;	/* another ether packet sent */
+
+done:
+	return ret;
+}
+
+void ipath_layer_set_piointbufavail_int(const ipath_type device)
+{
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return;
+	}
+
+	atomic_set_mask(INFINIPATH_S_PIOINTBUFAVAIL,
+			&devdata[device].ipath_sendctrl);
+
+	ipath_kput_kreg(device, kr_sendctrl, devdata[device].ipath_sendctrl);
+}
+
+void ipath_layer_enable_timer(const ipath_type device)
+{
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return;
+	}
+
+	/*
+	 * HT-400 has a design flaw where the chip and kernel idea
+	 * of the tail register don't always agree, and therefore we won't
+	 * get an interrupt on the next packet received.
+	 * If the board supports per packet receive interrupts, use it.
+	 * Otherwise, the timer function periodically checks for packets
+	 * to cover this case.
+	 * Either way, the timer is needed for verbs layer related
+	 * processing.
+	 */
+	if (devdata[device].ipath_flags & IPATH_GPIO_INTR) {
+		ipath_kput_kreg(device, kr_debugportselect, 0x2074076542310UL);
+		/* Enable GPIO bit 2 interrupt */
+		ipath_kput_kreg(device, kr_gpio_mask, (uint64_t)(1 << 2));
+	}
+
+	init_timer(&devdata[device].verbs_layer.l_timer);
+	devdata[device].verbs_layer.l_timer.function = ipath_verbs_timer;
+	devdata[device].verbs_layer.l_timer.data = (unsigned long)device;
+	devdata[device].verbs_layer.l_timer.expires = jiffies + 1;
+	add_timer(&devdata[device].verbs_layer.l_timer);
+}
+
+void ipath_layer_disable_timer(const ipath_type device)
+{
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return;
+	}
+
+	/* Disable GPIO bit 2 interrupt */
+	if (devdata[device].ipath_flags & IPATH_GPIO_INTR)
+		ipath_kput_kreg(device, kr_gpio_mask, 0);
+
+	del_timer_sync(&devdata[device].verbs_layer.l_timer);
+}
+
+/*
+ * Get the verbs layer flags.
+ */
+unsigned ipath_verbs_get_flags(const ipath_type device)
+{
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return 0;
+	}
+
+	return devdata[device].verbs_layer.l_flags;
+}
+
+/*
+ * Set the verbs layer flags.
+ */
+void ipath_verbs_set_flags(const ipath_type device, unsigned flags)
+{
+	ipath_type s;
+
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return;
+	}
+
+	devdata[device].verbs_layer.l_flags = flags;
+
+	for (s = 0; s < infinipath_max; s++) {
+		if (!(devdata[s].ipath_flags & IPATH_INITTED))
+			continue;
+		if ((flags & IPATH_VERBS_KERNEL_SMA) &&
+		    !(*devdata[s].ipath_statusp & IPATH_STATUS_SMA)) {
+			*devdata[s].ipath_statusp |= IPATH_STATUS_OIB_SMA;
+		} else {
+			*devdata[s].ipath_statusp &= ~IPATH_STATUS_OIB_SMA;
+		}
+	}
+}
+
+/*
+ * Return the size of the PKEY table for port 0.
+ */
+unsigned ipath_layer_get_npkeys(const ipath_type device)
+{
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return 0;
+	}
+
+	return ARRAY_SIZE(devdata[device].ipath_pd[0]->port_pkeys);
+}
+
+/*
+ * Return the indexed PKEY from the port 0 PKEY table.
+ */
+unsigned ipath_layer_get_pkey(const ipath_type device, unsigned index)
+{
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return 0;
+	}
+	if (index >= ARRAY_SIZE(devdata[device].ipath_pd[0]->port_pkeys))
+		return 0;
+
+	return devdata[device].ipath_pd[0]->port_pkeys[index];
+}
+
+/*
+ * Return the PKEY table for port 0.
+ */
+void ipath_layer_get_pkeys(const ipath_type device, uint16_t *pkeys)
+{
+	struct _ipath_portdata *pd;
+
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return;
+	}
+
+	pd = devdata[device].ipath_pd[0];
+	memcpy(pkeys, pd->port_pkeys, sizeof(pd->port_pkeys));
+}
+
+/*
+ * Decrecment the reference count for the given PKEY.
+ * Return true if this was the last reference and the hardware table entry
+ * needs to be changed.
+ */
+static inline int rm_pkey(ipath_devdata *dd, uint16_t key)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
+		if (dd->ipath_pkeys[i] != key)
+			continue;
+		if (atomic_dec_and_test(&dd->ipath_pkeyrefs[i])) {
+			dd->ipath_pkeys[i] = 0;
+			return 1;
+		}
+		break;
+	}
+	return 0;
+}
+
+/*
+ * Add the given PKEY to the hardware table.
+ * Return an error code if unable to add the entry, zero if no change,
+ * or 1 if the hardware PKEY register needs to be updated.
+ */
+static inline int add_pkey(ipath_devdata *dd, uint16_t key)
+{
+	int i;
+	uint16_t lkey = key & 0x7FFF;
+	int any = 0;
+
+	for (i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
+		if (!dd->ipath_pkeys[i]) {
+			any++;
+			continue;
+		}
+		/* If it matches exactly, try to increment the ref count */
+		if (dd->ipath_pkeys[i] == key) {
+			if (atomic_inc_return(&dd->ipath_pkeyrefs[i]) > 1)
+				return 0;
+			/* Lost the race. Look for an empty slot below. */
+			atomic_dec(&dd->ipath_pkeyrefs[i]);
+			any++;
+		}
+		/*
+		 * It makes no sense to have both the limited and unlimited
+		 * PKEY set at the same time since the unlimited one will
+		 * disable the limited one.
+		 */
+		if ((dd->ipath_pkeys[i] & 0x7FFF) == lkey)
+			return -EEXIST;
+	}
+	if (!any)
+		return -EBUSY;
+	for (i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
+		if (!dd->ipath_pkeys[i] &&
+		    atomic_inc_return(&dd->ipath_pkeyrefs[i]) == 1) {
+			/* for ipathstats, etc. */
+			ipath_stats.sps_pkeys[i] = lkey;
+			dd->ipath_pkeys[i] = key;
+			return 1;
+		}
+	}
+	return -EBUSY;
+}
+
+/*
+ * Set the PKEY table for port 0.
+ */
+int ipath_layer_set_pkeys(const ipath_type device, uint16_t *pkeys)
+{
+	ipath_portdata *pd;
+	ipath_devdata *dd;
+	int i;
+	int changed = 0;
+
+	if (device >= infinipath_max) {
+		_IPATH_DBG("Invalid unit %u\n", device);
+		return -EINVAL;
+	}
+
+	dd = &devdata[device];
+	pd = dd->ipath_pd[0];
+
+	for (i = 0; i > ARRAY_SIZE(pd->port_pkeys); i++) {
+		uint16_t key = pkeys[i];
+		uint16_t okey = pd->port_pkeys[i];
+
+		if (key == okey)
+			continue;
+		/*
+		 * The value of this PKEY table entry is changing.
+		 * Remove the old entry in the hardware's array of PKEYs.
+		 */
+		if (okey & 0x7FFF)
+			changed |= rm_pkey(dd, okey);
+		if (key & 0x7FFF) {
+			int ret = add_pkey(dd, key);
+
+			if (ret < 0)
+				key = 0;
+			else
+				changed |= ret;
+		}
+		pd->port_pkeys[i] = key;
+	}
+	if (changed) {
+		uint64_t pkey;
+
+		pkey = (uint64_t) dd->ipath_pkeys[0] |
+		    ((uint64_t) dd->ipath_pkeys[1] << 16) |
+		    ((uint64_t) dd->ipath_pkeys[2] << 32) |
+		    ((uint64_t) dd->ipath_pkeys[3] << 48);
+		_IPATH_VDBG("p0 new pkey reg %llx\n", pkey);
+		ipath_kput_kreg(pd->port_unit, kr_partitionkey, pkey);
+	}
+	return 0;
+}
+
+/*
+ * Registers that vary with the chip implementation constants (port)
+ * use this routine.
+ */
+uint64_t ipath_kget_kreg64_port(const ipath_type stype, ipath_kreg regno,
+				unsigned port)
+{
+	ipath_kreg tmp =
+	    (port < devdata[stype].ipath_portcnt && regno == kr_rcvhdraddr) ?
+	    regno + port :
+	    ((port < devdata[stype].ipath_portcnt
+	      && regno == kr_rcvhdrtailaddr) ? regno + port : __kr_invalid);
+	return ipath_kget_kreg64(stype, tmp);
+}
+
+/*
+ * Registers that vary with the chip implementation constants (port)
+ * use this routine.
+ */
+void ipath_kput_kreg_port(const ipath_type stype, ipath_kreg regno,
+			  unsigned port, uint64_t value)
+{
+	ipath_kreg tmp =
+	    (port < devdata[stype].ipath_portcnt && regno == kr_rcvhdraddr) ?
+	    regno + port :
+	    ((port < devdata[stype].ipath_portcnt
+	      && regno == kr_rcvhdrtailaddr) ? regno + port : __kr_invalid);
+	ipath_kput_kreg(stype, tmp, value);
+}
+
+EXPORT_SYMBOL(ipath_kset_linkstate);
+EXPORT_SYMBOL(ipath_kset_mtu);
+EXPORT_SYMBOL(ipath_layer_close);
+EXPORT_SYMBOL(ipath_layer_get_bcast);
+EXPORT_SYMBOL(ipath_layer_get_cr_errpkey);
+EXPORT_SYMBOL(ipath_layer_get_deviceid);
+EXPORT_SYMBOL(ipath_layer_get_flags);
+EXPORT_SYMBOL(ipath_layer_get_guid);
+EXPORT_SYMBOL(ipath_layer_get_ibmtu);
+EXPORT_SYMBOL(ipath_layer_get_lastibcstat);
+EXPORT_SYMBOL(ipath_layer_get_lid);
+EXPORT_SYMBOL(ipath_layer_get_mac);
+EXPORT_SYMBOL(ipath_layer_get_nguid);
+EXPORT_SYMBOL(ipath_layer_get_num_of_dev);
+EXPORT_SYMBOL(ipath_layer_get_pcidev);
+EXPORT_SYMBOL(ipath_layer_open);
+EXPORT_SYMBOL(ipath_layer_query_device);
+EXPORT_SYMBOL(ipath_layer_register);
+EXPORT_SYMBOL(ipath_layer_send);
+EXPORT_SYMBOL(ipath_layer_set_guid);
+EXPORT_SYMBOL(ipath_layer_set_piointbufavail_int);
+EXPORT_SYMBOL(ipath_layer_snapshot_counters);
+EXPORT_SYMBOL(ipath_layer_get_counters);
+EXPORT_SYMBOL(ipath_layer_want_buffer);
+EXPORT_SYMBOL(ipath_verbs_register);
+EXPORT_SYMBOL(ipath_verbs_send);
+EXPORT_SYMBOL(ipath_verbs_unregister);
+EXPORT_SYMBOL(ipath_set_sps_lid);
+EXPORT_SYMBOL(ipath_layer_enable_timer);
+EXPORT_SYMBOL(ipath_layer_disable_timer);
+EXPORT_SYMBOL(ipath_verbs_get_flags);
+EXPORT_SYMBOL(ipath_verbs_set_flags);
+EXPORT_SYMBOL(ipath_layer_get_npkeys);
+EXPORT_SYMBOL(ipath_layer_get_pkey);
+EXPORT_SYMBOL(ipath_layer_get_pkeys);
+EXPORT_SYMBOL(ipath_layer_set_pkeys);
Signature · 1 line
0.99.9n
#10 lore raw

RFC PATCH 9/13 ipath IB driver headers

Headers for ipath IB driver

---

 drivers/infiniband/hw/ipath/ipath_verbs.h |  527 +++++++++++++++++++++++++++++
 drivers/infiniband/hw/ipath/verbs_debug.h |  104 ++++++
 2 files changed, 631 insertions(+), 0 deletions(-)
 create mode 100644 drivers/infiniband/hw/ipath/ipath_verbs.h
 create mode 100644 drivers/infiniband/hw/ipath/verbs_debug.h

8b106f1a0a6cb02f702c4ef957acad1bf8225c7d

drivers/infiniband/hw/ipath/ipath_verbs.h new file +527 -0
new file mode 100644
index 0000000..4a4c65a
@@ -0,0 +1,527 @@
+/*
+ * Copyright (c) 2005. PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Patent licenses, if any, provided herein do not apply to
+ * combinations of this program with other software, or any other
+ * product whatsoever.
+ *
+ * $Id: ipath_verbs.h 4504 2005-12-16 06:15:47Z rjwalsh $
+ */
+
+#ifndef IPATH_VERBS_H
+#define IPATH_VERBS_H
+
+#include <linux/types.h>
+#include <linux/spinlock.h>
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <rdma/ib_pack.h>
+
+#include "ipath_kernel.h"
+#include "verbs_debug.h"
+
+#define CTL_IPATH_VERBS 0x70736e68	/* "spin" as a hex value, top level */
+#define CTL_IPATH_VERBS_FAULT 1
+#define CTL_IPATH_VERBS_DEBUG 2
+
+/*
+ * Increment this value if any changes that break userspace ABI
+ * compatibility are made.
+ */
+#define IPATH_UVERBS_ABI_VERSION       1
+
+/*
+ * Define an ib_cq_notify value that is not valid so we know when CQ
+ * notifications are armed.
+ */
+#define IB_CQ_NONE	(IB_CQ_NEXT_COMP + 1)
+
+enum {
+	IB_RNR_NAK = 0x20,
+
+	IB_NAK_PSN_ERROR = 0x60,
+	IB_NAK_INVALID_REQUEST = 0x61,
+	IB_NAK_REMOTE_ACCESS_ERROR = 0x62,
+	IB_NAK_REMOTE_OPERATIONAL_ERROR = 0x63,
+	IB_NAK_INVALID_RD_REQUEST = 0x64
+};
+
+/* IB Performance Manager status values */
+enum {
+	IB_PMA_SAMPLE_STATUS_DONE = 0x00,
+	IB_PMA_SAMPLE_STATUS_STARTED = 0x01,
+	IB_PMA_SAMPLE_STATUS_RUNNING = 0x02
+};
+
+/* Mandatory IB performance counter select values. */
+#define IB_PMA_PORT_XMIT_DATA	__constant_htons(0x0001)
+#define IB_PMA_PORT_RCV_DATA	__constant_htons(0x0002)
+#define IB_PMA_PORT_XMIT_PKTS	__constant_htons(0x0003)
+#define IB_PMA_PORT_RCV_PKTS	__constant_htons(0x0004)
+#define IB_PMA_PORT_XMIT_WAIT	__constant_htons(0x0005)
+
+struct ib_reth {
+	u64 vaddr;
+	u32 rkey;
+	u32 length;
+} __attribute__ ((packed));
+
+struct ib_atomic_eth {
+	u64 vaddr;
+	u32 rkey;
+	u64 swap_data;
+	u64 compare_data;
+} __attribute__ ((packed));
+
+struct ipath_other_headers {
+	uint32_t bth[3];
+	union {
+		struct {
+			uint32_t deth[2];
+			uint32_t imm_data;
+		} ud;
+		struct {
+			struct ib_reth reth;
+			uint32_t imm_data;
+		} rc;
+		struct {
+			uint32_t aeth;
+			uint64_t atomic_ack_eth;
+		} at;
+		uint32_t imm_data;
+		uint32_t aeth;
+		struct ib_atomic_eth atomic_eth;
+	} u;
+} __attribute__ ((packed));
+
+/*
+ * Note that UD packets with a GRH header are 8+40+12+8 = 68 bytes long
+ * (72 w/ imm_data).
+ * Only the first 56 bytes of the IB header will be in the
+ * eager header buffer.  The remaining 12 or 16 bytes are in the data buffer.
+ */
+struct ipath_ib_header {
+	uint16_t lrh[4];
+	union {
+		struct {
+			struct ib_grh grh;
+			struct ipath_other_headers oth;
+		} l;
+		struct ipath_other_headers oth;
+	} u;
+} __attribute__ ((packed));
+
+/*
+ * There is one struct ipath_mcast for each multicast GID.
+ * All attached QPs are then stored as a list of
+ * struct ipath_mcast_qp.
+ */
+struct ipath_mcast_qp {
+	struct list_head list;
+	struct ipath_qp *qp;
+};
+
+struct ipath_mcast {
+	struct rb_node rb_node;
+	union ib_gid mgid;
+	struct list_head qp_list;
+	wait_queue_head_t wait;
+	atomic_t refcount;
+};
+
+/* Memory region */
+struct ipath_mr {
+	struct ib_mr ibmr;
+	struct ipath_mregion mr;	/* must be last */
+};
+
+/* Fast memory region */
+struct ipath_fmr {
+	struct ib_fmr ibfmr;
+	u8 page_size;
+	struct ipath_mregion mr;	/* must be last */
+};
+
+/* Protection domain */
+struct ipath_pd {
+	struct ib_pd ibpd;
+	int user;		/* non-zero if created from user space */
+};
+
+/* Address Handle */
+struct ipath_ah {
+	struct ib_ah ibah;
+	struct ib_ah_attr attr;
+};
+
+/*
+ * Quick description of our CQ/QP locking scheme:
+ *
+ * We have one global lock that protects dev->cq/qp_table.  Each
+ * struct ipath_cq/qp also has its own lock.  An individual qp lock
+ * may be taken inside of an individual cq lock.  Both cqs attached to
+ * a qp may be locked, with the send cq locked first.  No other
+ * nesting should be done.
+ *
+ * Each struct ipath_cq/qp also has an atomic_t ref count.  The
+ * pointer from the cq/qp_table to the struct counts as one reference.
+ * This reference also is good for access through the consumer API, so
+ * modifying the CQ/QP etc doesn't need to take another reference.
+ * Access because of a completion being polled does need a reference.
+ *
+ * Finally, each struct ipath_cq/qp has a wait_queue_head_t for the
+ * destroy function to sleep on.
+ *
+ * This means that access from the consumer API requires nothing but
+ * taking the struct's lock.
+ *
+ * Access because of a completion event should go as follows:
+ * - lock cq/qp_table and look up struct
+ * - increment ref count in struct
+ * - drop cq/qp_table lock
+ * - lock struct, do your thing, and unlock struct
+ * - decrement ref count; if zero, wake up waiters
+ *
+ * To destroy a CQ/QP, we can do the following:
+ * - lock cq/qp_table, remove pointer, unlock cq/qp_table lock
+ * - decrement ref count
+ * - wait_event until ref count is zero
+ *
+ * It is the consumer's responsibilty to make sure that no QP
+ * operations (WQE posting or state modification) are pending when the
+ * QP is destroyed.  Also, the consumer must make sure that calls to
+ * qp_modify are serialized.
+ *
+ * Possible optimizations (wait for profile data to see if/where we
+ * have locks bouncing between CPUs):
+ * - split cq/qp table lock into n separate (cache-aligned) locks,
+ *   indexed (say) by the page in the table
+ */
+
+struct ipath_cq {
+	struct ib_cq ibcq;
+	struct tasklet_struct comptask;
+	spinlock_t lock;
+	u8 notify;
+	u8 triggered;
+	u32 head;		/* new records added to the head */
+	u32 tail;		/* poll_cq() reads from here. */
+	struct ib_wc queue[1];	/* this is actually ibcq.cqe + 1 */
+};
+
+/*
+ * Send work request queue entry.
+ * The size of the sg_list is determined when the QP is created and stored
+ * in qp->s_max_sge.
+ */
+struct ipath_swqe {
+	struct ib_send_wr wr;	/* don't use wr.sg_list */
+	u32 psn;		/* first packet sequence number */
+	u32 lpsn;		/* last packet sequence number */
+	u32 ssn;		/* send sequence number */
+	u32 length;		/* total length of data in sg_list */
+	struct ipath_sge sg_list[0];
+};
+
+/*
+ * Receive work request queue entry.
+ * The size of the sg_list is determined when the QP is created and stored
+ * in qp->r_max_sge.
+ */
+struct ipath_rwqe {
+	u64 wr_id;
+	u32 length;		/* total length of data in sg_list */
+	u8 num_sge;
+	struct ipath_sge sg_list[0];
+};
+
+struct ipath_rq {
+	spinlock_t lock;
+	u32 head;		/* new work requests posted to the head */
+	u32 tail;		/* receives pull requests from here. */
+	u32 size;		/* size of RWQE array */
+	u8 max_sge;
+	struct ipath_rwqe *wq;	/* RWQE array */
+};
+
+struct ipath_srq {
+	struct ib_srq ibsrq;
+	struct ipath_rq rq;
+	u32 limit;		/* send signal when number of RWQEs < limit */
+};
+
+/*
+ * Variables prefixed with s_ are for the requester (sender).
+ * Variables prefixed with r_ are for the responder (receiver).
+ * Variables prefixed with ack_ are for responder replies.
+ *
+ * Common variables are protected by both r_rq.lock and s_lock in that order
+ * which only happens in modify_qp() or changing the QP 'state'.
+ */
+struct ipath_qp {
+	struct ib_qp ibqp;
+	struct ipath_qp *next;		/* link list for QPN hash table */
+	struct list_head piowait;	/* link for wait PIO buf */
+	struct list_head timerwait;	/* link for waiting for timeouts */
+	struct ib_ah_attr remote_ah_attr;
+	struct ipath_ib_header s_hdr;	/* next packet header to send */
+	atomic_t refcount;
+	wait_queue_head_t wait;
+	struct tasklet_struct s_task;
+	struct ipath_sge_state *s_cur_sge;
+	struct ipath_sge_state s_sge;	/* current send request data */
+	struct ipath_sge_state s_rdma_sge; /* current RDMA read send data */
+	struct ipath_sge_state r_sge;	/* current receive data */
+	spinlock_t s_lock;
+	int s_flags;
+	u32 s_hdrwords;		/* size of s_hdr in 32 bit words */
+	u32 s_cur_size;		/* size of send packet in bytes */
+	u32 s_len;		/* total length of s_sge */
+	u32 s_rdma_len;		/* total length of s_rdma_sge */
+	u32 s_next_psn;		/* PSN for next request */
+	u32 s_last_psn;		/* last response PSN processed */
+	u32 s_psn;		/* current packet sequence number */
+	u32 s_rnr_timeout;	/* number of milliseconds for RNR timeout */
+	u32 s_ack_psn;		/* PSN for next ACK or RDMA_READ */
+	u64 s_ack_atomic;	/* data for atomic ACK */
+	u64 r_wr_id;		/* ID for current receive WQE */
+	u64 r_atomic_data;	/* data for last atomic op */
+	u32 r_atomic_psn;	/* PSN of last atomic op */
+	u32 r_len;		/* total length of r_sge */
+	u32 r_rcv_len;		/* receive data len processed */
+	u32 r_psn;		/* expected rcv packet sequence number */
+	u8 state;		/* QP state */
+	u8 s_state;		/* opcode of last packet sent */
+	u8 s_ack_state;		/* opcode of packet to ACK */
+	u8 s_nak_state;		/* non-zero if NAK is pending */
+	u8 r_state;		/* opcode of last packet received */
+	u8 r_reuse_sge;		/* for UC receive errors */
+	u8 r_sge_inx;		/* current index into sg_list */
+	u8 s_max_sge;		/* size of s_wq->sg_list */
+	u8 qp_access_flags;
+	u8 s_retry_cnt;		/* number of times to retry */
+	u8 s_rnr_retry_cnt;
+	u8 s_min_rnr_timer;
+	u8 s_retry;		/* requester retry counter */
+	u8 s_rnr_retry;		/* requester RNR retry counter */
+	u8 s_pkey_index;	/* PKEY index to use */
+	enum ib_mtu path_mtu;
+	atomic_t msn;		/* message sequence number */
+	u32 remote_qpn;
+	u32 qkey;		/* QKEY for this QP (for UD or RD) */
+	u32 s_size;		/* send work queue size */
+	u32 s_head;		/* new entries added here */
+	u32 s_tail;		/* next entry to process */
+	u32 s_cur;		/* current work queue entry */
+	u32 s_last;		/* last un-ACK'ed entry */
+	u32 s_ssn;		/* SSN of tail entry */
+	u32 s_lsn;		/* limit sequence number (credit) */
+	struct ipath_swqe *s_wq;	/* send work queue */
+	struct ipath_rq r_rq;	/* receive work queue */
+};
+
+/*
+ * Bit definitions for s_flags.
+ */
+#define IPATH_S_BUSY		0
+#define IPATH_S_SIGNAL_REQ_WR	1
+
+/*
+ * Since struct ipath_swqe is not a fixed size, we can't simply index into
+ * struct ipath_qp.s_wq.  This function does the array index computation.
+ */
+static inline struct ipath_swqe *get_swqe_ptr(struct ipath_qp *qp, unsigned n)
+{
+	return (struct ipath_swqe *)((char *) qp->s_wq +
+		(sizeof(struct ipath_swqe) +
+		 qp->s_max_sge * sizeof(struct ipath_sge)) * n);
+}
+
+/*
+ * Since struct ipath_rwqe is not a fixed size, we can't simply index into
+ * struct ipath_rq.wq.  This function does the array index computation.
+ */
+static inline struct ipath_rwqe *get_rwqe_ptr(struct ipath_rq *rq, unsigned n)
+{
+	return (struct ipath_rwqe *)((char *) rq->wq +
+		(sizeof(struct ipath_rwqe) +
+		 rq->max_sge * sizeof(struct ipath_sge)) * n);
+}
+
+#define QPN_MAX			(1 << 24)
+#define QPNMAP_ENTRIES		(QPN_MAX / PAGE_SIZE / 8)
+#define BITS_PER_PAGE		(PAGE_SIZE*8)
+#define BITS_PER_PAGE_MASK	(BITS_PER_PAGE-1)
+#define mk_qpn(qpt, map, off)	(((map) - (qpt)->map)*BITS_PER_PAGE + (off))
+#define find_next_offset(map, off)					\
+		find_next_zero_bit((map)->page, BITS_PER_PAGE, off)
+
+/*
+ * QPN-map pages start out as NULL, they get allocated upon
+ * first use and are never deallocated. This way,
+ * large bitmaps are not allocated unless large numbers of QPs are used.
+ */
+struct qpn_map {
+	atomic_t n_free;
+	void *page;
+};
+
+struct ipath_qp_table {
+	spinlock_t lock;
+	u32 last;		/* last QP number allocated */
+	u32 max;		/* size of the hash table */
+	u32 nmaps;		/* size of the map table */
+	struct ipath_qp **table;
+	struct qpn_map map[QPNMAP_ENTRIES];	/* bit map of free numbers */
+};
+
+struct ipath_lkey_table {
+	spinlock_t lock;
+	u32 next;		/* next unused index (speeds search) */
+	u32 gen;		/* generation count */
+	u32 max;		/* size of the table */
+	struct ipath_mregion **table;
+};
+
+struct ipath_opcode_stats {
+	u64	n_packets;	/* number of packets */
+	u64	n_bytes;	/* total number of bytes */
+};
+
+struct ipath_ibdev {
+	struct ib_device ibdev;
+	ipath_type ib_unit;	/* This is the device number */
+	u16 sm_lid;		/* in host order */
+	u8 sm_sl;
+	u8 mkeyprot_resv_lmc;
+	unsigned long mkey_lease_timeout;	/* non-zero when timer is set */
+
+	/* The following fields are really per port. */
+	struct ipath_qp_table qp_table;
+	struct ipath_lkey_table lk_table;
+	struct list_head pending[3];	/* FIFO of QPs waiting for ACKs */
+	struct list_head piowait;	/* list for wait PIO buf */
+	struct list_head rnrwait;	/* list of QPs waiting for RNR timer */
+	spinlock_t pending_lock;
+	__be64 sys_image_guid;	/* in network order */
+	__be64 gid_prefix;	/* in network order */
+	__be64 mkey;
+	u64 ipath_sword;	/* total dwords sent (sample result) */
+	u64 ipath_rword;	/* total dwords received (sample result) */
+	u64 ipath_spkts;	/* total packets sent (sample result) */
+	u64 ipath_rpkts;	/* total packets received (sample result) */
+	u64 n_multicast_xmit;	/* total multicast packets sent */
+	u64 n_multicast_rcv;	/* total multicast packets received */
+	u32 n_rc_resends;
+	u32 n_rc_acks;
+	u32 n_rc_qacks;
+	u32 n_seq_naks;
+	u32 n_rdma_seq;
+	u32 n_rnr_naks;
+	u32 n_other_naks;
+	u32 n_timeouts;
+	u32 n_pkt_drops;
+	u32 n_wqe_errs;
+	u32 n_rdma_dup_busy;
+	u32 n_piowait;
+	u32 n_no_piobuf;
+	u32 port_cap_flags;
+	u32 pma_sample_start;
+	u32 pma_sample_interval;
+	__be16 pma_counter_select[5];
+	u16 pma_tag;
+	u16 qkey_violations;
+	u16 mkey_violations;
+	u16 mkey_lease_period;
+	u16 pending_index;	/* which pending queue is active */
+	u8 pma_sample_status;
+	u8 subnet_timeout;
+	struct ipath_opcode_stats opstats[128];
+};
+
+struct ipath_ucontext {
+	struct ib_ucontext ibucontext;
+};
+
+static inline struct ipath_mr *to_imr(struct ib_mr *ibmr)
+{
+	return container_of(ibmr, struct ipath_mr, ibmr);
+}
+
+static inline struct ipath_fmr *to_ifmr(struct ib_fmr *ibfmr)
+{
+	return container_of(ibfmr, struct ipath_fmr, ibfmr);
+}
+
+static inline struct ipath_pd *to_ipd(struct ib_pd *ibpd)
+{
+	return container_of(ibpd, struct ipath_pd, ibpd);
+}
+
+static inline struct ipath_ah *to_iah(struct ib_ah *ibah)
+{
+	return container_of(ibah, struct ipath_ah, ibah);
+}
+
+static inline struct ipath_cq *to_icq(struct ib_cq *ibcq)
+{
+	return container_of(ibcq, struct ipath_cq, ibcq);
+}
+
+static inline struct ipath_srq *to_isrq(struct ib_srq *ibsrq)
+{
+	return container_of(ibsrq, struct ipath_srq, ibsrq);
+}
+
+static inline struct ipath_qp *to_iqp(struct ib_qp *ibqp)
+{
+	return container_of(ibqp, struct ipath_qp, ibqp);
+}
+
+static inline struct ipath_ibdev *to_idev(struct ib_device *ibdev)
+{
+	return container_of(ibdev, struct ipath_ibdev, ibdev);
+}
+
+int ipath_process_mad(struct ib_device *ibdev,
+		      int mad_flags,
+		      u8 port_num,
+		      struct ib_wc *in_wc,
+		      struct ib_grh *in_grh,
+		      struct ib_mad *in_mad, struct ib_mad *out_mad);
+
+static inline struct ipath_ucontext *to_iucontext(struct ib_ucontext
+						  *ibucontext)
+{
+	return container_of(ibucontext, struct ipath_ucontext, ibucontext);
+}
+
+#endif /* IPATH_VERBS_H */
drivers/infiniband/hw/ipath/verbs_debug.h new file +104 -0

104 lines not on this page · raw message

Signature · 1 line
0.99.9n
#11 lore raw

RFC PATCH 10/13 ipath verbs, part 1

First half of ipath verbs driver

---

 drivers/infiniband/hw/ipath/ipath_verbs.c | 3244 +++++++++++++++++++++++++++++
 1 files changed, 3244 insertions(+), 0 deletions(-)
 create mode 100644 drivers/infiniband/hw/ipath/ipath_verbs.c

72075ecec75f8c42e444a7d7d8ffcf340a845b96

drivers/infiniband/hw/ipath/ipath_verbs.c new file +3244 -0
new file mode 100644
index 0000000..808326e
@@ -0,0 +1,3244 @@
+/*
+ * Copyright (c) 2005. PathScale, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Patent licenses, if any, provided herein do not apply to
+ * combinations of this program with other software, or any other
+ * product whatsoever.
+ *
+ * $Id: ipath_verbs.c 4491 2005-12-15 22:20:31Z rjwalsh $
+ */
+
+#include <linux/config.h>
+#include <linux/version.h>
+#include <linux/pci.h>
+#include <linux/err.h>
+#include <rdma/ib_pack.h>
+#include <rdma/ib_smi.h>
+#include <rdma/ib_mad.h>
+#include <rdma/ib_user_verbs.h>
+
+#include <asm/uaccess.h>
+#include <asm-generic/bug.h>
+
+#include "ipath_common.h"
+#include "ips_common.h"
+#include "ipath_layer.h"
+#include "ipath_verbs.h"
+
+/*
+ * Compare the lower 24 bits of the two values.
+ * Returns an integer <, ==, or > than zero.
+ */
+static inline int cmp24(u32 a, u32 b)
+{
+	return (((int) a) - ((int) b)) << 8;
+}
+
+#define MODNAME "ib_ipath"
+#define DRIVER_LOAD_MSG "PathScale " MODNAME " loaded: "
+#define PFX MODNAME ": "
+
+
+/* Not static, because we don't want the compiler removing it */
+const char ipath_verbs_version[] = "ipath_verbs " _IPATH_IDSTR;
+
+unsigned int ib_ipath_qp_table_size = 251;
+module_param(ib_ipath_qp_table_size, uint, 0444);
+MODULE_PARM_DESC(ib_ipath_qp_table_size, "QP table size");
+
+unsigned int ib_ipath_lkey_table_size = 12;
+module_param(ib_ipath_lkey_table_size, uint, 0444);
+MODULE_PARM_DESC(ib_ipath_lkey_table_size,
+		 "LKEY table size in bits (2^n, 1 <= n <= 23)");
+
+unsigned int ib_ipath_debug;	/* debug mask */
+module_param(ib_ipath_debug, uint, 0644);
+MODULE_PARM_DESC(ib_ipath_debug, "Verbs debug mask");
+
+
+static void ipath_ud_loopback(struct ipath_qp *sqp, struct ipath_sge_state *ss,
+			      u32 len, struct ib_send_wr *wr, struct ib_wc *wc);
+static void ipath_ruc_loopback(struct ipath_qp *sqp, struct ib_wc *wc);
+static int ipath_destroy_qp(struct ib_qp *ibqp);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("PathScale <[email protected]>");
+MODULE_DESCRIPTION("Pathscale InfiniPath driver");
+
+enum {
+	IPATH_FAULT_RC_DROP_SEND_F = 1,
+	IPATH_FAULT_RC_DROP_SEND_M,
+	IPATH_FAULT_RC_DROP_SEND_L,
+	IPATH_FAULT_RC_DROP_SEND_O,
+	IPATH_FAULT_RC_DROP_RDMA_WRITE_F,
+	IPATH_FAULT_RC_DROP_RDMA_WRITE_M,
+	IPATH_FAULT_RC_DROP_RDMA_WRITE_L,
+	IPATH_FAULT_RC_DROP_RDMA_WRITE_O,
+	IPATH_FAULT_RC_DROP_RDMA_READ_RESP_F,
+	IPATH_FAULT_RC_DROP_RDMA_READ_RESP_M,
+	IPATH_FAULT_RC_DROP_RDMA_READ_RESP_L,
+	IPATH_FAULT_RC_DROP_RDMA_READ_RESP_O,
+	IPATH_FAULT_RC_DROP_ACK,
+};
+
+enum {
+	IPATH_TRANS_INVALID = 0,
+	IPATH_TRANS_ANY2RST,
+	IPATH_TRANS_RST2INIT,
+	IPATH_TRANS_INIT2INIT,
+	IPATH_TRANS_INIT2RTR,
+	IPATH_TRANS_RTR2RTS,
+	IPATH_TRANS_RTS2RTS,
+	IPATH_TRANS_SQERR2RTS,
+	IPATH_TRANS_ANY2ERR,
+	IPATH_TRANS_RTS2SQD,	/* XXX Wait for expected ACKs & signal event */
+	IPATH_TRANS_SQD2SQD,	/* error if not drained & parameter change */
+	IPATH_TRANS_SQD2RTS,	/* error if not drained */
+};
+
+enum {
+	IPATH_POST_SEND_OK = 0x0001,
+	IPATH_POST_RECV_OK = 0x0002,
+	IPATH_PROCESS_RECV_OK = 0x0004,
+	IPATH_PROCESS_SEND_OK = 0x0008,
+};
+
+static int state_ops[IB_QPS_ERR + 1] = {
+	[IB_QPS_RESET] = 0,
+	[IB_QPS_INIT] = IPATH_POST_RECV_OK,
+	[IB_QPS_RTR] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK,
+	[IB_QPS_RTS] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK |
+	    IPATH_POST_SEND_OK | IPATH_PROCESS_SEND_OK,
+	[IB_QPS_SQD] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK |
+	    IPATH_POST_SEND_OK,
+	[IB_QPS_SQE] = IPATH_POST_RECV_OK | IPATH_PROCESS_RECV_OK,
+	[IB_QPS_ERR] = 0,
+};
+
+/*
+ * Convert the AETH credit code into the number of credits.
+ */
+static u32 credit_table[31] = {
+	0,			/* 0 */
+	1,			/* 1 */
+	2,			/* 2 */
+	3,			/* 3 */
+	4,			/* 4 */
+	6,			/* 5 */
+	8,			/* 6 */
+	12,			/* 7 */
+	16,			/* 8 */
+	24,			/* 9 */
+	32,			/* A */
+	48,			/* B */
+	64,			/* C */
+	96,			/* D */
+	128,			/* E */
+	192,			/* F */
+	256,			/* 10 */
+	384,			/* 11 */
+	512,			/* 12 */
+	768,			/* 13 */
+	1024,			/* 14 */
+	1536,			/* 15 */
+	2048,			/* 16 */
+	3072,			/* 17 */
+	4096,			/* 18 */
+	6144,			/* 19 */
+	8192,			/* 1A */
+	12288,			/* 1B */
+	16384,			/* 1C */
+	24576,			/* 1D */
+	32768			/* 1E */
+};
+
+/*
+ * Convert the AETH RNR timeout code into the number of milliseconds.
+ */
+static u32 rnr_table[32] = {
+	656,			/* 0 */
+	1,			/* 1 */
+	1,			/* 2 */
+	1,			/* 3 */
+	1,			/* 4 */
+	1,			/* 5 */
+	1,			/* 6 */
+	1,			/* 7 */
+	1,			/* 8 */
+	1,			/* 9 */
+	1,			/* A */
+	1,			/* B */
+	1,			/* C */
+	1,			/* D */
+	2,			/* E */
+	2,			/* F */
+	3,			/* 10 */
+	4,			/* 11 */
+	6,			/* 12 */
+	8,			/* 13 */
+	11,			/* 14 */
+	16,			/* 15 */
+	21,			/* 16 */
+	31,			/* 17 */
+	41,			/* 18 */
+	62,			/* 19 */
+	82,			/* 1A */
+	123,			/* 1B */
+	164,			/* 1C */
+	246,			/* 1D */
+	328,			/* 1E */
+	492			/* 1F */
+};
+
+/*
+ * Translate ib_wr_opcode into ib_wc_opcode.
+ */
+static enum ib_wc_opcode wc_opcode[] = {
+	[IB_WR_RDMA_WRITE] = IB_WC_RDMA_WRITE,
+	[IB_WR_RDMA_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE,
+	[IB_WR_SEND] = IB_WC_SEND,
+	[IB_WR_SEND_WITH_IMM] = IB_WC_SEND,
+	[IB_WR_RDMA_READ] = IB_WC_RDMA_READ,
+	[IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP,
+	[IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD
+};
+
+/*
+ * Array of device pointers.
+ */
+static uint32_t number_of_devices;
+static struct ipath_ibdev **ipath_devices;
+
+/*
+ * Global table of GID to attached QPs.
+ * The table is global to all ipath devices since a send from one QP/device
+ * needs to be locally routed to any locally attached QPs on the same
+ * or different device.
+ */
+static struct rb_root mcast_tree;
+static spinlock_t mcast_lock = SPIN_LOCK_UNLOCKED;
+
+/*
+ * Allocate a structure to link a QP to the multicast GID structure.
+ */
+static struct ipath_mcast_qp *ipath_mcast_qp_alloc(struct ipath_qp *qp)
+{
+	struct ipath_mcast_qp *mqp;
+
+	mqp = kmalloc(sizeof(*mqp), GFP_KERNEL);
+	if (!mqp)
+		return NULL;
+
+	mqp->qp = qp;
+	atomic_inc(&qp->refcount);
+
+	return mqp;
+}
+
+static void ipath_mcast_qp_free(struct ipath_mcast_qp *mqp)
+{
+	struct ipath_qp *qp = mqp->qp;
+
+	/* Notify ipath_destroy_qp() if it is waiting. */
+	if (atomic_dec_and_test(&qp->refcount))
+		wake_up(&qp->wait);
+
+	kfree(mqp);
+}
+
+/*
+ * Allocate a structure for the multicast GID.
+ * A list of QPs will be attached to this structure.
+ */
+static struct ipath_mcast *ipath_mcast_alloc(union ib_gid *mgid)
+{
+	struct ipath_mcast *mcast;
+
+	mcast = kmalloc(sizeof(*mcast), GFP_KERNEL);
+	if (!mcast)
+		return NULL;
+
+	mcast->mgid = *mgid;
+	INIT_LIST_HEAD(&mcast->qp_list);
+	init_waitqueue_head(&mcast->wait);
+	atomic_set(&mcast->refcount, 0);
+
+	return mcast;
+}
+
+static void ipath_mcast_free(struct ipath_mcast *mcast)
+{
+	struct ipath_mcast_qp *p, *tmp;
+
+	list_for_each_entry_safe(p, tmp, &mcast->qp_list, list)
+		ipath_mcast_qp_free(p);
+
+	kfree(mcast);
+}
+
+/*
+ * Search the global table for the given multicast GID.
+ * Return it or NULL if not found.
+ * The caller is responsible for decrementing the reference count if found.
+ */
+static struct ipath_mcast *ipath_mcast_find(union ib_gid *mgid)
+{
+	struct rb_node *n;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mcast_lock, flags);
+	n = mcast_tree.rb_node;
+	while (n) {
+		struct ipath_mcast *mcast;
+		int ret;
+
+		mcast = rb_entry(n, struct ipath_mcast, rb_node);
+
+		ret = memcmp(mgid->raw, mcast->mgid.raw, sizeof(union ib_gid));
+		if (ret < 0)
+			n = n->rb_left;
+		else if (ret > 0)
+			n = n->rb_right;
+		else {
+			atomic_inc(&mcast->refcount);
+			spin_unlock_irqrestore(&mcast_lock, flags);
+			return mcast;
+		}
+	}
+	spin_unlock_irqrestore(&mcast_lock, flags);
+
+	return NULL;
+}
+
+/*
+ * Insert the multicast GID into the table and
+ * attach the QP structure.
+ * Return zero if both were added.
+ * Return EEXIST if the GID was already in the table but the QP was added.
+ * Return ESRCH if the QP was already attached and neither structure was added.
+ */
+static int ipath_mcast_add(struct ipath_mcast *mcast,
+			   struct ipath_mcast_qp *mqp)
+{
+	struct rb_node **n = &mcast_tree.rb_node;
+	struct rb_node *pn = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mcast_lock, flags);
+
+	while (*n) {
+		struct ipath_mcast *tmcast;
+		struct ipath_mcast_qp *p;
+		int ret;
+
+		pn = *n;
+		tmcast = rb_entry(pn, struct ipath_mcast, rb_node);
+
+		ret = memcmp(mcast->mgid.raw, tmcast->mgid.raw,
+			     sizeof(union ib_gid));
+		if (ret < 0) {
+			n = &pn->rb_left;
+			continue;
+		}
+		if (ret > 0) {
+			n = &pn->rb_right;
+			continue;
+		}
+
+		/* Search the QP list to see if this is already there. */
+		list_for_each_entry_rcu(p, &tmcast->qp_list, list) {
+			if (p->qp == mqp->qp) {
+				spin_unlock_irqrestore(&mcast_lock, flags);
+				return ESRCH;
+			}
+		}
+		list_add_tail_rcu(&mqp->list, &tmcast->qp_list);
+		spin_unlock_irqrestore(&mcast_lock, flags);
+		return EEXIST;
+	}
+
+	list_add_tail_rcu(&mqp->list, &mcast->qp_list);
+
+	atomic_inc(&mcast->refcount);
+	rb_link_node(&mcast->rb_node, pn, n);
+	rb_insert_color(&mcast->rb_node, &mcast_tree);
+
+	spin_unlock_irqrestore(&mcast_lock, flags);
+
+	return 0;
+}
+
+static int ipath_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid,
+				  u16 lid)
+{
+	struct ipath_qp *qp = to_iqp(ibqp);
+	struct ipath_mcast *mcast;
+	struct ipath_mcast_qp *mqp;
+
+	/*
+	 * Allocate data structures since its better to do this outside of
+	 * spin locks and it will most likely be needed.
+	 */
+	mcast = ipath_mcast_alloc(gid);
+	if (mcast == NULL)
+		return -ENOMEM;
+	mqp = ipath_mcast_qp_alloc(qp);
+	if (mqp == NULL) {
+		ipath_mcast_free(mcast);
+		return -ENOMEM;
+	}
+	switch (ipath_mcast_add(mcast, mqp)) {
+	case ESRCH:
+		/* Neither was used: can't attach the same QP twice. */
+		ipath_mcast_qp_free(mqp);
+		ipath_mcast_free(mcast);
+		return -EINVAL;
+	case EEXIST:		/* The mcast wasn't used */
+		ipath_mcast_free(mcast);
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static int ipath_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid,
+				  u16 lid)
+{
+	struct ipath_qp *qp = to_iqp(ibqp);
+	struct ipath_mcast *mcast = NULL;
+	struct ipath_mcast_qp *p, *tmp;
+	struct rb_node *n;
+	unsigned long flags;
+	int last = 0;
+
+	spin_lock_irqsave(&mcast_lock, flags);
+
+	/* Find the GID in the mcast table. */
+	n = mcast_tree.rb_node;
+	while (1) {
+		int ret;
+
+		if (n == NULL) {
+			spin_unlock_irqrestore(&mcast_lock, flags);
+			return 0;
+		}
+
+		mcast = rb_entry(n, struct ipath_mcast, rb_node);
+		ret = memcmp(gid->raw, mcast->mgid.raw, sizeof(union ib_gid));
+		if (ret < 0)
+			n = n->rb_left;
+		else if (ret > 0)
+			n = n->rb_right;
+		else
+			break;
+	}
+
+	/* Search the QP list. */
+	list_for_each_entry_safe(p, tmp, &mcast->qp_list, list) {
+		if (p->qp != qp)
+			continue;
+		/*
+		 * We found it, so remove it, but don't poison the forward link
+		 * until we are sure there are no list walkers.
+		 */
+		list_del_rcu(&p->list);
+
+		/* If this was the last attached QP, remove the GID too. */
+		if (list_empty(&mcast->qp_list)) {
+			rb_erase(&mcast->rb_node, &mcast_tree);
+			last = 1;
+		}
+		break;
+	}
+
+	spin_unlock_irqrestore(&mcast_lock, flags);
+
+	if (p) {
+		/*
+		 * Wait for any list walkers to finish before freeing the
+		 * list element.
+		 */
+		wait_event(mcast->wait, atomic_read(&mcast->refcount) <= 1);
+		ipath_mcast_qp_free(p);
+	}
+	if (last) {
+		atomic_dec(&mcast->refcount);
+		wait_event(mcast->wait, !atomic_read(&mcast->refcount));
+		ipath_mcast_free(mcast);
+	}
+
+	return 0;
+}
+
+/*
+ * Copy data to SGE memory.
+ */
+static void copy_sge(struct ipath_sge_state *ss, void *data, u32 length)
+{
+	struct ipath_sge *sge = &ss->sge;
+
+	while (length) {
+		u32 len = sge->length;
+
+		BUG_ON(len == 0);
+		if (len > length)
+			len = length;
+		memcpy(sge->vaddr, data, len);
+		sge->vaddr += len;
+		sge->length -= len;
+		sge->sge_length -= len;
+		if (sge->sge_length == 0) {
+			if (--ss->num_sge)
+				*sge = *ss->sg_list++;
+		} else if (sge->length == 0 && sge->mr != NULL) {
+			if (++sge->n >= IPATH_SEGSZ) {
+				if (++sge->m >= sge->mr->mapsz)
+					break;
+				sge->n = 0;
+			}
+			sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
+			sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
+		}
+		data += len;
+		length -= len;
+	}
+}
+
+/*
+ * Skip over length bytes of SGE memory.
+ */
+static void skip_sge(struct ipath_sge_state *ss, u32 length)
+{
+	struct ipath_sge *sge = &ss->sge;
+
+	while (length > sge->sge_length) {
+		length -= sge->sge_length;
+		ss->sge = *ss->sg_list++;
+	}
+	while (length) {
+		u32 len = sge->length;
+
+		BUG_ON(len == 0);
+		if (len > length)
+			len = length;
+		sge->vaddr += len;
+		sge->length -= len;
+		sge->sge_length -= len;
+		if (sge->sge_length == 0) {
+			if (--ss->num_sge)
+				*sge = *ss->sg_list++;
+		} else if (sge->length == 0 && sge->mr != NULL) {
+			if (++sge->n >= IPATH_SEGSZ) {
+				if (++sge->m >= sge->mr->mapsz)
+					break;
+				sge->n = 0;
+			}
+			sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
+			sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
+		}
+		length -= len;
+	}
+}
+
+static inline u32 alloc_qpn(struct ipath_qp_table *qpt)
+{
+	u32 i, offset, max_scan, qpn;
+	struct qpn_map *map;
+
+	qpn = qpt->last + 1;
+	if (qpn >= QPN_MAX)
+		qpn = 2;
+	offset = qpn & BITS_PER_PAGE_MASK;
+	map = &qpt->map[qpn / BITS_PER_PAGE];
+	max_scan = qpt->nmaps - !offset;
+	for (i = 0;;) {
+		if (unlikely(!map->page)) {
+			unsigned long page = get_zeroed_page(GFP_KERNEL);
+			unsigned long flags;
+
+			/*
+			 * Free the page if someone raced with us
+			 * installing it:
+			 */
+			spin_lock_irqsave(&qpt->lock, flags);
+			if (map->page)
+				free_page(page);
+			else
+				map->page = (void *)page;
+			spin_unlock_irqrestore(&qpt->lock, flags);
+			if (unlikely(!map->page))
+				break;
+		}
+		if (likely(atomic_read(&map->n_free))) {
+			do {
+				if (!test_and_set_bit(offset, map->page)) {
+					atomic_dec(&map->n_free);
+					qpt->last = qpn;
+					return qpn;
+				}
+				offset = find_next_offset(map, offset);
+				qpn = mk_qpn(qpt, map, offset);
+				/*
+				 * This test differs from alloc_pidmap().
+				 * If find_next_offset() does find a zero bit,
+				 * we don't need to check for QPN wrapping
+				 * around past our starting QPN.  We
+				 * just need to be sure we don't loop forever.
+				 */
+			} while (offset < BITS_PER_PAGE && qpn < QPN_MAX);
+		}
+		/*
+		 * In order to keep the number of pages allocated to a minimum,
+		 * we scan the all existing pages before increasing the size
+		 * of the bitmap table.
+		 */
+		if (++i > max_scan) {
+			if (qpt->nmaps == QPNMAP_ENTRIES)
+				break;
+			map = &qpt->map[qpt->nmaps++];
+			offset = 0;
+		} else if (map < &qpt->map[qpt->nmaps]) {
+			++map;
+			offset = 0;
+		} else {
+			map = &qpt->map[0];
+			offset = 2;
+		}
+		qpn = mk_qpn(qpt, map, offset);
+	}
+	return 0;
+}
+
+static inline void free_qpn(struct ipath_qp_table *qpt, u32 qpn)
+{
+	struct qpn_map *map;
+
+	map = qpt->map + qpn / BITS_PER_PAGE;
+	if (map->page)
+		clear_bit(qpn & BITS_PER_PAGE_MASK, map->page);
+	atomic_inc(&map->n_free);
+}
+
+/*
+ * Allocate the next available QPN and put the QP into the hash table.
+ * The hash table holds a reference to the QP.
+ */
+static int ipath_alloc_qpn(struct ipath_qp_table *qpt, struct ipath_qp *qp,
+			   enum ib_qp_type type)
+{
+	unsigned long flags;
+	u32 qpn;
+
+	if (type == IB_QPT_SMI)
+		qpn = 0;
+	else if (type == IB_QPT_GSI)
+		qpn = 1;
+	else {
+		/* Allocate the next available QPN */
+		qpn = alloc_qpn(qpt);
+		if (qpn == 0) {
+			return -ENOMEM;
+		}
+	}
+	qp->ibqp.qp_num = qpn;
+
+	/* Add the QP to the hash table. */
+	spin_lock_irqsave(&qpt->lock, flags);
+
+	qpn %= qpt->max;
+	qp->next = qpt->table[qpn];
+	qpt->table[qpn] = qp;
+	atomic_inc(&qp->refcount);
+
+	spin_unlock_irqrestore(&qpt->lock, flags);
+	return 0;
+}
+
+/*
+ * Remove the QP from the table so it can't be found asynchronously by
+ * the receive interrupt routine.
+ */
+static void ipath_free_qp(struct ipath_qp_table *qpt, struct ipath_qp *qp)
+{
+	struct ipath_qp *q, **qpp;
+	unsigned long flags;
+	int fnd = 0;
+
+	spin_lock_irqsave(&qpt->lock, flags);
+
+	/* Remove QP from the hash table. */
+	qpp = &qpt->table[qp->ibqp.qp_num % qpt->max];
+	for (; (q = *qpp) != NULL; qpp = &q->next) {
+		if (q == qp) {
+			*qpp = qp->next;
+			qp->next = NULL;
+			atomic_dec(&qp->refcount);
+			fnd = 1;
+			break;
+		}
+	}
+
+	spin_unlock_irqrestore(&qpt->lock, flags);
+
+	if (!fnd)
+		return;
+
+	/* If QPN is not reserved, mark QPN free in the bitmap. */
+	if (qp->ibqp.qp_num > 1)
+		free_qpn(qpt, qp->ibqp.qp_num);
+
+	wait_event(qp->wait, !atomic_read(&qp->refcount));
+}
+
+/*
+ * Remove all QPs from the table.
+ */
+static void ipath_free_all_qps(struct ipath_qp_table *qpt)
+{
+	unsigned long flags;
+	struct ipath_qp *qp, *nqp;
+	u32 n;
+
+	for (n = 0; n < qpt->max; n++) {
+		spin_lock_irqsave(&qpt->lock, flags);
+		qp = qpt->table[n];
+		qpt->table[n] = NULL;
+		spin_unlock_irqrestore(&qpt->lock, flags);
+
+		while (qp) {
+			nqp = qp->next;
+			if (qp->ibqp.qp_num > 1)
+				free_qpn(qpt, qp->ibqp.qp_num);
+			if (!atomic_dec_and_test(&qp->refcount) ||
+			    !ipath_destroy_qp(&qp->ibqp))
+				_VERBS_INFO("QP memory leak!\n");
+			qp = nqp;
+		}
+	}
+
+	for (n = 0; n < ARRAY_SIZE(qpt->map); n++) {
+		if (qpt->map[n].page)
+			free_page((unsigned long)qpt->map[n].page);
+	}
+}
+
+/*
+ * Return the QP with the given QPN.
+ * The caller is responsible for decrementing the QP reference count when done.
+ */
+static struct ipath_qp *ipath_lookup_qpn(struct ipath_qp_table *qpt, u32 qpn)
+{
+	unsigned long flags;
+	struct ipath_qp *qp;
+
+	spin_lock_irqsave(&qpt->lock, flags);
+
+	for (qp = qpt->table[qpn % qpt->max]; qp; qp = qp->next) {
+		if (qp->ibqp.qp_num == qpn) {
+			atomic_inc(&qp->refcount);
+			break;
+		}
+	}
+
+	spin_unlock_irqrestore(&qpt->lock, flags);
+	return qp;
+}
+
+static int ipath_alloc_lkey(struct ipath_lkey_table *rkt,
+			    struct ipath_mregion *mr)
+{
+	unsigned long flags;
+	u32 r;
+	u32 n;
+
+	spin_lock_irqsave(&rkt->lock, flags);
+
+	/* Find the next available LKEY */
+	r = n = rkt->next;
+	for (;;) {
+		if (rkt->table[r] == NULL)
+			break;
+		r = (r + 1) & (rkt->max - 1);
+		if (r == n) {
+			spin_unlock_irqrestore(&rkt->lock, flags);
+			_VERBS_INFO("LKEY table full\n");
+			return 0;
+		}
+	}
+	rkt->next = (r + 1) & (rkt->max - 1);
+	/*
+	 * Make sure lkey is never zero which is reserved to indicate an
+	 * unrestricted LKEY.
+	 */
+	rkt->gen++;
+	mr->lkey = (r << (32 - ib_ipath_lkey_table_size)) |
+	    ((((1 << (24 - ib_ipath_lkey_table_size)) - 1) & rkt->gen) << 8);
+	if (mr->lkey == 0) {
+		mr->lkey |= 1 << 8;
+		rkt->gen++;
+	}
+	rkt->table[r] = mr;
+	spin_unlock_irqrestore(&rkt->lock, flags);
+
+	return 1;
+}
+
+static void ipath_free_lkey(struct ipath_lkey_table *rkt, u32 lkey)
+{
+	unsigned long flags;
+	u32 r;
+
+	if (lkey == 0)
+		return;
+	r = lkey >> (32 - ib_ipath_lkey_table_size);
+	spin_lock_irqsave(&rkt->lock, flags);
+	rkt->table[r] = NULL;
+	spin_unlock_irqrestore(&rkt->lock, flags);
+}
+
+/*
+ * Check the IB SGE for validity and initialize our internal version of it.
+ * Return 1 if OK, else zero.
+ */
+static int ipath_lkey_ok(struct ipath_lkey_table *rkt, struct ipath_sge *isge,
+			 struct ib_sge *sge, int acc)
+{
+	struct ipath_mregion *mr;
+	size_t off;
+
+	/*
+	 * We use LKEY == zero to mean a physical kmalloc() address.
+	 * This is a bit of a hack since we rely on dma_map_single()
+	 * being reversible by calling bus_to_virt().
+	 */
+	if (sge->lkey == 0) {
+		isge->mr = NULL;
+		isge->vaddr = bus_to_virt(sge->addr);
+		isge->length = sge->length;
+		isge->sge_length = sge->length;
+		return 1;
+	}
+	spin_lock(&rkt->lock);
+	mr = rkt->table[(sge->lkey >> (32 - ib_ipath_lkey_table_size))];
+	spin_unlock(&rkt->lock);
+	if (unlikely(mr == NULL || mr->lkey != sge->lkey))
+		return 0;
+
+	off = sge->addr - mr->user_base;
+	if (unlikely(sge->addr < mr->user_base ||
+		     off + sge->length > mr->length ||
+		     (mr->access_flags & acc) != acc))
+		return 0;
+
+	off += mr->offset;
+	isge->mr = mr;
+	isge->m = 0;
+	isge->n = 0;
+	while (off >= mr->map[isge->m]->segs[isge->n].length) {
+		off -= mr->map[isge->m]->segs[isge->n].length;
+		if (++isge->n >= IPATH_SEGSZ) {
+			isge->m++;
+			isge->n = 0;
+		}
+	}
+	isge->vaddr = mr->map[isge->m]->segs[isge->n].vaddr + off;
+	isge->length = mr->map[isge->m]->segs[isge->n].length - off;
+	isge->sge_length = sge->length;
+	return 1;
+}
+
+/*
+ * Initialize the qp->s_sge after a restart.
+ * The QP s_lock should be held.
+ */
+static void ipath_init_restart(struct ipath_qp *qp, struct ipath_swqe *wqe)
+{
+	struct ipath_ibdev *dev;
+	u32 len;
+
+	len = ((qp->s_psn - wqe->psn) & 0xFFFFFF) *
+	    ib_mtu_enum_to_int(qp->path_mtu);
+	qp->s_sge.sge = wqe->sg_list[0];
+	qp->s_sge.sg_list = wqe->sg_list + 1;
+	qp->s_sge.num_sge = wqe->wr.num_sge;
+	skip_sge(&qp->s_sge, len);
+	qp->s_len = wqe->length - len;
+	dev = to_idev(qp->ibqp.device);
+	spin_lock(&dev->pending_lock);
+	if (qp->timerwait.next == LIST_POISON1)
+		list_add_tail(&qp->timerwait,
+			      &dev->pending[dev->pending_index]);
+	spin_unlock(&dev->pending_lock);
+}
+
+/*
+ * Check the IB virtual address, length, and RKEY.
+ * Return 1 if OK, else zero.
+ * The QP r_rq.lock should be held.
+ */
+static int ipath_rkey_ok(struct ipath_ibdev *dev, struct ipath_sge_state *ss,
+			 u32 len, u64 vaddr, u32 rkey, int acc)
+{
+	struct ipath_lkey_table *rkt = &dev->lk_table;
+	struct ipath_sge *sge = &ss->sge;
+	struct ipath_mregion *mr;
+	size_t off;
+
+	spin_lock(&rkt->lock);
+	mr = rkt->table[(rkey >> (32 - ib_ipath_lkey_table_size))];
+	spin_unlock(&rkt->lock);
+	if (unlikely(mr == NULL || mr->lkey != rkey))
+		return 0;
+
+	off = vaddr - mr->iova;
+	if (unlikely(vaddr < mr->iova || off + len > mr->length ||
+		     (mr->access_flags & acc) == 0))
+		return 0;
+
+	off += mr->offset;
+	sge->mr = mr;
+	sge->m = 0;
+	sge->n = 0;
+	while (off >= mr->map[sge->m]->segs[sge->n].length) {
+		off -= mr->map[sge->m]->segs[sge->n].length;
+		if (++sge->n >= IPATH_SEGSZ) {
+			sge->m++;
+			sge->n = 0;
+		}
+	}
+	sge->vaddr = mr->map[sge->m]->segs[sge->n].vaddr + off;
+	sge->length = mr->map[sge->m]->segs[sge->n].length - off;
+	sge->sge_length = len;
+	ss->sg_list = NULL;
+	ss->num_sge = 1;
+	return 1;
+}
+
+/*
+ * Add a new entry to the completion queue.
+ * This may be called with one of the qp->s_lock or qp->r_rq.lock held.
+ */
+static void ipath_cq_enter(struct ipath_cq *cq, struct ib_wc *entry, int sig)
+{
+	unsigned long flags;
+	u32 next;
+
+	spin_lock_irqsave(&cq->lock, flags);
+
+	cq->queue[cq->head] = *entry;
+	next = cq->head + 1;
+	if (next == cq->ibcq.cqe)
+		next = 0;
+	if (next != cq->tail)
+		cq->head = next;
+	else {
+		/* XXX - need to mark current wr as having an error... */
+	}
+
+	if (cq->notify == IB_CQ_NEXT_COMP ||
+	    (cq->notify == IB_CQ_SOLICITED && sig)) {
+		cq->notify = IB_CQ_NONE;
+		cq->triggered++;
+		/*
+		 * This will cause send_complete() to be called in
+		 * another thread.
+		 */
+		tasklet_schedule(&cq->comptask);
+	}
+
+	spin_unlock_irqrestore(&cq->lock, flags);
+
+	if (entry->status != IB_WC_SUCCESS)
+		to_idev(cq->ibcq.device)->n_wqe_errs++;
+}
+
+static void send_complete(unsigned long data)
+{
+	struct ipath_cq *cq = (struct ipath_cq *)data;
+
+	/*
+	 * The completion handler will most likely rearm the notification
+	 * and poll for all pending entries.  If a new completion entry
+	 * is added while we are in this routine, tasklet_schedule()
+	 * won't call us again until we return so we check triggered to
+	 * see if we need to call the handler again.
+	 */
+	for (;;) {
+		u8 triggered = cq->triggered;
+
+		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
+
+		if (cq->triggered == triggered)
+			return;
+	}
+}
+
+/*
+ * This is the QP state transition table.
+ * See ipath_modify_qp() for details.
+ */
+static const struct {
+	int trans;
+	u32 req_param[IB_QPT_RAW_IPV6];
+	u32 opt_param[IB_QPT_RAW_IPV6];
+} qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
+	[IB_QPS_RESET] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_INIT] = {
+			.trans = IPATH_TRANS_RST2INIT,
+			.req_param = {
+				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_ACCESS_FLAGS),
+				[IB_QPT_RC] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_ACCESS_FLAGS),
+			},
+		},
+	},
+	[IB_QPS_INIT] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_INIT] = {
+			.trans = IPATH_TRANS_INIT2INIT,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_ACCESS_FLAGS),
+				[IB_QPT_RC] = (IB_QP_PKEY_INDEX |
+					 IB_QP_PORT |
+					 IB_QP_ACCESS_FLAGS),
+			}
+		},
+		[IB_QPS_RTR] = {
+			.trans = IPATH_TRANS_INIT2RTR,
+			.req_param = {
+				[IB_QPT_UC] = (IB_QP_AV |
+					 IB_QP_PATH_MTU |
+					 IB_QP_DEST_QPN |
+					 IB_QP_RQ_PSN),
+				[IB_QPT_RC] = (IB_QP_AV |
+					 IB_QP_PATH_MTU |
+					 IB_QP_DEST_QPN |
+					 IB_QP_RQ_PSN |
+					 IB_QP_MAX_DEST_RD_ATOMIC |
+					 IB_QP_MIN_RNR_TIMER),
+			},
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_PKEY_INDEX |
+					 IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX),
+				[IB_QPT_RC] = (IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX),
+			}
+		}
+	},
+	[IB_QPS_RTR] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_RTS] = {
+			.trans = IPATH_TRANS_RTR2RTS,
+			.req_param = {
+				[IB_QPT_SMI] = IB_QP_SQ_PSN,
+				[IB_QPT_GSI] = IB_QP_SQ_PSN,
+				[IB_QPT_UD] = IB_QP_SQ_PSN,
+				[IB_QPT_UC] = IB_QP_SQ_PSN,
+				[IB_QPT_RC] = (IB_QP_TIMEOUT |
+					 IB_QP_RETRY_CNT |
+					 IB_QP_RNR_RETRY |
+					 IB_QP_SQ_PSN |
+					 IB_QP_MAX_QP_RD_ATOMIC),
+			},
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX |
+					 IB_QP_PATH_MIG_STATE),
+				[IB_QPT_RC] = (IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX |
+					 IB_QP_MIN_RNR_TIMER |
+					 IB_QP_PATH_MIG_STATE),
+			}
+		}
+	},
+	[IB_QPS_RTS] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_RTS] = {
+			.trans = IPATH_TRANS_RTS2RTS,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_ACCESS_FLAGS |
+					 IB_QP_ALT_PATH |
+					 IB_QP_PATH_MIG_STATE),
+				[IB_QPT_RC] = (IB_QP_ACCESS_FLAGS |
+					 IB_QP_ALT_PATH |
+					 IB_QP_PATH_MIG_STATE |
+					 IB_QP_MIN_RNR_TIMER),
+			}
+		},
+		[IB_QPS_SQD] = {
+			.trans = IPATH_TRANS_RTS2SQD,
+		},
+	},
+	[IB_QPS_SQD] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_RTS] = {
+			.trans = IPATH_TRANS_SQD2RTS,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PATH_MIG_STATE),
+				[IB_QPT_RC] = (IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_MIN_RNR_TIMER |
+					 IB_QP_PATH_MIG_STATE),
+			}
+		},
+		[IB_QPS_SQD] = {
+			.trans = IPATH_TRANS_SQD2SQD,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_PKEY_INDEX | IB_QP_QKEY),
+				[IB_QPT_UC] = (IB_QP_AV |
+					 IB_QP_TIMEOUT |
+					 IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX |
+					 IB_QP_PATH_MIG_STATE),
+				[IB_QPT_RC] = (IB_QP_AV |
+					 IB_QP_TIMEOUT |
+					 IB_QP_RETRY_CNT |
+					 IB_QP_RNR_RETRY |
+					 IB_QP_MAX_QP_RD_ATOMIC |
+					 IB_QP_MAX_DEST_RD_ATOMIC |
+					 IB_QP_CUR_STATE |
+					 IB_QP_ALT_PATH |
+					 IB_QP_ACCESS_FLAGS |
+					 IB_QP_PKEY_INDEX |
+					 IB_QP_MIN_RNR_TIMER |
+					 IB_QP_PATH_MIG_STATE),
+			}
+		}
+	},
+	[IB_QPS_SQE] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR },
+		[IB_QPS_RTS] = {
+			.trans = IPATH_TRANS_SQERR2RTS,
+			.opt_param = {
+				[IB_QPT_SMI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_GSI] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UD] = (IB_QP_CUR_STATE | IB_QP_QKEY),
+				[IB_QPT_UC] = IB_QP_CUR_STATE,
+				[IB_QPT_RC] = (IB_QP_CUR_STATE |
+					 IB_QP_MIN_RNR_TIMER),
+			}
+		}
+	},
+	[IB_QPS_ERR] = {
+		[IB_QPS_RESET] = { .trans = IPATH_TRANS_ANY2RST },
+		[IB_QPS_ERR] = { .trans = IPATH_TRANS_ANY2ERR }
+	}
+};
+
+/*
+ * Initialize the QP state to the reset state.
+ */
+static void ipath_reset_qp(struct ipath_qp *qp)
+{
+	qp->remote_qpn = 0;
+	qp->qkey = 0;
+	qp->qp_access_flags = 0;
+	qp->s_hdrwords = 0;
+	qp->s_psn = 0;
+	qp->r_psn = 0;
+	atomic_set(&qp->msn, 0);
+	if (qp->ibqp.qp_type == IB_QPT_RC) {
+		qp->s_state = IB_OPCODE_RC_SEND_LAST;
+		qp->r_state = IB_OPCODE_RC_SEND_LAST;
+	} else {
+		qp->s_state = IB_OPCODE_UC_SEND_LAST;
+		qp->r_state = IB_OPCODE_UC_SEND_LAST;
+	}
+	qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+	qp->s_nak_state = 0;
+	qp->s_rnr_timeout = 0;
+	qp->s_head = 0;
+	qp->s_tail = 0;
+	qp->s_cur = 0;
+	qp->s_last = 0;
+	qp->s_ssn = 1;
+	qp->s_lsn = 0;
+	qp->r_rq.head = 0;
+	qp->r_rq.tail = 0;
+	qp->r_reuse_sge = 0;
+}
+
+/*
+ * Flush send work queue.
+ * The QP s_lock should be held.
+ */
+static void ipath_sqerror_qp(struct ipath_qp *qp, struct ib_wc *wc)
+{
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
+
+	_VERBS_INFO("Send queue error on QP%d/%d: err: %d\n",
+		    qp->ibqp.qp_num, qp->remote_qpn, wc->status);
+
+	spin_lock(&dev->pending_lock);
+	/* XXX What if its already removed by the timeout code? */
+	if (qp->timerwait.next != LIST_POISON1)
+		list_del(&qp->timerwait);
+	if (qp->piowait.next != LIST_POISON1)
+		list_del(&qp->piowait);
+	spin_unlock(&dev->pending_lock);
+
+	ipath_cq_enter(to_icq(qp->ibqp.send_cq), wc, 1);
+	if (++qp->s_last >= qp->s_size)
+		qp->s_last = 0;
+
+	wc->status = IB_WC_WR_FLUSH_ERR;
+
+	while (qp->s_last != qp->s_head) {
+		wc->wr_id = wqe->wr.wr_id;
+		wc->opcode = wc_opcode[wqe->wr.opcode];
+		ipath_cq_enter(to_icq(qp->ibqp.send_cq), wc, 1);
+		if (++qp->s_last >= qp->s_size)
+			qp->s_last = 0;
+		wqe = get_swqe_ptr(qp, qp->s_last);
+	}
+	qp->s_cur = qp->s_tail = qp->s_head;
+	qp->state = IB_QPS_SQE;
+}
+
+/*
+ * Flush both send and receive work queues.
+ * QP r_rq.lock and s_lock should be held.
+ */
+static void ipath_error_qp(struct ipath_qp *qp)
+{
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ib_wc wc;
+
+	_VERBS_INFO("QP%d/%d in error state\n",
+		    qp->ibqp.qp_num, qp->remote_qpn);
+
+	spin_lock(&dev->pending_lock);
+	/* XXX What if its already removed by the timeout code? */
+	if (qp->timerwait.next != LIST_POISON1)
+		list_del(&qp->timerwait);
+	if (qp->piowait.next != LIST_POISON1)
+		list_del(&qp->piowait);
+	spin_unlock(&dev->pending_lock);
+
+	wc.status = IB_WC_WR_FLUSH_ERR;
+	wc.vendor_err = 0;
+	wc.byte_len = 0;
+	wc.imm_data = 0;
+	wc.qp_num = qp->ibqp.qp_num;
+	wc.src_qp = 0;
+	wc.wc_flags = 0;
+	wc.pkey_index = 0;
+	wc.slid = 0;
+	wc.sl = 0;
+	wc.dlid_path_bits = 0;
+	wc.port_num = 0;
+
+	while (qp->s_last != qp->s_head) {
+		struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
+
+		wc.wr_id = wqe->wr.wr_id;
+		wc.opcode = wc_opcode[wqe->wr.opcode];
+		if (++qp->s_last >= qp->s_size)
+			qp->s_last = 0;
+		ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 1);
+	}
+	qp->s_cur = qp->s_tail = qp->s_head;
+	qp->s_hdrwords = 0;
+	qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+
+	wc.opcode = IB_WC_RECV;
+	while (qp->r_rq.tail != qp->r_rq.head) {
+		wc.wr_id = get_rwqe_ptr(&qp->r_rq, qp->r_rq.tail)->wr_id;
+		if (++qp->r_rq.tail >= qp->r_rq.size)
+			qp->r_rq.tail = 0;
+		ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
+	}
+}
+
+static int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+			   int attr_mask)
+{
+	struct ipath_qp *qp = to_iqp(ibqp);
+	enum ib_qp_state cur_state, new_state;
+	u32 req_param, opt_param;
+	unsigned long flags;
+
+	if (attr_mask & IB_QP_CUR_STATE) {
+		cur_state = attr->cur_qp_state;
+		if (cur_state != IB_QPS_RTR &&
+		    cur_state != IB_QPS_RTS &&
+		    cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
+			return -EINVAL;
+		spin_lock_irqsave(&qp->r_rq.lock, flags);
+		spin_lock(&qp->s_lock);
+	} else {
+		spin_lock_irqsave(&qp->r_rq.lock, flags);
+		spin_lock(&qp->s_lock);
+		cur_state = qp->state;
+	}
+
+	if (attr_mask & IB_QP_STATE) {
+		new_state = attr->qp_state;
+		if (new_state < 0 || new_state > IB_QPS_ERR)
+			goto inval;
+	} else
+		new_state = cur_state;
+
+	switch (qp_state_table[cur_state][new_state].trans) {
+	case IPATH_TRANS_INVALID:
+		goto inval;
+
+	case IPATH_TRANS_ANY2RST:
+		ipath_reset_qp(qp);
+		break;
+
+	case IPATH_TRANS_ANY2ERR:
+		ipath_error_qp(qp);
+		break;
+
+	}
+
+	req_param =
+	    qp_state_table[cur_state][new_state].req_param[qp->ibqp.qp_type];
+	opt_param =
+	    qp_state_table[cur_state][new_state].opt_param[qp->ibqp.qp_type];
+
+	if ((req_param & attr_mask) != req_param)
+		goto inval;
+
+	if (attr_mask & ~(req_param | opt_param | IB_QP_STATE))
+		goto inval;
+
+	if (attr_mask & IB_QP_PKEY_INDEX) {
+		struct ipath_ibdev *dev = to_idev(ibqp->device);
+
+		if (attr->pkey_index >= ipath_layer_get_npkeys(dev->ib_unit))
+			goto inval;
+		qp->s_pkey_index = attr->pkey_index;
+	}
+
+	if (attr_mask & IB_QP_DEST_QPN)
+		qp->remote_qpn = attr->dest_qp_num;
+
+	if (attr_mask & IB_QP_SQ_PSN) {
+		qp->s_next_psn = attr->sq_psn;
+		qp->s_last_psn = qp->s_next_psn - 1;
+	}
+
+	if (attr_mask & IB_QP_RQ_PSN)
+		qp->r_psn = attr->rq_psn;
+
+	if (attr_mask & IB_QP_ACCESS_FLAGS)
+		qp->qp_access_flags = attr->qp_access_flags;
+
+	if (attr_mask & IB_QP_AV)
+		qp->remote_ah_attr = attr->ah_attr;
+
+	if (attr_mask & IB_QP_PATH_MTU)
+		qp->path_mtu = attr->path_mtu;
+
+	if (attr_mask & IB_QP_RETRY_CNT)
+		qp->s_retry = qp->s_retry_cnt = attr->retry_cnt;
+
+	if (attr_mask & IB_QP_RNR_RETRY) {
+		qp->s_rnr_retry = attr->rnr_retry;
+		if (qp->s_rnr_retry > 7)
+			qp->s_rnr_retry = 7;
+		qp->s_rnr_retry_cnt = qp->s_rnr_retry;
+	}
+
+	if (attr_mask & IB_QP_MIN_RNR_TIMER)
+		qp->s_min_rnr_timer = attr->min_rnr_timer & 0x1F;
+
+	if (attr_mask & IB_QP_QKEY)
+		qp->qkey = attr->qkey;
+
+	if (attr_mask & IB_QP_PKEY_INDEX)
+		qp->s_pkey_index = attr->pkey_index;
+
+	qp->state = new_state;
+	spin_unlock(&qp->s_lock);
+	spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+
+	/*
+	 * Try to move to ARMED if QP1 changed to the RTS state.
+	 */
+	if (qp->ibqp.qp_num == 1 && new_state == IB_QPS_RTS) {
+		struct ipath_ibdev *dev = to_idev(ibqp->device);
+
+		/*
+		 * Bounce the link even if it was active so the SM will
+		 * reinitialize the SMA's state.
+		 */
+		ipath_kset_linkstate((dev->ib_unit << 16) | IPATH_IB_LINKDOWN);
+		ipath_kset_linkstate((dev->ib_unit << 16) | IPATH_IB_LINKARM);
+	}
+	return 0;
+
+inval:
+	spin_unlock(&qp->s_lock);
+	spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+	return -EINVAL;
+}
+
+/*
+ * Compute the AETH (syndrome + MSN).
+ * The QP s_lock should be held.
+ */
+static u32 ipath_compute_aeth(struct ipath_qp *qp)
+{
+	u32 aeth = atomic_read(&qp->msn) & 0xFFFFFF;
+
+	if (qp->s_nak_state) {
+		aeth |= qp->s_nak_state << 24;
+	} else if (qp->ibqp.srq) {
+		/* Shared receive queues don't generate credits. */
+		aeth |= 0x1F << 24;
+	} else {
+		u32 min, max, x;
+		u32 credits;
+
+		/*
+		 * Compute the number of credits available (RWQEs).
+		 * XXX Not holding the r_rq.lock here so there is a small
+		 * chance that the pair of reads are not atomic.
+		 */
+		credits = qp->r_rq.head - qp->r_rq.tail;
+		if ((int)credits < 0)
+			credits += qp->r_rq.size;
+		/* Binary search the credit table to find the code to use. */
+		min = 0;
+		max = 31;
+		for (;;) {
+			x = (min + max) / 2;
+			if (credit_table[x] == credits)
+				break;
+			if (credit_table[x] > credits)
+				max = x;
+			else if (min == x)
+				break;
+			else
+				min = x;
+		}
+		aeth |= x << 24;
+	}
+	return cpu_to_be32(aeth);
+}
+
+
+static void no_bufs_available(struct ipath_qp *qp, struct ipath_ibdev *dev)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->pending_lock, flags);
+	if (qp->piowait.next == LIST_POISON1)
+		list_add_tail(&qp->piowait, &dev->piowait);
+	spin_unlock_irqrestore(&dev->pending_lock, flags);
+	/*
+	 * Note that as soon as ipath_layer_want_buffer() is called and
+	 * possibly before it returns, ipath_ib_piobufavail()
+	 * could be called.  If we are still in the tasklet function,
+	 * tasklet_schedule() will not call us until the next time
+	 * tasklet_schedule() is called.
+	 * We clear the tasklet flag now since we are committing to return
+	 * from the tasklet function.
+	 */
+	tasklet_unlock(&qp->s_task);
+	ipath_layer_want_buffer(dev->ib_unit);
+	dev->n_piowait++;
+}
+
+/*
+ * Process entries in the send work queue until the queue is exhausted.
+ * Only allow one CPU to send a packet per QP (tasklet).
+ * Otherwise, after we drop the QP lock, two threads could send
+ * packets out of order.
+ * This is similar to do_rc_send() below except we don't have timeouts or
+ * resends.
+ */
+static void do_uc_send(unsigned long data)
+{
+	struct ipath_qp *qp = (struct ipath_qp *)data;
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ipath_swqe *wqe;
+	unsigned long flags;
+	u16 lrh0;
+	u32 hwords;
+	u32 nwords;
+	u32 extra_bytes;
+	u32 bth0;
+	u32 bth2;
+	u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);
+	u32 len;
+	struct ipath_other_headers *ohdr;
+	struct ib_wc wc;
+
+	if (test_and_set_bit(IPATH_S_BUSY, &qp->s_flags))
+		return;
+
+	if (unlikely(qp->remote_ah_attr.dlid ==
+		     ipath_layer_get_lid(dev->ib_unit))) {
+		/* Pass in an uninitialized ib_wc to save stack space. */
+		ipath_ruc_loopback(qp, &wc);
+		clear_bit(IPATH_S_BUSY, &qp->s_flags);
+		return;
+	}
+
+	ohdr = &qp->s_hdr.u.oth;
+	if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
+		ohdr = &qp->s_hdr.u.l.oth;
+
+again:
+	/* Check for a constructed packet to be sent. */
+	if (qp->s_hdrwords != 0) {
+			/*
+			 * If no PIO bufs are available, return.
+			 * An interrupt will call ipath_ib_piobufavail()
+			 * when one is available.
+			 */
+			if (ipath_verbs_send(dev->ib_unit, qp->s_hdrwords,
+					     (uint32_t *) &qp->s_hdr,
+					     qp->s_cur_size, qp->s_cur_sge)) {
+				no_bufs_available(qp, dev);
+				return;
+			}
+		/* Record that we sent the packet and s_hdr is empty. */
+		qp->s_hdrwords = 0;
+	}
+
+	lrh0 = IPS_LRH_BTH;
+	/* header size in 32-bit words LRH+BTH = (8+12)/4. */
+	hwords = 5;
+
+	/*
+	 * The lock is needed to synchronize between
+	 * setting qp->s_ack_state and post_send().
+	 */
+	spin_lock_irqsave(&qp->s_lock, flags);
+
+	if (!(state_ops[qp->state] & IPATH_PROCESS_SEND_OK))
+		goto done;
+
+	bth0 = ipath_layer_get_pkey(dev->ib_unit, qp->s_pkey_index);
+
+	/* Send a request. */
+	wqe = get_swqe_ptr(qp, qp->s_last);
+	switch (qp->s_state) {
+	default:
+		/* Signal the completion of the last send (if there is one). */
+		if (qp->s_last != qp->s_tail) {
+			if (++qp->s_last == qp->s_size)
+				qp->s_last = 0;
+			if (!test_bit(IPATH_S_SIGNAL_REQ_WR, &qp->s_flags) ||
+			    (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
+				wc.wr_id = wqe->wr.wr_id;
+				wc.status = IB_WC_SUCCESS;
+				wc.opcode = wc_opcode[wqe->wr.opcode];
+				wc.vendor_err = 0;
+				wc.byte_len = wqe->length;
+				wc.qp_num = qp->ibqp.qp_num;
+				wc.src_qp = qp->remote_qpn;
+				wc.pkey_index = 0;
+				wc.slid = qp->remote_ah_attr.dlid;
+				wc.sl = qp->remote_ah_attr.sl;
+				wc.dlid_path_bits = 0;
+				wc.port_num = 0;
+				ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc,
+					       0);
+			}
+			wqe = get_swqe_ptr(qp, qp->s_last);
+		}
+		/* Check if send work queue is empty. */
+		if (qp->s_tail == qp->s_head)
+			goto done;
+		/*
+		 * Start a new request.
+		 */
+		qp->s_psn = wqe->psn = qp->s_next_psn;
+		qp->s_sge.sge = wqe->sg_list[0];
+		qp->s_sge.sg_list = wqe->sg_list + 1;
+		qp->s_sge.num_sge = wqe->wr.num_sge;
+		qp->s_len = len = wqe->length;
+		switch (wqe->wr.opcode) {
+		case IB_WR_SEND:
+		case IB_WR_SEND_WITH_IMM:
+			if (len > pmtu) {
+				qp->s_state = IB_OPCODE_UC_SEND_FIRST;
+				len = pmtu;
+				break;
+			}
+			if (wqe->wr.opcode == IB_WR_SEND) {
+				qp->s_state = IB_OPCODE_UC_SEND_ONLY;
+			} else {
+				qp->s_state =
+				    IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE;
+				/* Immediate data comes after the BTH */
+				ohdr->u.imm_data = wqe->wr.imm_data;
+				hwords += 1;
+			}
+			if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+				bth0 |= 1 << 23;
+			break;
+
+		case IB_WR_RDMA_WRITE:
+		case IB_WR_RDMA_WRITE_WITH_IMM:
+			ohdr->u.rc.reth.vaddr =
+			    cpu_to_be64(wqe->wr.wr.rdma.remote_addr);
+			ohdr->u.rc.reth.rkey =
+			    cpu_to_be32(wqe->wr.wr.rdma.rkey);
+			ohdr->u.rc.reth.length = cpu_to_be32(len);
+			hwords += sizeof(struct ib_reth) / 4;
+			if (len > pmtu) {
+				qp->s_state = IB_OPCODE_UC_RDMA_WRITE_FIRST;
+				len = pmtu;
+				break;
+			}
+			if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
+				qp->s_state = IB_OPCODE_UC_RDMA_WRITE_ONLY;
+			} else {
+				qp->s_state =
+				    IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE;
+				/* Immediate data comes after the RETH */
+				ohdr->u.rc.imm_data = wqe->wr.imm_data;
+				hwords += 1;
+				if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+					bth0 |= 1 << 23;
+			}
+			break;
+
+		default:
+			goto done;
+		}
+		if (++qp->s_tail >= qp->s_size)
+			qp->s_tail = 0;
+		break;
+
+	case IB_OPCODE_UC_SEND_FIRST:
+		qp->s_state = IB_OPCODE_UC_SEND_MIDDLE;
+		/* FALLTHROUGH */
+	case IB_OPCODE_UC_SEND_MIDDLE:
+		len = qp->s_len;
+		if (len > pmtu) {
+			len = pmtu;
+			break;
+		}
+		if (wqe->wr.opcode == IB_WR_SEND)
+			qp->s_state = IB_OPCODE_UC_SEND_LAST;
+		else {
+			qp->s_state = IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE;
+			/* Immediate data comes after the BTH */
+			ohdr->u.imm_data = wqe->wr.imm_data;
+			hwords += 1;
+		}
+		if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+			bth0 |= 1 << 23;
+		break;
+
+	case IB_OPCODE_UC_RDMA_WRITE_FIRST:
+		qp->s_state = IB_OPCODE_UC_RDMA_WRITE_MIDDLE;
+		/* FALLTHROUGH */
+	case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:
+		len = qp->s_len;
+		if (len > pmtu) {
+			len = pmtu;
+			break;
+		}
+		if (wqe->wr.opcode == IB_WR_RDMA_WRITE)
+			qp->s_state = IB_OPCODE_UC_RDMA_WRITE_LAST;
+		else {
+			qp->s_state =
+			    IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE;
+			/* Immediate data comes after the BTH */
+			ohdr->u.imm_data = wqe->wr.imm_data;
+			hwords += 1;
+			if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+				bth0 |= 1 << 23;
+		}
+		break;
+	}
+	bth2 = qp->s_next_psn++ & 0xFFFFFF;
+	qp->s_len -= len;
+	bth0 |= qp->s_state << 24;
+
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+
+	/* Construct the header. */
+	extra_bytes = (4 - len) & 3;
+	nwords = (len + extra_bytes) >> 2;
+	if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
+		/* Header size in 32-bit words. */
+		hwords += 10;
+		lrh0 = IPS_LRH_GRH;
+		qp->s_hdr.u.l.grh.version_tclass_flow =
+		    cpu_to_be32((6 << 28) |
+				(qp->remote_ah_attr.grh.traffic_class << 20) |
+				qp->remote_ah_attr.grh.flow_label);
+		qp->s_hdr.u.l.grh.paylen =
+		    cpu_to_be16(((hwords - 12) + nwords + SIZE_OF_CRC) << 2);
+		qp->s_hdr.u.l.grh.next_hdr = 0x1B;
+		qp->s_hdr.u.l.grh.hop_limit = qp->remote_ah_attr.grh.hop_limit;
+		/* The SGID is 32-bit aligned. */
+		qp->s_hdr.u.l.grh.sgid.global.subnet_prefix = dev->gid_prefix;
+		qp->s_hdr.u.l.grh.sgid.global.interface_id =
+		    ipath_layer_get_guid(dev->ib_unit);
+		qp->s_hdr.u.l.grh.dgid = qp->remote_ah_attr.grh.dgid;
+	}
+	qp->s_hdrwords = hwords;
+	qp->s_cur_sge = &qp->s_sge;
+	qp->s_cur_size = len;
+	lrh0 |= qp->remote_ah_attr.sl << 4;
+	qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
+	/* DEST LID */
+	qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
+	qp->s_hdr.lrh[2] = cpu_to_be16(hwords + nwords + SIZE_OF_CRC);
+	qp->s_hdr.lrh[3] = cpu_to_be16(ipath_layer_get_lid(dev->ib_unit));
+	bth0 |= extra_bytes << 20;
+	ohdr->bth[0] = cpu_to_be32(bth0);
+	ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
+	ohdr->bth[2] = cpu_to_be32(bth2);
+
+	/* Check for more work to do. */
+	goto again;
+
+done:
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+	clear_bit(IPATH_S_BUSY, &qp->s_flags);
+}
+
+/*
+ * Process entries in the send work queue until credit or queue is exhausted.
+ * Only allow one CPU to send a packet per QP (tasklet).
+ * Otherwise, after we drop the QP s_lock, two threads could send
+ * packets out of order.
+ */
+static void do_rc_send(unsigned long data)
+{
+	struct ipath_qp *qp = (struct ipath_qp *)data;
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ipath_swqe *wqe;
+	struct ipath_sge_state *ss;
+	unsigned long flags;
+	u16 lrh0;
+	u32 hwords;
+	u32 nwords;
+	u32 extra_bytes;
+	u32 bth0;
+	u32 bth2;
+	u32 pmtu = ib_mtu_enum_to_int(qp->path_mtu);
+	u32 len;
+	struct ipath_other_headers *ohdr;
+	char newreq;
+
+	if (test_and_set_bit(IPATH_S_BUSY, &qp->s_flags))
+		return;
+
+	if (unlikely(qp->remote_ah_attr.dlid ==
+		     ipath_layer_get_lid(dev->ib_unit))) {
+		struct ib_wc wc;
+
+		/*
+		 * Pass in an uninitialized ib_wc to be consistent with
+		 * other places where ipath_ruc_loopback() is called.
+		 */
+		ipath_ruc_loopback(qp, &wc);
+		clear_bit(IPATH_S_BUSY, &qp->s_flags);
+		return;
+	}
+
+	ohdr = &qp->s_hdr.u.oth;
+	if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
+		ohdr = &qp->s_hdr.u.l.oth;
+
+again:
+	/* Check for a constructed packet to be sent. */
+	if (qp->s_hdrwords != 0) {
+			/*
+			 * If no PIO bufs are available, return.
+			 * An interrupt will call ipath_ib_piobufavail()
+			 * when one is available.
+			 */
+			if (ipath_verbs_send(dev->ib_unit, qp->s_hdrwords,
+					     (uint32_t *) &qp->s_hdr,
+					     qp->s_cur_size, qp->s_cur_sge)) {
+				no_bufs_available(qp, dev);
+				return;
+			}
+		/* Record that we sent the packet and s_hdr is empty. */
+		qp->s_hdrwords = 0;
+	}
+
+	lrh0 = IPS_LRH_BTH;
+	/* header size in 32-bit words LRH+BTH = (8+12)/4. */
+	hwords = 5;
+
+	/*
+	 * The lock is needed to synchronize between
+	 * setting qp->s_ack_state, resend timer, and post_send().
+	 */
+	spin_lock_irqsave(&qp->s_lock, flags);
+
+	bth0 = ipath_layer_get_pkey(dev->ib_unit, qp->s_pkey_index);
+
+	/* Sending responses has higher priority over sending requests. */
+	if (qp->s_ack_state != IB_OPCODE_RC_ACKNOWLEDGE) {
+		/*
+		 * Send a response.
+		 * Note that we are in the responder's side of the QP context.
+		 */
+		switch (qp->s_ack_state) {
+		case IB_OPCODE_RC_RDMA_READ_REQUEST:
+			ss = &qp->s_rdma_sge;
+			len = qp->s_rdma_len;
+			if (len > pmtu) {
+				len = pmtu;
+				qp->s_ack_state =
+				    IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST;
+			} else {
+				qp->s_ack_state =
+				    IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY;
+			}
+			qp->s_rdma_len -= len;
+			bth0 |= qp->s_ack_state << 24;
+			ohdr->u.aeth = ipath_compute_aeth(qp);
+			hwords++;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST:
+			qp->s_ack_state =
+			    IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE;
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:
+			ss = &qp->s_rdma_sge;
+			len = qp->s_rdma_len;
+			if (len > pmtu) {
+				len = pmtu;
+			} else {
+				ohdr->u.aeth = ipath_compute_aeth(qp);
+				hwords++;
+				qp->s_ack_state =
+				    IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST;
+			}
+			qp->s_rdma_len -= len;
+			bth0 |= qp->s_ack_state << 24;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST:
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY:
+			/*
+			 * We have to prevent new requests from changing
+			 * the r_sge state while a ipath_verbs_send()
+			 * is in progress.
+			 * Changing r_state allows the receiver
+			 * to continue processing new packets.
+			 * We do it here now instead of above so
+			 * that we are sure the packet was sent before
+			 * changing the state.
+			 */
+			qp->r_state = IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST;
+			qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+			goto send_req;
+
+		case IB_OPCODE_RC_COMPARE_SWAP:
+		case IB_OPCODE_RC_FETCH_ADD:
+			ss = NULL;
+			len = 0;
+			qp->r_state = IB_OPCODE_RC_SEND_LAST;
+			qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+			bth0 |= IB_OPCODE_ATOMIC_ACKNOWLEDGE << 24;
+			ohdr->u.at.aeth = ipath_compute_aeth(qp);
+			ohdr->u.at.atomic_ack_eth =
+			    cpu_to_be64(qp->s_ack_atomic);
+			hwords += sizeof(ohdr->u.at) / 4;
+			break;
+
+		default:
+			/* Send a regular ACK. */
+			ss = NULL;
+			len = 0;
+			qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+			bth0 |= qp->s_ack_state << 24;
+			ohdr->u.aeth = ipath_compute_aeth(qp);
+			hwords++;
+		}
+		bth2 = qp->s_ack_psn++ & 0xFFFFFF;
+	} else {
+	      send_req:
+		if (!(state_ops[qp->state] & IPATH_PROCESS_SEND_OK) ||
+		    qp->s_rnr_timeout)
+			goto done;
+
+		/* Send a request. */
+		wqe = get_swqe_ptr(qp, qp->s_cur);
+		switch (qp->s_state) {
+		default:
+			/*
+			 * Resend an old request or start a new one.
+			 *
+			 * We keep track of the current SWQE so that
+			 * we don't reset the "furthest progress" state
+			 * if we need to back up.
+			 */
+			newreq = 0;
+			if (qp->s_cur == qp->s_tail) {
+				/* Check if send work queue is empty. */
+				if (qp->s_tail == qp->s_head)
+					goto done;
+				qp->s_psn = wqe->psn = qp->s_next_psn;
+				newreq = 1;
+			}
+			/*
+			 * Note that we have to be careful not to modify the
+			 * original work request since we may need to resend
+			 * it.
+			 */
+			qp->s_sge.sge = wqe->sg_list[0];
+			qp->s_sge.sg_list = wqe->sg_list + 1;
+			qp->s_sge.num_sge = wqe->wr.num_sge;
+			qp->s_len = len = wqe->length;
+			ss = &qp->s_sge;
+			bth2 = 0;
+			switch (wqe->wr.opcode) {
+			case IB_WR_SEND:
+			case IB_WR_SEND_WITH_IMM:
+				/* If no credit, return. */
+				if (qp->s_lsn != (u32) -1 &&
+				    cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {
+					goto done;
+				}
+				wqe->lpsn = wqe->psn;
+				if (len > pmtu) {
+					wqe->lpsn += (len - 1) / pmtu;
+					qp->s_state = IB_OPCODE_RC_SEND_FIRST;
+					len = pmtu;
+					break;
+				}
+				if (wqe->wr.opcode == IB_WR_SEND) {
+					qp->s_state = IB_OPCODE_RC_SEND_ONLY;
+				} else {
+					qp->s_state =
+					    IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE;
+					/* Immediate data comes after the BTH */
+					ohdr->u.imm_data = wqe->wr.imm_data;
+					hwords += 1;
+				}
+				if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+					bth0 |= 1 << 23;
+				bth2 = 1 << 31;	/* Request ACK. */
+				if (++qp->s_cur == qp->s_size)
+					qp->s_cur = 0;
+				break;
+
+			case IB_WR_RDMA_WRITE:
+				if (newreq)
+					qp->s_lsn++;
+				/* FALLTHROUGH */
+			case IB_WR_RDMA_WRITE_WITH_IMM:
+				/* If no credit, return. */
+				if (qp->s_lsn != (u32) -1 &&
+				    cmp24(wqe->ssn, qp->s_lsn + 1) > 0) {
+					goto done;
+				}
+				ohdr->u.rc.reth.vaddr =
+				    cpu_to_be64(wqe->wr.wr.rdma.remote_addr);
+				ohdr->u.rc.reth.rkey =
+				    cpu_to_be32(wqe->wr.wr.rdma.rkey);
+				ohdr->u.rc.reth.length = cpu_to_be32(len);
+				hwords += sizeof(struct ib_reth) / 4;
+				wqe->lpsn = wqe->psn;
+				if (len > pmtu) {
+					wqe->lpsn += (len - 1) / pmtu;
+					qp->s_state =
+					    IB_OPCODE_RC_RDMA_WRITE_FIRST;
+					len = pmtu;
+					break;
+				}
+				if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
+					qp->s_state =
+					    IB_OPCODE_RC_RDMA_WRITE_ONLY;
+				} else {
+					qp->s_state =
+					    IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE;
+					/* Immediate data comes after RETH */
+					ohdr->u.rc.imm_data = wqe->wr.imm_data;
+					hwords += 1;
+					if (wqe->wr.
+					    send_flags & IB_SEND_SOLICITED)
+						bth0 |= 1 << 23;
+				}
+				bth2 = 1 << 31;	/* Request ACK. */
+				if (++qp->s_cur == qp->s_size)
+					qp->s_cur = 0;
+				break;
+
+			case IB_WR_RDMA_READ:
+				ohdr->u.rc.reth.vaddr =
+				    cpu_to_be64(wqe->wr.wr.rdma.remote_addr);
+				ohdr->u.rc.reth.rkey =
+				    cpu_to_be32(wqe->wr.wr.rdma.rkey);
+				ohdr->u.rc.reth.length = cpu_to_be32(len);
+				qp->s_state = IB_OPCODE_RC_RDMA_READ_REQUEST;
+				hwords += sizeof(ohdr->u.rc.reth) / 4;
+				if (newreq) {
+					qp->s_lsn++;
+					/*
+					 * Adjust s_next_psn to count the
+					 * expected number of responses.
+					 */
+					if (len > pmtu)
+						qp->s_next_psn +=
+						    (len - 1) / pmtu;
+					wqe->lpsn = qp->s_next_psn++;
+				}
+				ss = NULL;
+				len = 0;
+				if (++qp->s_cur == qp->s_size)
+					qp->s_cur = 0;
+				break;
+
+			case IB_WR_ATOMIC_CMP_AND_SWP:
+			case IB_WR_ATOMIC_FETCH_AND_ADD:
+				qp->s_state =
+				    wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ?
+				    IB_OPCODE_RC_COMPARE_SWAP :
+				    IB_OPCODE_RC_FETCH_ADD;
+				ohdr->u.atomic_eth.vaddr =
+				    cpu_to_be64(wqe->wr.wr.atomic.remote_addr);
+				ohdr->u.atomic_eth.rkey =
+				    cpu_to_be32(wqe->wr.wr.atomic.rkey);
+				ohdr->u.atomic_eth.swap_data =
+				    cpu_to_be64(wqe->wr.wr.atomic.swap);
+				ohdr->u.atomic_eth.compare_data =
+				    cpu_to_be64(wqe->wr.wr.atomic.compare_add);
+				hwords += sizeof(struct ib_atomic_eth) / 4;
+				if (newreq) {
+					qp->s_lsn++;
+					wqe->lpsn = wqe->psn;
+				}
+				if (++qp->s_cur == qp->s_size)
+					qp->s_cur = 0;
+				ss = NULL;
+				len = 0;
+				break;
+
+			default:
+				goto done;
+			}
+			if (newreq) {
+				if (++qp->s_tail >= qp->s_size)
+					qp->s_tail = 0;
+			}
+			bth2 |= qp->s_psn++ & 0xFFFFFF;
+			if ((int)(qp->s_psn - qp->s_next_psn) > 0)
+				qp->s_next_psn = qp->s_psn;
+			spin_lock(&dev->pending_lock);
+			if (qp->timerwait.next == LIST_POISON1) {
+				list_add_tail(&qp->timerwait,
+					      &dev->pending[dev->
+							    pending_index]);
+			}
+			spin_unlock(&dev->pending_lock);
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST:
+			/*
+			 * This case can only happen if a send is
+			 * restarted.  See ipath_restart_rc().
+			 */
+			ipath_init_restart(qp, wqe);
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_SEND_FIRST:
+			qp->s_state = IB_OPCODE_RC_SEND_MIDDLE;
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_SEND_MIDDLE:
+			bth2 = qp->s_psn++ & 0xFFFFFF;
+			if ((int)(qp->s_psn - qp->s_next_psn) > 0)
+				qp->s_next_psn = qp->s_psn;
+			ss = &qp->s_sge;
+			len = qp->s_len;
+			if (len > pmtu) {
+				/*
+				 * Request an ACK every 1/2 MB to avoid
+				 * retransmit timeouts.
+				 */
+				if (((wqe->length - len) % (512 * 1024)) == 0)
+					bth2 |= 1 << 31;
+				len = pmtu;
+				break;
+			}
+			if (wqe->wr.opcode == IB_WR_SEND)
+				qp->s_state = IB_OPCODE_RC_SEND_LAST;
+			else {
+				qp->s_state =
+				    IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE;
+				/* Immediate data comes after the BTH */
+				ohdr->u.imm_data = wqe->wr.imm_data;
+				hwords += 1;
+			}
+			if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+				bth0 |= 1 << 23;
+			bth2 |= 1 << 31;	/* Request ACK. */
+			if (++qp->s_cur >= qp->s_size)
+				qp->s_cur = 0;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST:
+			/*
+			 * This case can only happen if a RDMA write is
+			 * restarted.  See ipath_restart_rc().
+			 */
+			ipath_init_restart(qp, wqe);
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_RDMA_WRITE_FIRST:
+			qp->s_state = IB_OPCODE_RC_RDMA_WRITE_MIDDLE;
+			/* FALLTHROUGH */
+		case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:
+			bth2 = qp->s_psn++ & 0xFFFFFF;
+			if ((int)(qp->s_psn - qp->s_next_psn) > 0)
+				qp->s_next_psn = qp->s_psn;
+			ss = &qp->s_sge;
+			len = qp->s_len;
+			if (len > pmtu) {
+				/*
+				 * Request an ACK every 1/2 MB to avoid
+				 * retransmit timeouts.
+				 */
+				if (((wqe->length - len) % (512 * 1024)) == 0)
+					bth2 |= 1 << 31;
+				len = pmtu;
+				break;
+			}
+			if (wqe->wr.opcode == IB_WR_RDMA_WRITE)
+				qp->s_state = IB_OPCODE_RC_RDMA_WRITE_LAST;
+			else {
+				qp->s_state =
+				    IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE;
+				/* Immediate data comes after the BTH */
+				ohdr->u.imm_data = wqe->wr.imm_data;
+				hwords += 1;
+				if (wqe->wr.send_flags & IB_SEND_SOLICITED)
+					bth0 |= 1 << 23;
+			}
+			bth2 |= 1 << 31;	/* Request ACK. */
+			if (++qp->s_cur >= qp->s_size)
+				qp->s_cur = 0;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:
+			/*
+			 * This case can only happen if a RDMA read is
+			 * restarted.  See ipath_restart_rc().
+			 */
+			ipath_init_restart(qp, wqe);
+			len = ((qp->s_psn - wqe->psn) & 0xFFFFFF) * pmtu;
+			ohdr->u.rc.reth.vaddr =
+			    cpu_to_be64(wqe->wr.wr.rdma.remote_addr + len);
+			ohdr->u.rc.reth.rkey =
+			    cpu_to_be32(wqe->wr.wr.rdma.rkey);
+			ohdr->u.rc.reth.length = cpu_to_be32(qp->s_len);
+			qp->s_state = IB_OPCODE_RC_RDMA_READ_REQUEST;
+			hwords += sizeof(ohdr->u.rc.reth) / 4;
+			bth2 = qp->s_psn++ & 0xFFFFFF;
+			if ((int)(qp->s_psn - qp->s_next_psn) > 0)
+				qp->s_next_psn = qp->s_psn;
+			ss = NULL;
+			len = 0;
+			if (++qp->s_cur == qp->s_size)
+				qp->s_cur = 0;
+			break;
+
+		case IB_OPCODE_RC_RDMA_READ_REQUEST:
+		case IB_OPCODE_RC_COMPARE_SWAP:
+		case IB_OPCODE_RC_FETCH_ADD:
+			/*
+			 * We shouldn't start anything new until this request
+			 * is finished.  The ACK will handle rescheduling us.
+			 * XXX The number of outstanding ones is negotiated
+			 * at connection setup time (see pg. 258,289)?
+			 * XXX Also, if we support multiple outstanding
+			 * requests, we need to check the WQE IB_SEND_FENCE
+			 * flag and not send a new request if a RDMA read or
+			 * atomic is pending.
+			 */
+			goto done;
+		}
+		qp->s_len -= len;
+		bth0 |= qp->s_state << 24;
+		/* XXX queue resend timeout. */
+	}
+	/* Make sure it is non-zero before dropping the lock. */
+	qp->s_hdrwords = hwords;
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+
+	/* Construct the header. */
+	extra_bytes = (4 - len) & 3;
+	nwords = (len + extra_bytes) >> 2;
+	if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
+		/* Header size in 32-bit words. */
+		hwords += 10;
+		lrh0 = IPS_LRH_GRH;
+		qp->s_hdr.u.l.grh.version_tclass_flow =
+		    cpu_to_be32((6 << 28) |
+				(qp->remote_ah_attr.grh.traffic_class << 20) |
+				qp->remote_ah_attr.grh.flow_label);
+		qp->s_hdr.u.l.grh.paylen =
+		    cpu_to_be16(((hwords - 12) + nwords + SIZE_OF_CRC) << 2);
+		qp->s_hdr.u.l.grh.next_hdr = 0x1B;
+		qp->s_hdr.u.l.grh.hop_limit = qp->remote_ah_attr.grh.hop_limit;
+		/* The SGID is 32-bit aligned. */
+		qp->s_hdr.u.l.grh.sgid.global.subnet_prefix = dev->gid_prefix;
+		qp->s_hdr.u.l.grh.sgid.global.interface_id =
+		    ipath_layer_get_guid(dev->ib_unit);
+		qp->s_hdr.u.l.grh.dgid = qp->remote_ah_attr.grh.dgid;
+		qp->s_hdrwords = hwords;
+	}
+	qp->s_cur_sge = ss;
+	qp->s_cur_size = len;
+	lrh0 |= qp->remote_ah_attr.sl << 4;
+	qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
+	/* DEST LID */
+	qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
+	qp->s_hdr.lrh[2] = cpu_to_be16(hwords + nwords + SIZE_OF_CRC);
+	qp->s_hdr.lrh[3] = cpu_to_be16(ipath_layer_get_lid(dev->ib_unit));
+	bth0 |= extra_bytes << 20;
+	ohdr->bth[0] = cpu_to_be32(bth0);
+	ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
+	ohdr->bth[2] = cpu_to_be32(bth2);
+
+	/* Check for more work to do. */
+	goto again;
+
+done:
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+	clear_bit(IPATH_S_BUSY, &qp->s_flags);
+}
+
+static void send_rc_ack(struct ipath_qp *qp)
+{
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	u16 lrh0;
+	u32 bth0;
+	u32 hwords;
+	struct ipath_other_headers *ohdr;
+
+	/* Construct the header. */
+	ohdr = &qp->s_hdr.u.oth;
+	lrh0 = IPS_LRH_BTH;
+	/* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4. */
+	hwords = 6;
+	if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
+		ohdr = &qp->s_hdr.u.l.oth;
+		/* Header size in 32-bit words. */
+		hwords += 10;
+		lrh0 = IPS_LRH_GRH;
+		qp->s_hdr.u.l.grh.version_tclass_flow =
+		    cpu_to_be32((6 << 28) |
+				(qp->remote_ah_attr.grh.traffic_class << 20) |
+				qp->remote_ah_attr.grh.flow_label);
+		qp->s_hdr.u.l.grh.paylen =
+		    cpu_to_be16(((hwords - 12) + SIZE_OF_CRC) << 2);
+		qp->s_hdr.u.l.grh.next_hdr = 0x1B;
+		qp->s_hdr.u.l.grh.hop_limit = qp->remote_ah_attr.grh.hop_limit;
+		/* The SGID is 32-bit aligned. */
+		qp->s_hdr.u.l.grh.sgid.global.subnet_prefix = dev->gid_prefix;
+		qp->s_hdr.u.l.grh.sgid.global.interface_id =
+		    ipath_layer_get_guid(dev->ib_unit);
+		qp->s_hdr.u.l.grh.dgid = qp->remote_ah_attr.grh.dgid;
+	}
+	bth0 = ipath_layer_get_pkey(dev->ib_unit, qp->s_pkey_index);
+	ohdr->u.aeth = ipath_compute_aeth(qp);
+	if (qp->s_ack_state >= IB_OPCODE_RC_COMPARE_SWAP) {
+		bth0 |= IB_OPCODE_ATOMIC_ACKNOWLEDGE << 24;
+		ohdr->u.at.atomic_ack_eth = cpu_to_be64(qp->s_ack_atomic);
+		hwords += sizeof(ohdr->u.at.atomic_ack_eth) / 4;
+	} else {
+		bth0 |= IB_OPCODE_RC_ACKNOWLEDGE << 24;
+	}
+	lrh0 |= qp->remote_ah_attr.sl << 4;
+	qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
+	/* DEST LID */
+	qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
+	qp->s_hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC);
+	qp->s_hdr.lrh[3] = cpu_to_be16(ipath_layer_get_lid(dev->ib_unit));
+	ohdr->bth[0] = cpu_to_be32(bth0);
+	ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
+	ohdr->bth[2] = cpu_to_be32(qp->s_ack_psn & 0xFFFFFF);
+
+	/*
+	 * If we can send the ACK, clear the ACK state.
+	 */
+	if (ipath_verbs_send(dev->ib_unit, hwords, (uint32_t *) &qp->s_hdr,
+			     0, NULL) == 0) {
+		qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
+		dev->n_rc_qacks++;
+	}
+}
+
+/*
+ * Back up the requester to resend the last un-ACKed request.
+ * The QP s_lock should be held.
+ */
+static void ipath_restart_rc(struct ipath_qp *qp, u32 psn, struct ib_wc *wc)
+{
+	struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
+	struct ipath_ibdev *dev;
+	u32 n;
+
+	/*
+	 * If there are no requests pending, we are done.
+	 */
+	if (cmp24(psn, qp->s_next_psn) >= 0 || qp->s_last == qp->s_tail)
+		goto done;
+
+	if (qp->s_retry == 0) {
+		wc->wr_id = wqe->wr.wr_id;
+		wc->status = IB_WC_RETRY_EXC_ERR;
+		wc->opcode = wc_opcode[wqe->wr.opcode];
+		wc->vendor_err = 0;
+		wc->byte_len = 0;
+		wc->qp_num = qp->ibqp.qp_num;
+		wc->src_qp = qp->remote_qpn;
+		wc->pkey_index = 0;
+		wc->slid = qp->remote_ah_attr.dlid;
+		wc->sl = qp->remote_ah_attr.sl;
+		wc->dlid_path_bits = 0;
+		wc->port_num = 0;
+		ipath_sqerror_qp(qp, wc);
+		return;
+	}
+	qp->s_retry--;
+
+	/*
+	 * Remove the QP from the timeout queue.
+	 * Note: it may already have been removed by ipath_ib_timer().
+	 */
+	dev = to_idev(qp->ibqp.device);
+	spin_lock(&dev->pending_lock);
+	if (qp->timerwait.next != LIST_POISON1)
+		list_del(&qp->timerwait);
+	spin_unlock(&dev->pending_lock);
+
+	if (wqe->wr.opcode == IB_WR_RDMA_READ)
+		dev->n_rc_resends++;
+	else
+		dev->n_rc_resends += (int)qp->s_psn - (int)psn;
+
+	/*
+	 * If we are starting the request from the beginning, let the
+	 * normal send code handle initialization.
+	 */
+	qp->s_cur = qp->s_last;
+	if (cmp24(psn, wqe->psn) <= 0) {
+		qp->s_state = IB_OPCODE_RC_SEND_LAST;
+		qp->s_psn = wqe->psn;
+	} else {
+		n = qp->s_cur;
+		for (;;) {
+			if (++n == qp->s_size)
+				n = 0;
+			if (n == qp->s_tail) {
+				if (cmp24(psn, qp->s_next_psn) >= 0) {
+					qp->s_cur = n;
+					wqe = get_swqe_ptr(qp, n);
+				}
+				break;
+			}
+			wqe = get_swqe_ptr(qp, n);
+			if (cmp24(psn, wqe->psn) < 0)
+				break;
+			qp->s_cur = n;
+		}
+		qp->s_psn = psn;
+
+		/*
+		 * Reset the state to restart in the middle of a request.
+		 * Don't change the s_sge, s_cur_sge, or s_cur_size.
+		 * See do_rc_send().
+		 */
+		switch (wqe->wr.opcode) {
+		case IB_WR_SEND:
+		case IB_WR_SEND_WITH_IMM:
+			qp->s_state = IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST;
+			break;
+
+		case IB_WR_RDMA_WRITE:
+		case IB_WR_RDMA_WRITE_WITH_IMM:
+			qp->s_state = IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST;
+			break;
+
+		case IB_WR_RDMA_READ:
+			qp->s_state = IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE;
+			break;
+
+		default:
+			/*
+			 * This case shouldn't happen since its only
+			 * one PSN per req.
+			 */
+			qp->s_state = IB_OPCODE_RC_SEND_LAST;
+		}
+	}
+
+done:
+	tasklet_schedule(&qp->s_task);
+}
+
+/*
+ * Handle RC and UC post sends.
+ */
+static int ipath_post_rc_send(struct ipath_qp *qp, struct ib_send_wr *wr)
+{
+	struct ipath_swqe *wqe;
+	unsigned long flags;
+	u32 next;
+	int i, j;
+	int acc;
+
+	/*
+	 * Don't allow RDMA reads or atomic operations on UC or
+	 * undefined operations.
+	 * Make sure buffer is large enough to hold the result for atomics.
+	 */
+	if (qp->ibqp.qp_type == IB_QPT_UC) {
+		if ((unsigned) wr->opcode >= IB_WR_RDMA_READ)
+			return -EINVAL;
+	} else if ((unsigned) wr->opcode > IB_WR_ATOMIC_FETCH_AND_ADD)
+		return -EINVAL;
+	else if (wr->opcode >= IB_WR_ATOMIC_CMP_AND_SWP &&
+		 (wr->num_sge == 0 || wr->sg_list[0].length < sizeof(u64) ||
+		  wr->sg_list[0].addr & 0x7))
+		return -EINVAL;
+
+	/* IB spec says that num_sge == 0 is OK. */
+	if (wr->num_sge > qp->s_max_sge)
+		return -ENOMEM;
+
+	spin_lock_irqsave(&qp->s_lock, flags);
+	next = qp->s_head + 1;
+	if (next >= qp->s_size)
+		next = 0;
+	if (next == qp->s_last) {
+		spin_unlock_irqrestore(&qp->s_lock, flags);
+		return -EINVAL;
+	}
+
+	wqe = get_swqe_ptr(qp, qp->s_head);
+	wqe->wr = *wr;
+	wqe->ssn = qp->s_ssn++;
+	wqe->sg_list[0].mr = NULL;
+	wqe->sg_list[0].vaddr = NULL;
+	wqe->sg_list[0].length = 0;
+	wqe->sg_list[0].sge_length = 0;
+	wqe->length = 0;
+	acc = wr->opcode >= IB_WR_RDMA_READ ? IB_ACCESS_LOCAL_WRITE : 0;
+	for (i = 0, j = 0; i < wr->num_sge; i++) {
+		if (to_ipd(qp->ibqp.pd)->user && wr->sg_list[i].lkey == 0) {
+			spin_unlock_irqrestore(&qp->s_lock, flags);
+			return -EINVAL;
+		}
+		if (wr->sg_list[i].length == 0)
+			continue;
+		if (!ipath_lkey_ok(&to_idev(qp->ibqp.device)->lk_table,
+				   &wqe->sg_list[j], &wr->sg_list[i], acc)) {
+			spin_unlock_irqrestore(&qp->s_lock, flags);
+			return -EINVAL;
+		}
+		wqe->length += wr->sg_list[i].length;
+		j++;
+	}
+	wqe->wr.num_sge = j;
+	qp->s_head = next;
+	/*
+	 * Wake up the send tasklet if the QP is not waiting
+	 * for an RNR timeout.
+	 */
+	next = qp->s_rnr_timeout;
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+
+	if (next == 0) {
+		if (qp->ibqp.qp_type == IB_QPT_UC)
+			do_uc_send((unsigned long) qp);
+		else
+			do_rc_send((unsigned long) qp);
+	}
+	return 0;
+}
+
+/*
+ * Note that we actually send the data as it is posted instead of putting
+ * the request into a ring buffer.  If we wanted to use a ring buffer,
+ * we would need to save a reference to the destination address in the SWQE.
+ */
+static int ipath_post_ud_send(struct ipath_qp *qp, struct ib_send_wr *wr)
+{
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	struct ipath_other_headers *ohdr;
+	struct ib_ah_attr *ah_attr;
+	struct ipath_sge_state ss;
+	struct ipath_sge *sg_list;
+	struct ib_wc wc;
+	u32 hwords;
+	u32 nwords;
+	u32 len;
+	u32 extra_bytes;
+	u32 bth0;
+	u16 lrh0;
+	u16 lid;
+	int i;
+
+	if (!(state_ops[qp->state] & IPATH_PROCESS_SEND_OK))
+		return 0;
+
+	/* IB spec says that num_sge == 0 is OK. */
+	if (wr->num_sge > qp->s_max_sge)
+		return -EINVAL;
+
+	if (wr->num_sge > 1) {
+		sg_list = kmalloc((qp->s_max_sge - 1) * sizeof(*sg_list),
+				  GFP_ATOMIC);
+		if (!sg_list)
+			return -ENOMEM;
+	} else
+		sg_list = NULL;
+
+	/* Check the buffer to send. */
+	ss.sg_list = sg_list;
+	ss.sge.mr = NULL;
+	ss.sge.vaddr = NULL;
+	ss.sge.length = 0;
+	ss.sge.sge_length = 0;
+	ss.num_sge = 0;
+	len = 0;
+	for (i = 0; i < wr->num_sge; i++) {
+		/* Check LKEY */
+		if (to_ipd(qp->ibqp.pd)->user && wr->sg_list[i].lkey == 0)
+			return -EINVAL;
+
+		if (wr->sg_list[i].length == 0)
+			continue;
+		if (!ipath_lkey_ok(&dev->lk_table, ss.num_sge ?
+				   sg_list + ss.num_sge : &ss.sge,
+				   &wr->sg_list[i], 0)) {
+			return -EINVAL;
+		}
+		len += wr->sg_list[i].length;
+		ss.num_sge++;
+	}
+	extra_bytes = (4 - len) & 3;
+	nwords = (len + extra_bytes) >> 2;
+
+	/* Construct the header. */
+	ah_attr = &to_iah(wr->wr.ud.ah)->attr;
+	if (ah_attr->dlid >= 0xC000 && ah_attr->dlid < 0xFFFF)
+		dev->n_multicast_xmit++;
+	if (unlikely(ah_attr->dlid == ipath_layer_get_lid(dev->ib_unit))) {
+		/* Pass in an uninitialized ib_wc to save stack space. */
+		ipath_ud_loopback(qp, &ss, len, wr, &wc);
+		goto done;
+	}
+	if (ah_attr->ah_flags & IB_AH_GRH) {
+		/* Header size in 32-bit words. */
+		hwords = 17;
+		lrh0 = IPS_LRH_GRH;
+		ohdr = &qp->s_hdr.u.l.oth;
+		qp->s_hdr.u.l.grh.version_tclass_flow =
+		    cpu_to_be32((6 << 28) |
+				(ah_attr->grh.traffic_class << 20) |
+				ah_attr->grh.flow_label);
+		qp->s_hdr.u.l.grh.paylen =
+		    cpu_to_be16(((wr->opcode ==
+				  IB_WR_SEND_WITH_IMM ? 6 : 5) + nwords +
+				 SIZE_OF_CRC) << 2);
+		qp->s_hdr.u.l.grh.next_hdr = 0x1B;
+		qp->s_hdr.u.l.grh.hop_limit = ah_attr->grh.hop_limit;
+		/* The SGID is 32-bit aligned. */
+		qp->s_hdr.u.l.grh.sgid.global.subnet_prefix = dev->gid_prefix;
+		qp->s_hdr.u.l.grh.sgid.global.interface_id =
+		    ipath_layer_get_guid(dev->ib_unit);
+		qp->s_hdr.u.l.grh.dgid = ah_attr->grh.dgid;
+		/*
+		 * Don't worry about sending to locally attached
+		 * multicast QPs.  It is unspecified by the spec. what happens.
+		 */
+	} else {
+		/* Header size in 32-bit words. */
+		hwords = 7;
+		lrh0 = IPS_LRH_BTH;
+		ohdr = &qp->s_hdr.u.oth;
+	}
+	if (wr->opcode == IB_WR_SEND_WITH_IMM) {
+		ohdr->u.ud.imm_data = wr->imm_data;
+		wc.imm_data = wr->imm_data;
+		hwords += 1;
+		bth0 = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE << 24;
+	} else if (wr->opcode == IB_WR_SEND) {
+		wc.imm_data = 0;
+		bth0 = IB_OPCODE_UD_SEND_ONLY << 24;
+	} else
+		return -EINVAL;
+	lrh0 |= ah_attr->sl << 4;
+	if (qp->ibqp.qp_type == IB_QPT_SMI)
+		lrh0 |= 0xF000;	/* Set VL */
+	qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
+	qp->s_hdr.lrh[1] = cpu_to_be16(ah_attr->dlid);	/* DEST LID */
+	qp->s_hdr.lrh[2] = cpu_to_be16(hwords + nwords + SIZE_OF_CRC);
+	lid = ipath_layer_get_lid(dev->ib_unit);
+	qp->s_hdr.lrh[3] = lid ? cpu_to_be16(lid) : IB_LID_PERMISSIVE;
+	if (wr->send_flags & IB_SEND_SOLICITED)
+		bth0 |= 1 << 23;
+	bth0 |= extra_bytes << 20;
+	bth0 |= qp->ibqp.qp_type == IB_QPT_SMI ? IPS_DEFAULT_P_KEY :
+	    ipath_layer_get_pkey(dev->ib_unit, qp->s_pkey_index);
+	ohdr->bth[0] = cpu_to_be32(bth0);
+	ohdr->bth[1] = cpu_to_be32(wr->wr.ud.remote_qpn);
+	/* XXX Could lose a PSN count but not worth locking */
+	ohdr->bth[2] = cpu_to_be32(qp->s_psn++ & 0xFFFFFF);
+	/*
+	 * Qkeys with the high order bit set mean use the
+	 * qkey from the QP context instead of the WR.
+	 */
+	ohdr->u.ud.deth[0] = cpu_to_be32((int)wr->wr.ud.remote_qkey < 0 ?
+					 qp->qkey : wr->wr.ud.remote_qkey);
+	ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num);
+	if (ipath_verbs_send(dev->ib_unit, hwords, (uint32_t *) &qp->s_hdr,
+			     len, &ss))
+		dev->n_no_piobuf++;
+
+done:
+	/* Queue the completion status entry. */
+	if (!test_bit(IPATH_S_SIGNAL_REQ_WR, &qp->s_flags) ||
+	    (wr->send_flags & IB_SEND_SIGNALED)) {
+		wc.wr_id = wr->wr_id;
+		wc.status = IB_WC_SUCCESS;
+		wc.vendor_err = 0;
+		wc.opcode = IB_WC_SEND;
+		wc.byte_len = len;
+		wc.qp_num = qp->ibqp.qp_num;
+		wc.src_qp = 0;
+		wc.wc_flags = 0;
+		/* XXX initialize other fields? */
+		ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc, 0);
+	}
+	kfree(sg_list);
+
+	return 0;
+}
+
+/*
+ * This may be called from interrupt context.
+ */
+static int ipath_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
+			   struct ib_send_wr **bad_wr)
+{
+	struct ipath_qp *qp = to_iqp(ibqp);
+	int err = 0;
+
+	/* Check that state is OK to post send. */
+	if (!(state_ops[qp->state] & IPATH_POST_SEND_OK)) {
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+
+	for (; wr; wr = wr->next) {
+		switch (qp->ibqp.qp_type) {
+		case IB_QPT_UC:
+		case IB_QPT_RC:
+			err = ipath_post_rc_send(qp, wr);
+			break;
+
+		case IB_QPT_SMI:
+		case IB_QPT_GSI:
+		case IB_QPT_UD:
+			err = ipath_post_ud_send(qp, wr);
+			break;
+
+		default:
+			err = -EINVAL;
+		}
+		if (err) {
+			*bad_wr = wr;
+			break;
+		}
+	}
+	return err;
+}
+
+/*
+ * This may be called from interrupt context.
+ */
+static int ipath_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
+			      struct ib_recv_wr **bad_wr)
+{
+	struct ipath_qp *qp = to_iqp(ibqp);
+	unsigned long flags;
+
+	/* Check that state is OK to post receive. */
+	if (!(state_ops[qp->state] & IPATH_POST_RECV_OK)) {
+		*bad_wr = wr;
+		return -EINVAL;
+	}
+
+	for (; wr; wr = wr->next) {
+		struct ipath_rwqe *wqe;
+		u32 next;
+		int i, j;
+
+		if (wr->num_sge > qp->r_rq.max_sge) {
+			*bad_wr = wr;
+			return -ENOMEM;
+		}
+
+		spin_lock_irqsave(&qp->r_rq.lock, flags);
+		next = qp->r_rq.head + 1;
+		if (next >= qp->r_rq.size)
+			next = 0;
+		if (next == qp->r_rq.tail) {
+			spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+			*bad_wr = wr;
+			return -ENOMEM;
+		}
+
+		wqe = get_rwqe_ptr(&qp->r_rq, qp->r_rq.head);
+		wqe->wr_id = wr->wr_id;
+		wqe->sg_list[0].mr = NULL;
+		wqe->sg_list[0].vaddr = NULL;
+		wqe->sg_list[0].length = 0;
+		wqe->sg_list[0].sge_length = 0;
+		wqe->length = 0;
+		for (i = 0, j = 0; i < wr->num_sge; i++) {
+			/* Check LKEY */
+			if (to_ipd(qp->ibqp.pd)->user &&
+			    wr->sg_list[i].lkey == 0) {
+				spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+				*bad_wr = wr;
+				return -EINVAL;
+			}
+			if (wr->sg_list[i].length == 0)
+				continue;
+			if (!ipath_lkey_ok(&to_idev(qp->ibqp.device)->lk_table,
+					   &wqe->sg_list[j], &wr->sg_list[i],
+					   IB_ACCESS_LOCAL_WRITE)) {
+				spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+				*bad_wr = wr;
+				return -EINVAL;
+			}
+			wqe->length += wr->sg_list[i].length;
+			j++;
+		}
+		wqe->num_sge = j;
+		qp->r_rq.head = next;
+		spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+	}
+	return 0;
+}
+
+/*
+ * This may be called from interrupt context.
+ */
+static int ipath_post_srq_receive(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
+				  struct ib_recv_wr **bad_wr)
+{
+	struct ipath_srq *srq = to_isrq(ibsrq);
+	struct ipath_ibdev *dev = to_idev(ibsrq->device);
+	unsigned long flags;
+
+	for (; wr; wr = wr->next) {
+		struct ipath_rwqe *wqe;
+		u32 next;
+		int i, j;
+
+		if (wr->num_sge > srq->rq.max_sge) {
+			*bad_wr = wr;
+			return -ENOMEM;
+		}
+
+		spin_lock_irqsave(&srq->rq.lock, flags);
+		next = srq->rq.head + 1;
+		if (next >= srq->rq.size)
+			next = 0;
+		if (next == srq->rq.tail) {
+			spin_unlock_irqrestore(&srq->rq.lock, flags);
+			*bad_wr = wr;
+			return -ENOMEM;
+		}
+
+		wqe = get_rwqe_ptr(&srq->rq, srq->rq.head);
+		wqe->wr_id = wr->wr_id;
+		wqe->sg_list[0].mr = NULL;
+		wqe->sg_list[0].vaddr = NULL;
+		wqe->sg_list[0].length = 0;
+		wqe->sg_list[0].sge_length = 0;
+		wqe->length = 0;
+		for (i = 0, j = 0; i < wr->num_sge; i++) {
+			/* Check LKEY */
+			if (to_ipd(srq->ibsrq.pd)->user &&
+			    wr->sg_list[i].lkey == 0) {
+				spin_unlock_irqrestore(&srq->rq.lock, flags);
+				*bad_wr = wr;
+				return -EINVAL;
+			}
+			if (wr->sg_list[i].length == 0)
+				continue;
+			if (!ipath_lkey_ok(&dev->lk_table,
+					   &wqe->sg_list[j], &wr->sg_list[i],
+					   IB_ACCESS_LOCAL_WRITE)) {
+				spin_unlock_irqrestore(&srq->rq.lock, flags);
+				*bad_wr = wr;
+				return -EINVAL;
+			}
+			wqe->length += wr->sg_list[i].length;
+			j++;
+		}
+		wqe->num_sge = j;
+		srq->rq.head = next;
+		spin_unlock_irqrestore(&srq->rq.lock, flags);
+	}
+	return 0;
+}
+
+/*
+ * This is called from ipath_qp_rcv() to process an incomming UD packet
+ * for the given QP.
+ * Called at interrupt level.
+ */
+static void ipath_ud_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
+			 int has_grh, void *data, u32 tlen, struct ipath_qp *qp)
+{
+	struct ipath_other_headers *ohdr;
+	int opcode;
+	u32 hdrsize;
+	u32 pad;
+	unsigned long flags;
+	struct ib_wc wc;
+	u32 qkey;
+	u32 src_qp;
+	struct ipath_rq *rq;
+	struct ipath_srq *srq;
+	struct ipath_rwqe *wqe;
+
+	/* Check for GRH */
+	if (!has_grh) {
+		ohdr = &hdr->u.oth;
+		hdrsize = 8 + 12 + 8;	/* LRH + BTH + DETH */
+		qkey = be32_to_cpu(ohdr->u.ud.deth[0]);
+		src_qp = be32_to_cpu(ohdr->u.ud.deth[1]);
+	} else {
+		ohdr = &hdr->u.l.oth;
+		hdrsize = 8 + 40 + 12 + 8;	/* LRH + GRH + BTH + DETH */
+		/*
+		 * The header with GRH is 68 bytes and the
+		 * core driver sets the eager header buffer
+		 * size to 56 bytes so the last 12 bytes of
+		 * the IB header is in the data buffer.
+		 */
+		qkey = be32_to_cpu(((u32 *) data)[1]);
+		src_qp = be32_to_cpu(((u32 *) data)[2]);
+		data += 12;
+	}
+	src_qp &= 0xFFFFFF;
+
+	/* Check that the qkey matches. */
+	if (unlikely(qkey != qp->qkey)) {
+		/* XXX OK to lose a count once in a while. */
+		dev->qkey_violations++;
+		dev->n_pkt_drops++;
+		return;
+	}
+
+	/* Get the number of bytes the message was padded by. */
+	pad = (ohdr->bth[0] >> 12) & 3;
+	if (unlikely(tlen < (hdrsize + pad + 4))) {
+		/* Drop incomplete packets. */
+		dev->n_pkt_drops++;
+		return;
+	}
+
+	/*
+	 * A GRH is expected to preceed the data even if not
+	 * present on the wire.
+	 */
+	wc.byte_len = tlen - (hdrsize + pad + 4) + sizeof(struct ib_grh);
+
+	/*
+	 * The opcode is in the low byte when its in network order
+	 * (top byte when in host order).
+	 */
+	opcode = *(u8 *) (&ohdr->bth[0]);
+	if (opcode == IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE) {
+		if (has_grh) {
+			wc.imm_data = *(u32 *) data;
+			data += sizeof(u32);
+		} else
+			wc.imm_data = ohdr->u.ud.imm_data;
+		wc.wc_flags = IB_WC_WITH_IMM;
+		hdrsize += sizeof(u32);
+	} else if (opcode == IB_OPCODE_UD_SEND_ONLY) {
+		wc.imm_data = 0;
+		wc.wc_flags = 0;
+	} else {
+		dev->n_pkt_drops++;
+		return;
+	}
+
+	/*
+	 * Get the next work request entry to find where to put the data.
+	 * Note that it is safe to drop the lock after changing rq->tail
+	 * since ipath_post_receive() won't fill the empty slot.
+	 */
+	if (qp->ibqp.srq) {
+		srq = to_isrq(qp->ibqp.srq);
+		rq = &srq->rq;
+	} else {
+		srq = NULL;
+		rq = &qp->r_rq;
+	}
+	spin_lock_irqsave(&rq->lock, flags);
+	if (rq->tail == rq->head) {
+		spin_unlock_irqrestore(&rq->lock, flags);
+		dev->n_pkt_drops++;
+		return;
+	}
+	/* Silently drop packets which are too big. */
+	wqe = get_rwqe_ptr(rq, rq->tail);
+	if (wc.byte_len > wqe->length) {
+		spin_unlock_irqrestore(&rq->lock, flags);
+		dev->n_pkt_drops++;
+		return;
+	}
+	wc.wr_id = wqe->wr_id;
+	qp->r_sge.sge = wqe->sg_list[0];
+	qp->r_sge.sg_list = wqe->sg_list + 1;
+	qp->r_sge.num_sge = wqe->num_sge;
+	if (++rq->tail >= rq->size)
+		rq->tail = 0;
+	if (srq && srq->ibsrq.event_handler) {
+		u32 n;
+
+		if (rq->head < rq->tail)
+			n = rq->size + rq->head - rq->tail;
+		else
+			n = rq->head - rq->tail;
+		if (n < srq->limit) {
[… more lines in the raw message]

Lines of this file are not shown. View the raw message.

The rest of this diff is not shown. View the raw message.

Signature · 1 line
0.99.9n

Part of the diff is not shown here. View the raw message.