ipath - almost-final round of patches for submission
PATCH 1/1 ipath - core device driver
In reply to Bryan O'SullivanSigned-off-by Bryan O'Sullivan <[email protected]>
drivers/infiniband/hw/ipath/ipath_driver.c new file +2022 -0
@@ -0,0 +1,2022 @@ +/* +* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved. +* +* This software is available to you under a choice of one of two +* licenses. You may choose to be licensed under the terms of the GNU +* General Public License (GPL) Version 2, available from the file +* COPYING in the main directory of this source tree, or the +* OpenIB.org BSD license below: +* +* Redistribution and use in source and binary forms, with or +* without modification, are permitted provided that the following +* conditions are met: +* +* - Redistributions of source code must retain the above +* copyright notice, this list of conditions and the following +* disclaimer. +* +* - Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials +* provided with the distribution. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ + +#include <linux/spinlock.h> +#include <linux/idr.h> +#include <linux/pci.h> +#include <linux/delay.h> +#include <linux/netdevice.h> +#include <linux/vmalloc.h> + +/* +* InfiniPath only supports message-based interrupts. It has no interrupt +* pins. Therefore we require CONFIG_PCI_MSI be enabled. We want a +* very visible failure if the support isn't present. +*/ +#ifndef CONFIG_PCI_MSI +#error "Without CONFIG_PCI_MSI, interrupts will not work" +#endif + +#include "ipath_kernel.h" +#include "ips_common.h" +#include "ipath_layer.h" + +static void ipath_update_pio_bufs (struct ipath_devdata *); + +const char *ipath_get_unit_name (int unit) +{ +static char iname[16 ]; +snprintf (iname,sizeof iname,"infinipath%u" , unit); +return iname; +} + +EXPORT_SYMBOL_GPL (ipath_get_unit_name); + +#define DRIVER_LOAD_MSG "PathScale " IPATH_DRV_NAME " loaded: " +#define PFX IPATH_DRV_NAME ": " + +/* +* The size has to be longer than this string, so we can append +* board/chip information to it in the init code. +*/ +const char ipath_core_version[] =IPATH_IDSTR "\n" ; + +static struct idr unit_table; +DEFINE_SPINLOCK (ipath_devs_lock); +LIST_HEAD (ipath_dev_list); + +unsigned ipath_debug = __IPATH_INFO; + +module_param_named (debug, ipath_debug, uint,S_IWUSR |S_IRUGO ); +MODULE_PARM_DESC (debug,"mask for debug prints" ); +EXPORT_SYMBOL_GPL (ipath_debug); + +MODULE_LICENSE ("GPL" ); +MODULE_AUTHOR ("PathScale <[email protected]>" ); +MODULE_DESCRIPTION ("Pathscale InfiniPath driver" ); + +const char *ipath_ibcstatus_str[] = { +"Disabled" , +"LinkUp" , +"PollActive" , +"PollQuiet" , +"SleepDelay" , +"SleepQuiet" , +"LState6" ,/* unused */ +"LState7" ,/* unused */ +"CfgDebounce" , +"CfgRcvfCfg" , +"CfgWaitRmt" , +"CfgIdle" , +"RecovRetrain" , +"LState0xD" ,/* unused */ +"RecovWaitRmt" , +"RecovIdle" , +}; + +/* +* These variables are initialized in the chip-specific files +* but are defined here. +*/ +u16 ipath_gpio_sda_num, ipath_gpio_scl_num; +u64 ipath_gpio_sda, ipath_gpio_scl; +u64 infinipath_i_bitsextant, infinipath_e_bitsextant, + infinipath_hwe_bitsextant; +u32 infinipath_i_rcvavail_mask, infinipath_i_rcvurg_mask; + +static void __devexit ipath_remove_one(struct pci_dev *); +static int __devinit ipath_init_one (struct pci_dev *, +const struct pci_device_id *); + +/* Only needed for registration, nothing else needs this info */ +#define PCI_VENDOR_ID_PATHSCALE 0x1fc1 +#define PCI_DEVICE_ID_INFINIPATH_HT 0xd +#define PCI_DEVICE_ID_INFINIPATH_PE800 0x10 + +static const struct pci_device_id ipath_pci_tbl[] = { + {PCI_DEVICE (PCI_VENDOR_ID_PATHSCALE , +PCI_DEVICE_ID_INFINIPATH_HT )}, + {PCI_DEVICE (PCI_VENDOR_ID_PATHSCALE , +PCI_DEVICE_ID_INFINIPATH_PE800 )}, +}; + +MODULE_DEVICE_TABLE (pci, ipath_pci_tbl); + +static struct pci_driver ipath_driver = { + .name =IPATH_DRV_NAME , + .probe = ipath_init_one, + .remove =__devexit_p (ipath_remove_one), + .id_table = ipath_pci_tbl, +}; + +/* +* This is where port 0's rcvhdrtail register is written back; we also +* want nothing else sharing the cache line, so make it a cache line +* in size. Used for all units. +*/ +u64 *ipath_port0_rcvhdrtail; +dma_addr_t ipath_port0_rcvhdrtail_dma; +static int port0_rcvhdrtail_refs; + +static inline void read_bars (struct ipath_devdata *dd,struct pci_dev *dev, +u32 *bar0,u32 *bar1) +{ +int ret; + + ret =pci_read_config_dword (dev,PCI_BASE_ADDRESS_0 , bar0); +if (ret) +ipath_dev_err (dd,"failed to read bar0 before enable: " +"error %d\n" , -ret); + + ret =pci_read_config_dword (dev,PCI_BASE_ADDRESS_1 , bar1); +if (ret) +ipath_dev_err (dd,"failed to read bar1 before enable: " +"error %d\n" , -ret); + +ipath_dbg ("Read bar0 %x bar1 %x\n" , *bar0, *bar1); +} + +static void ipath_free_devdata (struct pci_dev *pdev, +struct ipath_devdata *dd) +{ +unsigned long flags; + +if (dd->ipath_unit !=-1 ) { +spin_lock_irqsave (&ipath_devs_lock, flags); +idr_remove (&unit_table, dd->ipath_unit); +list_del (&dd->ipath_list); +spin_unlock_irqrestore (&ipath_devs_lock, flags); + } +dma_free_coherent (&pdev->dev,sizeof (*dd), dd, dd->ipath_dma_addr); +} + +static struct ipath_devdata *ipath_alloc_devdata (struct pci_dev *pdev) +{ +unsigned long flags; +struct ipath_devdata *dd; +dma_addr_t dma_addr; +int ret; + +if (!idr_pre_get (&unit_table,GFP_KERNEL )) { + dd =ERR_PTR (-ENOMEM ); + goto bail; + } + + dd =dma_alloc_coherent (&pdev->dev,sizeof (*dd), &dma_addr, +GFP_KERNEL ); + +if (!dd) { + dd =ERR_PTR (-ENOMEM ); + goto bail; + } + + dd->ipath_dma_addr = dma_addr; + dd->ipath_unit =-1 ; + +spin_lock_irqsave (&ipath_devs_lock, flags); + + ret =idr_get_new (&unit_table, dd, &dd->ipath_unit); +if (ret <0 ) { + printk(KERN_ERR IPATH_DRV_NAME + ": Could not allocate unit ID: error %d\n", -ret); +ipath_free_devdata (pdev, dd); + dd =ERR_PTR (ret); + goto bail_unlock; + } + +list_add (&dd->ipath_list, &ipath_dev_list); + +bail_unlock: +spin_unlock_irqrestore (&ipath_devs_lock, flags); + +bail: +return dd; +} + +static inline struct ipath_devdata *__ipath_lookup (int unit) +{ +return idr_find (&unit_table, unit); +} + +struct ipath_devdata *ipath_lookup (int unit) +{ +struct ipath_devdata *dd; +unsigned long flags; + +spin_lock_irqsave (&ipath_devs_lock, flags); + dd =__ipath_lookup (unit); +spin_unlock_irqrestore (&ipath_devs_lock, flags); + +return dd; +} + +int ipath_count_units (int *npresentp,int *nupp,u32 *maxportsp) +{ +int nunits, npresent, nup; +struct ipath_devdata *dd; +unsigned long flags; +u32 maxports; + + nunits = npresent = nup = maxports =0 ; + +spin_lock_irqsave (&ipath_devs_lock, flags); + +list_for_each_entry (dd, &ipath_dev_list, ipath_list) { + nunits++; +if ((dd->ipath_flags &IPATH_PRESENT ) && dd->ipath_kregbase) + npresent++; +if (dd->ipath_lid && + !(dd->ipath_flags & (IPATH_DISABLED |IPATH_LINKDOWN + |IPATH_LINKUNK ))) + nup++; +if (dd->ipath_cfgports > maxports) + maxports = dd->ipath_cfgports; + } + +spin_unlock_irqrestore (&ipath_devs_lock, flags); + +if (npresentp) + *npresentp = npresent; +if (nupp) + *nupp = nup; +if (maxportsp) + *maxportsp = maxports; + +return nunits; +} + +static int init_port0_rcvhdrtail (struct pci_dev *pdev) +{ +int ret; + +mutex_lock (&ipath_mutex); + +if (!ipath_port0_rcvhdrtail) { + ipath_port0_rcvhdrtail = +dma_alloc_coherent (&pdev->dev, +IPATH_PORT0_RCVHDRTAIL_SIZE , + &ipath_port0_rcvhdrtail_dma, +GFP_KERNEL ); + +if (!ipath_port0_rcvhdrtail) { + ret = -ENOMEM ; + goto bail; + } + } + port0_rcvhdrtail_refs++; + ret =0 ; + +bail: +mutex_unlock (&ipath_mutex); + +return ret; +} + +static void cleanup_port0_rcvhdrtail (struct pci_dev *pdev) +{ +mutex_lock (&ipath_mutex); + +if (!--port0_rcvhdrtail_refs) { +dma_free_coherent (&pdev->dev,IPATH_PORT0_RCVHDRTAIL_SIZE , + ipath_port0_rcvhdrtail, + ipath_port0_rcvhdrtail_dma); + ipath_port0_rcvhdrtail =NULL ; + } + +mutex_unlock (&ipath_mutex); +} + +/* +* These next two routines are placeholders in case we don't have per-arch +* code for controlling write combining. If explicit control of write +* combining is not available, performance will probably be awful. +*/ + +int __attribute__((weak))ipath_enable_wc (struct ipath_devdata *dd) +{ +return -EOPNOTSUPP ; +} + +void __attribute__((weak))ipath_disable_wc (struct ipath_devdata *dd) +{ +} + +static int __devinit ipath_init_one (struct pci_dev *pdev, +const struct pci_device_id *ent) +{ +int ret, len, j; +struct ipath_devdata *dd; +unsigned long long addr; +u32 bar0 =0 , bar1 =0 ; +u8 rev; + + ret =init_port0_rcvhdrtail (pdev); +if (ret <0 ) { + printk(KERN_ERR IPATH_DRV_NAME +": Could not allocate port0_rcvhdrtail: error %d\n" , + -ret); + goto fail; + } + + dd =ipath_alloc_devdata (pdev); +if (IS_ERR (dd)) { + ret =PTR_ERR (dd); + printk(KERN_ERR IPATH_DRV_NAME + ": Could not allocate devdata: error %d\n", -ret); + goto fail; + } + +ipath_cdbg (VERBOSE ,"initializing unit #%u\n" , dd->ipath_unit); + +read_bars (dd, pdev, &bar0, &bar1); + + ret =pci_enable_device (pdev); +if (ret) { +/* This can happen if we did a chip reset, and then failed +* to reprogram the BAR, or the chip reset due to an +* internal error. Both of these cases cause the BAR to be +* reset back to initial state. For the latter case, the AER +* sticky error bit at offset 0x718 should be set, but the +* Linux kernel doesn't yet know about that, it appears. If +* the original BAR was retained in the kernel data +* structures, this may be OK +*/ +ipath_dbg ("pci_enable unit %u failed: error %d\n" , + dd->ipath_unit, -ret); + } + addr =pci_resource_start (pdev,0 ); + len =pci_resource_len (pdev,0 ); +ipath_cdbg (VERBOSE ,"regbase (0) %llx len %d irq %x, vend %x/%x " +"driver_data %lx\n" , addr, len, pdev->irq, ent->vendor, + ent->device, ent->driver_data); + +read_bars (dd, pdev, &bar0, &bar1); + +if (!bar1 && !(bar0 & ~0xf )) { +if (addr) { +dev_info (&pdev->dev,"BAR is 0 (probable RESET), " +"rewriting as %llx\n" , addr); + ret =pci_write_config_dword ( + pdev,PCI_BASE_ADDRESS_0 , addr); +if (ret) +ipath_dev_err (dd,"rewrite of BAR0 " +"failed: err %d\n" , -ret); + ret =pci_write_config_dword ( + pdev,PCI_BASE_ADDRESS_1 , addr >>32 ); +if (ret) +ipath_dev_err (dd,"rewrite of BAR1 " +"failed: err %d\n" , -ret); + } +else +ipath_dev_err (dd,"BAR is 0 (probable RESET), " +"not usable until reboot\n" ); + } + + ret =pci_request_regions (pdev,IPATH_DRV_NAME ); +if (ret) +dev_info (&pdev->dev,"pci_request_regions unit %u fails: " +"err %d\n" , dd->ipath_unit, -ret); + + ret =pci_set_dma_mask (pdev,DMA_64BIT_MASK ); +if (ret) +dev_info (&pdev->dev,"pci_set_dma_mask unit %u " +"fails: %d\n" , dd->ipath_unit, ret); + +pci_set_master (pdev);/* probably not be needed for HT */ + +/* +* Save BARs to rewrite after device reset. Save all 64 bits of +* BAR, just in case. +*/ + dd->ipath_pcibar0 = addr; + dd->ipath_pcibar1 = addr >>32 ; + dd->ipath_deviceid = ent->device;/* save for later use */ + dd->ipath_vendorid = ent->vendor; + +/* setup the chip-specific functions, as early as possible. */ +switch (ent->device) { +case PCI_DEVICE_ID_INFINIPATH_HT : +ipath_init_ht400_funcs (dd); +break ; +case PCI_DEVICE_ID_INFINIPATH_PE800 : +ipath_init_pe800_funcs (dd); +break ; +default : +ipath_dev_err (dd,"Found unknown PathScale deviceid 0x%x, " +"failing\n" , ent->device); +return -ENODEV ; + } + +for (j =0 ; j <6 ; j++) { +if (!pdev->resource[j].start) +continue ; +ipath_cdbg (VERBOSE ,"BAR %d start %lx, end %lx, len %lx\n" , + j, pdev->resource[j].start, + pdev->resource[j].end, +pci_resource_len (pdev, j)); + } + +pci_set_master (pdev); + +if (!addr) { +ipath_dev_err (dd,"No valid address in BAR 0!\n" ); +return -ENODEV ; + } + + dd->ipath_deviceid = ent->device;/* save for later use */ + dd->ipath_vendorid = ent->vendor; + + ret =pci_read_config_byte (pdev,PCI_REVISION_ID , &rev); +if (ret) { +ipath_dev_err (dd,"Failed to read PCI revision ID unit " +"%u: err %d\n" , dd->ipath_unit, -ret); +return ret;/* shouldn't ever happen */ + }else + dd->ipath_pcirev = rev; + + dd->ipath_kregbase =ioremap_nocache (addr, len); +if (!dd->ipath_kregbase) { +ipath_dbg ("Unable to map io addr %llx to kvirt, failing\n" , + addr); + ret = -ENOMEM ; + goto fail; + } + dd->ipath_kregend = (u64 __iomem *) + ((void __iomem *)dd->ipath_kregbase + len); + dd->ipath_physaddr = addr;/* used for io_remap, etc. */ +/* for user mmap */ + dd->ipath_kregvirt = (u64 __iomem *)phys_to_virt (addr); +ipath_cdbg (VERBOSE ,"mapped io addr %llx to kregbase %p " +"kregvirt %p\n" , addr, dd->ipath_kregbase, + dd->ipath_kregvirt); + +/* set these up before registering the interrupt handler */ + dd->pcidev = pdev; +pci_set_drvdata (pdev, dd); + +/* +* clear ipath_flags here instead of in ipath_init_chip as it is set +* by ipath_setup_htconfig. +*/ + dd->ipath_flags =0 ; + +if (dd->ipath_f_bus (dd, pdev)) +ipath_dev_err (dd,"Failed to setup config space; " +"continuing anyway\n" ); + +/* +* set up our interrupt handler; SA_SHIRQ probably not needed, +* since MSI interrupts shouldn't be shared but won't hurt for now. +* check 0 irq after we return from chip-specific bus setup, since +* that can affect this due to setup +*/ +if (!pdev->irq) +ipath_dev_err (dd,"irq is 0, BIOS error? Interrupts won't " +"work\n" ); +else { + ret =request_irq (pdev->irq, ipath_intr,SA_SHIRQ , +IPATH_DRV_NAME , dd); +if (ret) +ipath_dev_err (dd,"Couldn't setup irq handler, " +"irq=%u: %d\n" , pdev->irq, ret); + } + + ret =ipath_init_chip (dd,0 );/* do the chip-specific init */ + +if (!ret) { + ret =ipath_enable_wc (dd); + +if (ret) { +ipath_dev_err (dd,"Write combining not enabled " +"(err %d): performance may be poor\n" , + -ret); + ret =0 ; + } + } + +ipath_device_create_group (&pdev->dev, dd); +ipath_user_add (dd); + +ipath_layer_add (dd); + +/* +* We used to cleanup here, with pci_release_regions, etc. but that +* can cause other problems if we want to run diags, etc., so +* instead defer that until driver unload. As long as the driver is +* unloaded, no memory leaks result. +*/ + +fail:/* after we've done at least some of the pci setup */ +if (ret == -EPERM ) +/* +* disabled device, don't want module load error; just want +* to carry status through to this point +*/ + ret =0 ; + +return ret; +} + +static void __devexit ipath_remove_one(struct pci_dev *pdev) +{ +struct ipath_devdata *dd; + +ipath_cdbg (VERBOSE ,"removing, pdev=%p\n" , pdev); +if (!pdev) +return ; + + dd =pci_get_drvdata (pdev); +ipath_layer_del (dd); +ipath_user_del (dd); +ipath_device_remove_group (&pdev->dev, dd); +pci_set_drvdata (pdev,NULL ); +ipath_cdbg (VERBOSE ,"Releasing pci memory regions, dd %p, " +"unit %u\n" , dd, (u32 ) dd->ipath_unit); +if (dd->ipath_kregbase) { +ipath_cdbg (VERBOSE ,"Unmapping kregbase %p\n" , + dd->ipath_kregbase); +iounmap ((volatile void __iomem *)dd->ipath_kregbase); + dd->ipath_kregbase =NULL ; + } +pci_release_regions (pdev); +ipath_cdbg (VERBOSE ,"calling pci_disable_device\n" ); +pci_disable_device (pdev); + +ipath_free_devdata (pdev, dd); +cleanup_port0_rcvhdrtail (pdev); +} + +/* general driver use */ +DEFINE_MUTEX (ipath_mutex); +static DEFINE_SPINLOCK (ipath_pioavail_lock); + +/** +* ipath_disarm_piobufs - cancel a range of PIO buffers +* @dd: the infinipath device +* @first: the first PIO buffer to cancel +* @cnt: the number of PIO buffers to cancel +* +* cancel a range of PIO buffers, used when they might be armed, but +* not triggered. Used at init to ensure buffer state, and also user +* process close, in case it died while writing to a PIO buffer +* Also after errors. +*/ +void ipath_disarm_piobufs (struct ipath_devdata *dd,unsigned first, +unsigned cnt) +{ +unsigned i, last = first + cnt; +u64 sendctrl, sendorig; + +ipath_cdbg (PKT ,"disarm %u PIObufs first=%u\n" , cnt, first); + sendorig = dd->ipath_sendctrl |INFINIPATH_S_DISARM ; +for (i = first; i < last; i++) { + sendctrl = sendorig | + (i <<INFINIPATH_S_DISARMPIOBUF_SHIFT ); +ipath_write_kreg (dd, dd->ipath_kregs->kr_sendctrl, + sendctrl); + } +// write it again with current value, in case ipath_sendctrl changed +// while we were looping; no critical bits that would require +// locking +ipath_write_kreg (dd, dd->ipath_kregs->kr_sendctrl, + dd->ipath_sendctrl); +} + +/** +* ipath_wait_linkstate - wait for an IB link state change to occur +* @dd: the infinipath device +* @state: the state to wait for +* @msecs: the number of milliseconds to wait +* +* wait up to msecs milliseconds for IB link state change to occur for +* now, take the easy polling route. Currently used only by +* ipath_layer_set_linkstate. Returns 0 if state reached, otherwise +* -ETIMEDOUT state can have multiple states set, for any of several +* transitions. +*/ +int ipath_wait_linkstate (struct ipath_devdata *dd,u32 state,int msecs) +{ + dd->ipath_sma_state_wanted = state; +wait_event_interruptible_timeout (ipath_sma_state_wait, + (dd->ipath_flags & state), +msecs_to_jiffies (msecs)); + dd->ipath_sma_state_wanted =0 ; + +if (!(dd->ipath_flags & state)) { +u64 val; +ipath_cdbg (SMA ,"Didn't reach linkstate %s within %u ms\n" , +/* test INIT ahead of DOWN, both can be set */ + (state &IPATH_LINKINIT ) ?"INIT" : + ((state &IPATH_LINKDOWN ) ?"DOWN" : + ((state &IPATH_LINKARMED ) ?"ARM" :"ACTIVE" )), + msecs); + val =ipath_read_kreg64 (dd, dd->ipath_kregs->kr_ibcstatus); +ipath_cdbg (VERBOSE ,"ibcc=%llx ibcstatus=%llx (%s)\n" , + (unsigned long long )ipath_read_kreg64 ( + dd, dd->ipath_kregs->kr_ibcctrl), + (unsigned long long ) val, + ipath_ibcstatus_str[val &0xf ]); + } +return (dd->ipath_flags & state) ?0 : -ETIMEDOUT ; +} + +void ipath_decode_err (char *buf,size_t blen,ipath_err_t err) +{ + *buf ='\0' ; +if (err &INFINIPATH_E_RHDRLEN ) +strlcat (buf,"rhdrlen " , blen); +if (err &INFINIPATH_E_RBADTID ) +strlcat (buf,"rbadtid " , blen); +if (err &INFINIPATH_E_RBADVERSION ) +strlcat (buf,"rbadversion " , blen); +if (err &INFINIPATH_E_RHDR ) +strlcat (buf,"rhdr " , blen); +if (err &INFINIPATH_E_RLONGPKTLEN ) +strlcat (buf,"rlongpktlen " , blen); +if (err &INFINIPATH_E_RSHORTPKTLEN ) +strlcat (buf,"rshortpktlen " , blen); +if (err &INFINIPATH_E_RMAXPKTLEN ) +strlcat (buf,"rmaxpktlen " , blen); +if (err &INFINIPATH_E_RMINPKTLEN ) +strlcat (buf,"rminpktlen " , blen); +if (err &INFINIPATH_E_RFORMATERR ) +strlcat (buf,"rformaterr " , blen); +if (err &INFINIPATH_E_RUNSUPVL ) +strlcat (buf,"runsupvl " , blen); +if (err &INFINIPATH_E_RUNEXPCHAR ) +strlcat (buf,"runexpchar " , blen); +if (err &INFINIPATH_E_RIBFLOW ) +strlcat (buf,"ribflow " , blen); +if (err &INFINIPATH_E_REBP ) +strlcat (buf,"EBP " , blen); +if (err &INFINIPATH_E_SUNDERRUN ) +strlcat (buf,"sunderrun " , blen); +if (err &INFINIPATH_E_SPIOARMLAUNCH ) +strlcat (buf,"spioarmlaunch " , blen); +if (err &INFINIPATH_E_SUNEXPERRPKTNUM ) +strlcat (buf,"sunexperrpktnum " , blen); +if (err &INFINIPATH_E_SDROPPEDDATAPKT ) +strlcat (buf,"sdroppeddatapkt " , blen); +if (err &INFINIPATH_E_SDROPPEDSMPPKT ) +strlcat (buf,"sdroppedsmppkt " , blen); +if (err &INFINIPATH_E_SMAXPKTLEN ) +strlcat (buf,"smaxpktlen " , blen); +if (err &INFINIPATH_E_SMINPKTLEN ) +strlcat (buf,"sminpktlen " , blen); +if (err &INFINIPATH_E_SUNSUPVL ) +strlcat (buf,"sunsupVL " , blen); +if (err &INFINIPATH_E_SPKTLEN ) +strlcat (buf,"spktlen " , blen); +if (err &INFINIPATH_E_INVALIDADDR ) +strlcat (buf,"invalidaddr " , blen); +if (err &INFINIPATH_E_RICRC ) +strlcat (buf,"CRC " , blen); +if (err &INFINIPATH_E_RVCRC ) +strlcat (buf,"VCRC " , blen); +if (err &INFINIPATH_E_RRCVEGRFULL ) +strlcat (buf,"rcvegrfull " , blen); +if (err &INFINIPATH_E_RRCVHDRFULL ) +strlcat (buf,"rcvhdrfull " , blen); +if (err &INFINIPATH_E_IBSTATUSCHANGED ) +strlcat (buf,"ibcstatuschg " , blen); +if (err &INFINIPATH_E_RIBLOSTLINK ) +strlcat (buf,"riblostlink " , blen); +if (err &INFINIPATH_E_HARDWARE ) +strlcat (buf,"hardware " , blen); +if (err &INFINIPATH_E_RESET ) +strlcat (buf,"reset " , blen); +} + +/** +* get_rhf_errstring - decode RHF errors +* @err: the err number +* @msg: the output buffer +* @len: the length of the output buffer +* +* only used one place now, may want more later +*/ +static void get_rhf_errstring (u32 err,char *msg,size_t len) +{ +/* if no errors, and so don't need to check what's first */ + *msg ='\0' ; + +if (err &INFINIPATH_RHF_H_ICRCERR ) +strlcat (msg,"icrcerr " , len); +if (err &INFINIPATH_RHF_H_VCRCERR ) +strlcat (msg,"vcrcerr " , len); +if (err &INFINIPATH_RHF_H_PARITYERR ) +strlcat (msg,"parityerr " , len); +if (err &INFINIPATH_RHF_H_LENERR ) +strlcat (msg,"lenerr " , len); +if (err &INFINIPATH_RHF_H_MTUERR ) +strlcat (msg,"mtuerr " , len); +if (err &INFINIPATH_RHF_H_IHDRERR ) +/* infinipath hdr checksum error */ +strlcat (msg,"ipathhdrerr " , len); +if (err &INFINIPATH_RHF_H_TIDERR ) +strlcat (msg,"tiderr " , len); +if (err &INFINIPATH_RHF_H_MKERR ) +/* bad port, offset, etc. */ +strlcat (msg,"invalid ipathhdr " , len); +if (err &INFINIPATH_RHF_H_IBERR ) +strlcat (msg,"iberr " , len); +if (err &INFINIPATH_RHF_L_SWA ) +strlcat (msg,"swA " , len); +if (err &INFINIPATH_RHF_L_SWB ) +strlcat (msg,"swB " , len); +} + +/** +* ipath_get_egrbuf - get an eager buffer +* @dd: the infinipath device +* @bufnum: the eager buffer to get +* @err: unused +* +* must only be called if ipath_pd[port] is known to be allocated +*/ +static inline void *ipath_get_egrbuf (struct ipath_devdata *dd,u32 bufnum, +int err) +{ +return dd->ipath_port0_skbs ? + (void *)dd->ipath_port0_skbs[bufnum]->data :NULL ; +} + +/* +* ipath_rcv_sma - receive an sma packet +* @dd: the infinipath device +* @tlen: the total packet len +* @rc: the packet header +* @ebuf: the packet data +* +* Separate for better overall optimization +*/ +static void ipath_rcv_sma (struct ipath_devdata *dd,u32 tlen,u64 * rc, +void *ebuf) +{ +int sindex, slen, elen; +void *smbuf; +u8 pad, *bthbytes; + +/* another SMA packet received */ + ipath_stats.sps_sma_rpkts++; + + bthbytes = (u8 *) ((struct ips_message_header *)&rc[1 ])->bth; + + pad = (bthbytes[1 ] >>4 ) &3 ; + elen = tlen - (IPATH_SMA_HDRSZ + pad + (u32 )sizeof (u32)); +if (elen > (IPATH_SMA_MAX_PKTSZ -IPATH_SMA_HDRSZ )) + elen =IPATH_SMA_MAX_PKTSZ -IPATH_SMA_HDRSZ ; + +spin_lock_irq (&ipath_sma_lock); + sindex = ipath_sma_next; + smbuf = ipath_sma_data[sindex].buf; + ipath_sma_data[sindex].unit = dd->ipath_unit; + slen = ipath_sma_data[ipath_sma_next].len; +memcpy (smbuf, &rc[1 ],IPATH_SMA_HDRSZ ); +memcpy (smbuf +IPATH_SMA_HDRSZ , ebuf, elen); +if (slen) { +/* +* overwriting a yet unread old one (buffer wrap), +* have to advance ipath_sma_first to next oldest +*/ + +/* count OK packets that we drop */ + ipath_stats.sps_krdrops++; +if (++ipath_sma_first >=IPATH_NUM_SMA_PKTS ) + ipath_sma_first =0 ; + } + slen = ipath_sma_data[sindex].len = elen +IPATH_SMA_HDRSZ ; +if (++ipath_sma_next >=IPATH_NUM_SMA_PKTS ) + ipath_sma_next =0 ; +spin_unlock_irq (&ipath_sma_lock); +} + +/** +* ipath_alloc_skb - allocate an skb and buffer with possible constraints +* @dd: the infinipath device +* @gfp_mask: the sk_buff SFP mask +*/ +struct sk_buff *ipath_alloc_skb (struct ipath_devdata *dd, +gfp_t gfp_mask) +{ +struct sk_buff *skb; +u32 len; + +/* +* Only fully supported way to handle this is to allocate lots +* extra, align as needed, and then do skb_reserve(). That wastes +* a lot of memory... I'll have to hack this into infinipath_copy +* also. +*/ + +/* +* We need 4 extra bytes for unaligned transfer copying +*/ +if (dd->ipath_flags &IPATH_4BYTE_TID ) { +/* we need a 4KB multiple alignment, and there is no way +* to do it except to allocate extra and then skb_reserve +* enough to bring it up to the right alignment. +*/ + len = dd->ipath_ibmaxlen +4 + (1 <<11 ) -1 ; + } +else + len = dd->ipath_ibmaxlen +4 ; + skb =__dev_alloc_skb (len, gfp_mask); +if (!skb) { +ipath_dev_err (dd,"Failed to allocate skbuff, length %u\n" , + len); +return NULL ; + } +if (dd->ipath_flags &IPATH_4BYTE_TID ) { +u32 una = ((1 <<11 ) -1 ) & (unsigned long )(skb->data +4 ); +if (una) +skb_reserve (skb,4 + (1 <<11 ) - una); +else +skb_reserve (skb,4 ); + }else +skb_reserve (skb,4 ); +return skb; +} + +/** +* ipath_rcv_layer - receive a packet for the layered (ethernet) driver +* @dd: the infinipath device +* @etail: the sk_buff number +* @tlen: the total packet length +* @hdr: the ethernet header +* +* Separate routine for better overall optimization +*/ +static void ipath_rcv_layer (struct ipath_devdata *dd,u32 etail, +u32 tlen,struct ether_header *hdr) +{ +u32 elen; +u8 pad, *bthbytes; +struct sk_buff *skb, *nskb; + +if (dd->ipath_port0_skbs && hdr->sub_opcode ==OPCODE_ENCAP ) { +/* +* Allocate a new sk_buff to replace the one we give +* to the network stack. +*/ + nskb =ipath_alloc_skb (dd,GFP_ATOMIC ); +if (!nskb) { +/* count OK packets that we drop */ + ipath_stats.sps_krdrops++; +return ; + } + + bthbytes = (u8 *) hdr->bth; + pad = (bthbytes[1 ] >>4 ) &3 ; +/* +CRC32 */ + elen = tlen - (sizeof (*hdr) + pad +sizeof (u32)); + + skb = dd->ipath_port0_skbs[etail]; + dd->ipath_port0_skbs[etail] = nskb; +skb_put (skb, elen); + + dd->ipath_f_put_tid (dd, etail + (u64 __iomem *) + ((char __iomem *) dd->ipath_kregbase + + dd->ipath_rcvegrbase),0 , +virt_to_phys (nskb->data)); + +__ipath_layer_rcv (dd, hdr, skb); + +/* another ether packet received */ + ipath_stats.sps_ether_rpkts++; + } +else if (hdr->sub_opcode ==OPCODE_LID_ARP ) +__ipath_layer_rcv_lid (dd, hdr); +} + +/* +* ipath_kreceive - receive a packet +* @dd: the infinipath device +* +* called from interrupt handler for errors or receive interrupt +*/ +void ipath_kreceive (struct ipath_devdata *dd) +{ +u64 *rc; +void *ebuf; +const u32 rsize = dd->ipath_rcvhdrentsize;/* words */ +const u32 maxcnt = dd->ipath_rcvhdrcnt * rsize;/* words */ +u32 etail =-1 , l, hdrqtail, sma_this_time =0 ; +struct ips_message_header *hdr; +u32 eflags, i, etype, tlen, pkttot =0 ; +static u64 totcalls;/* stats, may eventually remove */ +char emsg[128 ]; + +if (!dd->ipath_hdrqtailptr) { +ipath_dev_err (dd, +"hdrqtailptr not set, can't do receives\n" ); +return ; + } + +if (test_and_set_bit (0 , &dd->ipath_rcv_pending)) { +/* There is already a thread processing this queue. */ +return ; + } + +if (dd->ipath_port0head == + (uint32_t )__le64_to_cpu (*dd->ipath_hdrqtailptr)) + goto done; + +gotmore: +/* +* read only once at start. If in flood situation, this helps +* performance slightly. If more arrive while we are processing, +* we'll come back here and do them +*/ + hdrqtail = (uint32_t )__le64_to_cpu (*dd->ipath_hdrqtailptr); + +for (i =0 , l = dd->ipath_port0head; l != hdrqtail; i++) { +u32 qp; +u8 *bthbytes; + + rc = (u64 *) (dd->ipath_pd[0 ]->port_rcvhdrq + (l <<2 )); + hdr = (struct ips_message_header *)&rc[1 ]; +/* +* could make a network order version of IPATH_KD_QP, and +* do the obvious shift before masking to speed this up. +*/ + qp =ntohl (hdr->bth[1 ]) &0xffffff ; + bthbytes = (u8 *) hdr->bth; + + eflags =ips_get_hdr_err_flags ((u32 *) rc); + etype =ips_get_rcv_type ((u32 *) rc); +/* total length */ + tlen =ips_get_length_in_bytes ((u32 *) rc); + ebuf =NULL ; +if (etype !=RCVHQ_RCV_TYPE_EXPECTED ) { +/* +* it turns out that the chips uses an eager buffer +* for all non-expected packets, whether it "needs" +* one or not. So always get the index, but don't +* set ebuf (so we try to copy data) unless the +* length requires it. +*/ + etail =ips_get_index ((u32 *) rc); +if (tlen >sizeof (*hdr) || + etype ==RCVHQ_RCV_TYPE_NON_KD ) + ebuf =ipath_get_egrbuf (dd, etail,0 ); + } + +/* +* both tiderr and ipathhdrerr are set for all plain IB +* packets; only ipathhdrerr should be set. +*/ + +if (etype !=RCVHQ_RCV_TYPE_NON_KD && + etype !=RCVHQ_RCV_TYPE_ERROR && +ips_get_ipath_ver (hdr->iph.ver_port_tid_offset) != +IPS_PROTO_VERSION ) { +ipath_cdbg (PKT ,"Bad InfiniPath protocol version " +"%x\n" , etype); + } + +if (eflags & ~(INFINIPATH_RHF_H_TIDERR | +INFINIPATH_RHF_H_IHDRERR )) { +get_rhf_errstring (eflags, emsg,sizeof emsg); +ipath_cdbg (PKT ,"RHFerrs %x hdrqtail=%x typ=%u " +"tlen=%x opcode=%x egridx=%x: %s\n" , + eflags, l, etype, tlen, bthbytes[0 ], +ips_get_index ((u32 *) rc), emsg); + }else if (etype ==RCVHQ_RCV_TYPE_NON_KD ) { +/* +* If there is a userland SMA and this is a MAD +* packet, then pass it to the userland SMA. +* +* We don't care if we miss a few SMA packets here, +* or get a few too many, due to unsynchronised +* access to ipath_sma_alive. +*/ +if (ipath_sma_alive && qp <=1 ) { +/* +* count OK packets that we drop because SMA +* isn't yet running, or because we are in +* an sma flood (no point in constantly +* acquiring the spin lock, and overwriting +* previous packets). Eventually things +* will recover. Similarly if the sma +* consumer is so far behind that we would +* overwrite (yes, it's outside the lock) +*/ +if (!ipath_sma_data_spare || + ipath_sma_data[ipath_sma_next].len || + ++sma_this_time >IPATH_NUM_SMA_PKTS ) + ipath_stats.sps_krdrops++; +else if (ebuf) +ipath_rcv_sma (dd, tlen, rc, ebuf); + }else { +int ret =__ipath_verbs_rcv (dd, rc +1 , + ebuf, tlen); +if (ret == -ENODEV ) +ipath_cdbg (VERBOSE , +"received IB packet, " +"not SMA (QP=%x)\n" , qp); + } + }else if (etype ==RCVHQ_RCV_TYPE_EAGER ) { +if (qp ==IPATH_KD_QP && + bthbytes[0 ] == ipath_layer_rcv_opcode && + ebuf) +ipath_rcv_layer (dd, etail, tlen, + (struct ether_header *)hdr); +else +ipath_cdbg (PKT ,"typ %x, opcode %x (eager, " +"qp=%x), len %x; ignored\n" , + etype, bthbytes[0 ], qp, tlen); + } +else if (etype ==RCVHQ_RCV_TYPE_EXPECTED ) +ipath_dbg ("Bug: Expected TID, opcode %x; ignored\n" , + hdr->bth[0 ] &0xff ); +else if (eflags & (INFINIPATH_RHF_H_TIDERR | +INFINIPATH_RHF_H_IHDRERR )) { +/* +* This is a type 3 packet, only the LRH is in the +* rcvhdrq, the rest of the header is in the eager +* buffer. +*/ +u8 opcode; +if (ebuf) { + bthbytes = (u8 *) ebuf; + opcode = *bthbytes; + } +else + opcode =0 ; +get_rhf_errstring (eflags, emsg,sizeof emsg); +ipath_dbg ("Err %x (%s), opcode %x, egrbuf %x, " +"len %x\n" , eflags, emsg, opcode, etail, + tlen); + }else { +/* +* error packet, type of error unknown. +* Probably type 3, but we don't know, so don't +* even try to print the opcode, etc. +*/ +ipath_dbg ("Error Pkt, but no eflags! egrbuf %x, " +"len %x\nhdrq@%lx;hdrq+%x rhf: %llx; " +"hdr %llx %llx %llx %llx %llx\n" , + etail, tlen, (unsigned long ) rc, l, + (unsigned long long ) rc[0 ], + (unsigned long long ) rc[1 ], + (unsigned long long ) rc[2 ], + (unsigned long long ) rc[3 ], + (unsigned long long ) rc[4 ], + (unsigned long long ) rc[5 ]); + } + l += rsize; +if (l >= maxcnt) + l =0 ; +/* +* update for each packet, to help prevent overflows if we +* have lots of packets. +*/ + (void )ipath_write_ureg (dd, ur_rcvhdrhead, + dd->ipath_rhdrhead_intr_off | l,0 ); +if (etype !=RCVHQ_RCV_TYPE_EXPECTED ) + (void )ipath_write_ureg (dd, ur_rcvegrindexhead, + etail,0 ); + } + + pkttot += i; + + dd->ipath_port0head = l; + +if (hdrqtail != (uint32_t )__le64_to_cpu (*dd->ipath_hdrqtailptr)) +/* more arrived while we handled first batch */ + goto gotmore; + +if (pkttot > ipath_stats.sps_maxpkts_call) + ipath_stats.sps_maxpkts_call = pkttot; + ipath_stats.sps_port0pkts += pkttot; + ipath_stats.sps_avgpkts_call = + ipath_stats.sps_port0pkts / ++totcalls; + +if (sma_this_time)/* only once at end, not each time */ +wake_up_interruptible (&ipath_sma_wait); + +done: +clear_bit (0 , &dd->ipath_rcv_pending); +smp_mb__after_clear_bit (); +} + +/** +* ipath_update_pio_bufs - update shadow copy of the PIO availability map +* @dd: the infinipath device +* +* called whenever our local copy indicates we have run out of send buffers +* NOTE: This can be called from interrupt context by some code +* and from non-interrupt context by ipath_getpiobuf(). +*/ + +static void ipath_update_pio_bufs (struct ipath_devdata *dd) +{ +unsigned long flags; +int i; +const unsigned piobregs = (unsigned )dd->ipath_pioavregs; + +/* If the generation (check) bits have changed, then we update the +* busy bit for the corresponding PIO buffer. This algorithm will +* modify positions to the value they already have in some cases +* (i.e., no change), but it's faster than changing only the bits +* that have changed. +* +* We would like to do this atomicly, to avoid spinlocks in the +* critical send path, but that's not really possible, given the +* type of changes, and that this routine could be called on +* multiple cpu's simultaneously, so we lock in this routine only, +* to avoid conflicting updates; all we change is the shadow, and +* it's a single 64 bit memory location, so by definition the update +* is atomic in terms of what other cpu's can see in testing the +* bits. The spin_lock overhead isn't too bad, since it only +* happens when all buffers are in use, so only cpu overhead, not +* latency or bandwidth is affected. +*/ +#define _IPATH_ALL_CHECKBITS 0x5555555555555555ULL +if (!dd->ipath_pioavailregs_dma) { +ipath_dbg ("Update shadow pioavail, but regs_dma NULL!\n" ); +return ; + } +if (ipath_debug & __IPATH_VERBDBG) { +/* only if packet debug and verbose */ +volatile unsigned long long *dma = + (volatile unsigned long long *) + dd->ipath_pioavailregs_dma; +unsigned long *shadow = dd->ipath_pioavailshadow; + +ipath_cdbg (PKT ,"Refill avail, dma0=%llx shad0=%lx, " +"d1=%llx s1=%lx, d2=%llx s2=%lx, d3=%llx " +"s3=%lx\n" , + (unsigned long long )__le64_to_cpu (dma[0 ]), + shadow[0 ], + (unsigned long long )__le64_to_cpu (dma[1 ]), + shadow[1 ], + (unsigned long long )__le64_to_cpu (dma[2 ]), + shadow[2 ], + (unsigned long long )__le64_to_cpu (dma[3 ]), + shadow[3 ]); +if (piobregs >4 ) +ipath_cdbg ( +PKT ,"2nd group, dma4=%llx shad4=%lx, " +"d5=%llx s5=%lx, d6=%llx s6=%lx, " +"d7=%llx s7=%lx\n" , + (unsigned long long )__le64_to_cpu (dma[4 ]), + shadow[4 ], + (unsigned long long )__le64_to_cpu (dma[5 ]), + shadow[5 ], + (unsigned long long )__le64_to_cpu (dma[6 ]), + shadow[6 ], + (unsigned long long )__le64_to_cpu (dma[7 ]), + shadow[7 ]); + } +spin_lock_irqsave (&ipath_pioavail_lock, flags); +for (i =0 ; i < piobregs; i++) { +u64 pchbusy, pchg, piov, pnew; +/* +* Chip Errata: bug 6641; even and odd qwords>3 are swapped +*/ +if (i >3 ) { +if (i &1 ) + piov =__le64_to_cpu ( + dd->ipath_pioavailregs_dma[i -1 ]); +else + piov =__le64_to_cpu ( + dd->ipath_pioavailregs_dma[i +1 ]); + }else + piov =__le64_to_cpu (dd->ipath_pioavailregs_dma[i]); + pchg = _IPATH_ALL_CHECKBITS & + ~(dd->ipath_pioavailshadow[i] ^ piov); + pchbusy = pchg <<INFINIPATH_SENDPIOAVAIL_BUSY_SHIFT ; +if (pchg && (pchbusy & dd->ipath_pioavailshadow[i])) { + pnew = dd->ipath_pioavailshadow[i] & ~pchbusy; + pnew |= piov & pchbusy; + dd->ipath_pioavailshadow[i] = pnew; + } + } +spin_unlock_irqrestore (&ipath_pioavail_lock, flags); +} + +/** +* ipath_setrcvhdrsize - set the receive header size +* @dd: the infinipath device +* @rhdrsize: the receive header size +* +* called from user init code, and also layered driver init +*/ +int ipath_setrcvhdrsize (struct ipath_devdata *dd,unsigned rhdrsize) +{ +int ret =0 ; + +if (dd->ipath_flags &IPATH_RCVHDRSZ_SET ) { +if (dd->ipath_rcvhdrsize != rhdrsize) { +dev_info (&dd->pcidev->dev, +"Error: can't set protocol header " +"size %u, already %u\n" , + rhdrsize, dd->ipath_rcvhdrsize); + ret = -EAGAIN ; + }else +ipath_cdbg (VERBOSE ,"Reuse same protocol header " +"size %u\n" , dd->ipath_rcvhdrsize); + }else if (rhdrsize > (dd->ipath_rcvhdrentsize - + (sizeof (u64) /sizeof (u32)))) { +ipath_dbg ("Error: can't set protocol header size %u " +"(> max %u)\n" , rhdrsize, + dd->ipath_rcvhdrentsize - + (u32) (sizeof (u64) /sizeof (u32))); + ret = -EOVERFLOW ; + }else { + dd->ipath_flags |=IPATH_RCVHDRSZ_SET ; + dd->ipath_rcvhdrsize = rhdrsize; +ipath_write_kreg (dd, dd->ipath_kregs->kr_rcvhdrsize, + dd->ipath_rcvhdrsize); +ipath_cdbg (VERBOSE ,"Set protocol header size to %u\n" , + dd->ipath_rcvhdrsize); + } +return ret; +} + +/** +* ipath_getpiobuf - find an available pio buffer +* @dd: the infinipath device +* @pbufnum: the buffer number is placed here +* +* do appropriate marking as busy, etc. +* returns buffer number if one found (>=0), negative number is error. +* Used by ipath_sma_send_pkt and ipath_layer_send +*/ +u32 __iomem *ipath_getpiobuf (struct ipath_devdata *dd,u32 * pbufnum) +{ +int i, j, starti, updated =0 ; +unsigned piobcnt, iter; +unsigned long flags; +unsigned long *shadow = dd->ipath_pioavailshadow; +u32 __iomem *buf; + + piobcnt = (unsigned )(dd->ipath_piobcnt2k + + dd->ipath_piobcnt4k); + starti = dd->ipath_lastport_piobuf; + iter = piobcnt - starti; +if (dd->ipath_upd_pio_shadow) { +/* +* Minor optimization. If we had no buffers on last call, +* start out by doing the update; continue and do scan even +* if no buffers were updated, to be paranoid +*/ +ipath_update_pio_bufs (dd); +/* we scanned here, don't do it at end of scan */ + updated =1 ; + i = starti; + }else + i = dd->ipath_lastpioindex; + +rescan: +/* +* while test_and_set_bit() is atomic, we do that and then the +* change_bit(), and the pair is not. See if this is the cause +* of the remaining armlaunch errors. +*/ +spin_lock_irqsave (&ipath_pioavail_lock, flags); +for (j =0 ; j < iter; j++, i++) { +if (i >= piobcnt) + i = starti; +/* +* To avoid bus lock overhead, we first find a candidate +* buffer, then do the test and set, and continue if that +* fails. +*/ +if (test_bit ((2 * i) +1 , shadow) || +test_and_set_bit ((2 * i) +1 , shadow)) +continue ; +/* flip generation bit */ +change_bit (2 * i, shadow); +break ; + } +spin_unlock_irqrestore (&ipath_pioavail_lock, flags); + +if (j == iter) { +volatile unsigned long long *dma = + (volatile unsigned long long *) + dd->ipath_pioavailregs_dma; + +/* +* first time through; shadow exhausted, but may be real +* buffers available, so go see; if any updated, rescan +* (once) +*/ +if (!updated) { +ipath_update_pio_bufs (dd); + updated =1 ; + i = starti; + goto rescan; + } + dd->ipath_upd_pio_shadow =1 ; +/* +* not atomic, but if we lose one once in a while, that's OK +*/ + ipath_stats.sps_nopiobufs++; +if (!(++dd->ipath_consec_nopiobuf %100000 )) { +ipath_dbg ( +"%u pio sends with no bufavail; dmacopy: " +"%llx %llx %llx %llx; shadow: " +"%lx %lx %lx %lx\n" , + dd->ipath_consec_nopiobuf, + (unsigned long long )__le64_to_cpu (dma[0 ]), + (unsigned long long )__le64_to_cpu (dma[1 ]), + (unsigned long long )__le64_to_cpu (dma[2 ]), + (unsigned long long )__le64_to_cpu (dma[3 ]), + shadow[0 ], shadow[1 ], shadow[2 ], + shadow[3 ]); +/* +* 4 buffers per byte, 4 registers above, cover rest +* below +*/ +if ((dd->ipath_piobcnt2k + dd->ipath_piobcnt4k) > + (sizeof (shadow[0 ]) *4 *4 )) +ipath_dbg ("2nd group: dmacopy: %llx %llx " +"%llx %llx; shadow: %lx %lx " +"%lx %lx\n" , + (unsigned long long ) +__le64_to_cpu (dma[4 ]), + (unsigned long long ) +__le64_to_cpu (dma[5 ]), + (unsigned long long ) +__le64_to_cpu (dma[6 ]), + (unsigned long long ) +__le64_to_cpu (dma[7 ]), + shadow[4 ], shadow[5 ], + shadow[6 ], shadow[7 ]); + } +return NULL ; + } + +if (updated) +/* +* ran out of bufs, now some (at least this one we just +* got) are now available, so tell the layered driver. +*/ +__ipath_layer_intr (dd,IPATH_LAYER_INT_SEND_CONTINUE ); + +/* +* set next starting place. Since it's just an optimization, +* it doesn't matter who wins on this, so no locking +*/ + dd->ipath_lastpioindex = i +1 ; +if (dd->ipath_upd_pio_shadow) + dd->ipath_upd_pio_shadow =0 ; +if (dd->ipath_consec_nopiobuf) + dd->ipath_consec_nopiobuf =0 ; +if (i < dd->ipath_piobcnt2k) + buf = (u32 __iomem *) (dd->ipath_pio2kbase + + i * dd->ipath_palign); +else + buf = (u32 __iomem *) + (dd->ipath_pio4kbase + + (i - dd->ipath_piobcnt2k) * dd->ipath_4kalign); +ipath_cdbg (VERBOSE ,"Return piobuf%u %uk @ %p\n" , + i, (i < dd->ipath_piobcnt2k) ?2 :4 , buf); +if (pbufnum) + *pbufnum = i; +return buf; +} + +/** +* ipath_create_rcvhdrq - create a receive header queue +* @dd: the infinipath device +* @pd: the port data +* +* this *must* be physically contiguous memory, and for now, +* that limits it to what kmalloc can do. +*/ +int ipath_create_rcvhdrq (struct ipath_devdata *dd, +struct ipath_portdata *pd) +{ +int ret =0 , amt; + + amt =ALIGN (dd->ipath_rcvhdrcnt * dd->ipath_rcvhdrentsize * +sizeof (u32),PAGE_SIZE ); +if (!pd->port_rcvhdrq) { +/* +* not using REPEAT isn't viable; at 128KB, we can easily +* fail this. The problem with REPEAT is we can block here +* "forever". There isn't an inbetween, unfortunately. We +* could reduce the risk by never freeing the rcvhdrq except +* at unload, but even then, the first time a port is used, +* we could delay for some time... +*/ +gfp_t gfp_flags =GFP_USER | __GFP_COMP; + + pd->port_rcvhdrq =dma_alloc_coherent ( + &dd->pcidev->dev, amt, &pd->port_rcvhdrq_phys, + gfp_flags); + +if (!pd->port_rcvhdrq) { +ipath_dev_err (dd,"attempt to allocate %d bytes " +"for port %u rcvhdrq failed\n" , + amt, pd->port_port); + ret = -ENOMEM ; + goto bail; + } + + pd->port_rcvhdrq_size = amt; + +ipath_cdbg (VERBOSE ,"%d pages at %p (phys %lx) size=%lu " +"for port %u rcvhdr Q\n" , + amt >>PAGE_SHIFT , pd->port_rcvhdrq, + (unsigned long ) pd->port_rcvhdrq_phys, + (unsigned long ) pd->port_rcvhdrq_size, + pd->port_port); + }else { +/* +* clear for security, sanity, and/or debugging, each +* time we reuse +*/ +memset (pd->port_rcvhdrq,0 , amt); + } + +/* +* tell chip each time we init it, even if we are re-using previous +* memory (we zero it at process close) +*/ +ipath_cdbg (VERBOSE ,"writing port %d rcvhdraddr as %lx\n" , + pd->port_port, (unsigned long ) pd->port_rcvhdrq_phys); +ipath_write_kreg_port (dd, dd->ipath_kregs->kr_rcvhdraddr, + pd->port_port, pd->port_rcvhdrq_phys); + + ret =0 ; +bail: +return ret; +} + +int ipath_waitfor_complete (struct ipath_devdata *dd,ipath_kreg reg_id, +u64 bits_to_wait_for,u64 * valp) +{ +u64 timeout, lastval, val; + + lastval =ipath_read_kreg64 (dd, reg_id); +/* wait a ridiculously long time */ + timeout =get_cycles () +0x10000000ULL ; +do { + val =ipath_read_kreg64 (dd, reg_id); +/* set so they have something, even on failures. */ + *valp = val; +if ((val & bits_to_wait_for) == bits_to_wait_for) +return 0 ; +if (val != lastval) +ipath_cdbg (VERBOSE ,"Changed from %llx to %llx, " +"waiting for %llx bits\n" , + (unsigned long long ) lastval, + (unsigned long long ) val, + (unsigned long long ) bits_to_wait_for); +cond_resched (); +if (get_cycles () > timeout) { +ipath_dbg ("Didn't get bits %llx in register 0x%x, " +"got %llx\n" , + (unsigned long long ) bits_to_wait_for, + reg_id, (unsigned long long ) *valp); +return ENODEV ; + } + }while (1 ); +} + +/** +* ipath_waitfor_mdio_cmdready - wait for last command to complete +* @dd: the infinipath device +* +* Like ipath_waitfor_complete(), but we wait for the CMDVALID bit to go +* away indicating the last command has completed. It doesn't return data +*/ +int ipath_waitfor_mdio_cmdready (struct ipath_devdata *dd) +{ +u64 timeout; +u64 val; + +/* wait a ridiculously long time */ + timeout =get_cycles () +0x10000000ULL ; +do { + val =ipath_read_kreg64 (dd, dd->ipath_kregs->kr_mdio); +if (!(val &IPATH_MDIO_CMDVALID )) +return 0 ; +cond_resched (); +if (get_cycles () > timeout) { +ipath_dbg ("CMDVALID stuck in mdio reg? (%llx)\n" , + (unsigned long long ) val); +return ENODEV ; + } + }while (1 ); +} + +void ipath_set_ib_lstate (struct ipath_devdata *dd,int which) +{ +static const char *what[4 ] = { + [0 ] ="DOWN" , + [INFINIPATH_IBCC_LINKCMD_INIT ] ="INIT" , + [INFINIPATH_IBCC_LINKCMD_ARMED ] ="ARMED" , + [INFINIPATH_IBCC_LINKCMD_ACTIVE ] ="ACTIVE" + }; +ipath_cdbg (SMA ,"Trying to move unit %u to %s, current ltstate " +"is %s\n" , dd->ipath_unit, + what[(which >>INFINIPATH_IBCC_LINKCMD_SHIFT ) & +INFINIPATH_IBCC_LINKCMD_MASK ], + ipath_ibcstatus_str[ + (ipath_read_kreg64 + (dd, dd->ipath_kregs->kr_ibcstatus) >> +INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT ) & +INFINIPATH_IBCS_LINKTRAININGSTATE_MASK ]); + +ipath_write_kreg (dd, dd->ipath_kregs->kr_ibcctrl, + dd->ipath_ibcctrl | which); +} + +/** +* ipath_read_kreg64_port - read a device's per-port 64-bit kernel register +* @dd: the infinipath device +* @regno: the register number to read +* @port: the port containing the register +* +* Registers that vary with the chip implementation constants (port) +* use this routine. +*/ +u64 ipath_read_kreg64_port (const struct ipath_devdata *dd,ipath_kreg regno, +unsigned port) +{ +u16 where; + +if (port < dd->ipath_portcnt && + (regno == dd->ipath_kregs->kr_rcvhdraddr || + regno == dd->ipath_kregs->kr_rcvhdrtailaddr)) + where = regno + port; +else + where =-1 ; + +return ipath_read_kreg64 (dd, where); +} + +/** +* ipath_write_kreg_port - write a device's per-port 64-bit kernel register +* @dd: the infinipath device +* @regno: the register number to write +* @port: the port containing the register +* @value: the value to write +* +* Registers that vary with the chip implementation constants (port) +* use this routine. +*/ +void ipath_write_kreg_port (const struct ipath_devdata *dd,ipath_kreg regno, +unsigned port,u64 value) +{ +u16 where; + +if (port < dd->ipath_portcnt && + (regno == dd->ipath_kregs->kr_rcvhdraddr || + regno == dd->ipath_kregs->kr_rcvhdrtailaddr)) + where = regno + port; +else + where =-1 ; + +ipath_write_kreg (dd, where, value); +} + +/** +* ipath_shutdown_device - shut down a device +* @dd: the infinipath device +* +* This is called to make the device quiet when we are about to +* unload the driver, and also when the device is administratively +* disabled. It does not free any data structures. +* Everything it does has to be setup again by ipath_init_chip(dd,1) +*/ +void ipath_shutdown_device (struct ipath_devdata *dd) +{ +u64 val; + +ipath_dbg ("Shutting down the device\n" ); + + dd->ipath_flags |=IPATH_LINKUNK ; + dd->ipath_flags &= ~(IPATH_INITTED |IPATH_LINKDOWN | +IPATH_LINKINIT |IPATH_LINKARMED | +IPATH_LINKACTIVE ); + *dd->ipath_statusp &= ~(IPATH_STATUS_IB_CONF | +IPATH_STATUS_IB_READY ); + +/* mask interrupts, but not errors */ +ipath_write_kreg (dd, dd->ipath_kregs->kr_intmask,0ULL ); + + dd->ipath_rcvctrl =0 ; +ipath_write_kreg (dd, dd->ipath_kregs->kr_rcvctrl, + dd->ipath_rcvctrl); + +/* +* gracefully stop all sends allowing any in progress to trickle out +* first. +*/ +ipath_write_kreg (dd, dd->ipath_kregs->kr_sendctrl,0ULL ); +/* flush it */ + val =ipath_read_kreg64 (dd, dd->ipath_kregs->kr_scratch); +/* +* enough for anything that's going to trickle out to have actually +* done so. +*/ +udelay (5 ); + +/* +* abort any armed or launched PIO buffers that didn't go. (self +* clearing). Will cause any packet currently being transmitted to +* go out with an EBP, and may also cause a short packet error on +* the receiver. +*/ +ipath_write_kreg (dd, dd->ipath_kregs->kr_sendctrl, +INFINIPATH_S_ABORT ); + +ipath_set_ib_lstate (dd,INFINIPATH_IBCC_LINKINITCMD_DISABLE << +INFINIPATH_IBCC_LINKINITCMD_SHIFT ); + +/* +* we are shutting down, so tell the layered driver. We don't do +* this on just a link state change, much like ethernet, a cable +* unplug, etc. doesn't change driver state +*/ +ipath_layer_intr (dd,IPATH_LAYER_INT_IF_DOWN ); + +/* disable IBC */ + dd->ipath_control &= ~INFINIPATH_C_LINKENABLE ; +ipath_write_kreg (dd, dd->ipath_kregs->kr_control, + dd->ipath_control); + +/* +* clear SerdesEnable and turn the leds off; do this here because +* we are unloading, so don't count on interrupts to move along +* Turn the LEDs off explictly for the same reason. +*/ + dd->ipath_f_quiet_serdes (dd); + dd->ipath_f_setextled (dd,0 ,0 ); + +if (dd->ipath_stats_timer_active) { +del_timer_sync (&dd->ipath_stats_timer); + dd->ipath_stats_timer_active =0 ; + } + +/* +* clear all interrupts and errors, so that the next time the driver +* is loaded or device is enabled, we know that whatever is set +* happened while we were unloaded +*/ +ipath_write_kreg (dd, dd->ipath_kregs->kr_hwerrclear, + ~0ULL &~INFINIPATH_HWE_MEMBISTFAILED ); +ipath_write_kreg (dd, dd->ipath_kregs->kr_errorclear,-1LL ); +ipath_write_kreg (dd, dd->ipath_kregs->kr_intclear,-1LL ); +} + +/** +* ipath_free_pddata - free a port's allocated data +* @dd: the infinipath device +* @port: the port +* @freehdrq: free the port data structure if true +* +* when closing, free up any allocated data for a port, if the +* reference count goes to zero +* Note: this also optionally frees the portdata itself! +* Any changes here have to be matched up with the reinit case +* of ipath_init_chip(), which calls this routine on reinit after reset. +*/ +void ipath_free_pddata (struct ipath_devdata *dd,u32 port,int freehdrq) +{ +struct ipath_portdata *pd = dd->ipath_pd[port]; + +if (!pd) +return ; +if (freehdrq) +/* +* only clear and free portdata if we are going to also +* release the hdrq, otherwise we leak the hdrq on each +* open/close cycle +*/ + dd->ipath_pd[port] =NULL ; +if (freehdrq && pd->port_rcvhdrq) { +ipath_cdbg (VERBOSE ,"free closed port %d rcvhdrq @ %p " +"(size=%lu)\n" , pd->port_port, pd->port_rcvhdrq, + (unsigned long ) pd->port_rcvhdrq_size); +dma_free_coherent (&dd->pcidev->dev, pd->port_rcvhdrq_size, + pd->port_rcvhdrq, pd->port_rcvhdrq_phys); + pd->port_rcvhdrq =NULL ; + } +if (port && pd->port_rcvegrbuf) { +/* always free this */ +if (pd->port_rcvegrbuf) { +unsigned e; + +for (e =0 ; e < pd->port_rcvegrbuf_chunks; e++) { +void *base = pd->port_rcvegrbuf[e]; +size_t size = pd->port_rcvegrbuf_size; + +ipath_cdbg (VERBOSE ,"egrbuf free(%p, %lu), " +"chunk %u/%u\n" , base, + (unsigned long ) size, + e, pd->port_rcvegrbuf_chunks); +dma_free_coherent ( + &dd->pcidev->dev, size, base, + pd->port_rcvegrbuf_phys[e]); + } +vfree (pd->port_rcvegrbuf); + pd->port_rcvegrbuf =NULL ; +vfree (pd->port_rcvegrbuf_phys); + pd->port_rcvegrbuf_phys =NULL ; + } + pd->port_rcvegrbuf_chunks =0 ; + }else if (port ==0 && dd->ipath_port0_skbs) { +unsigned e; +struct sk_buff **skbs = dd->ipath_port0_skbs; + + dd->ipath_port0_skbs =NULL ; +ipath_cdbg (VERBOSE ,"free closed port %d ipath_port0_skbs " +"@ %p\n" , pd->port_port, skbs); +for (e =0 ; e < dd->ipath_rcvegrcnt; e++) +if (skbs[e]) +dev_kfree_skb (skbs[e]); +vfree (skbs); + } +if (freehdrq) { +kfree (pd->port_tid_pg_list); +kfree (pd); + } +} + +int __init infinipath_init (void ) +{ +int ret; + + ipath_dbg(KERN_INFO DRIVER_LOAD_MSG "%s", ipath_core_version); + +/* +* These must all be called before the driver is registered +* with the PCI subsystem. + */ + spin_lock_init(&ipath_pioavail_lock); +spin_lock_init (&ipath_sma_lock); + +spin_lock_init (&ipath_devs_lock); +idr_init (&unit_table); +if (!idr_pre_get (&unit_table,GFP_KERNEL )) { + ret = -ENOMEM ; + goto bail; + } + + ret =pci_register_driver (&ipath_driver); +if (ret <0 ) { + printk(KERN_ERR IPATH_DRV_NAME + ": Unable to register driver: error %d\n", -ret); + goto bail_unit; + } + + ret =ipath_driver_create_group (&ipath_driver.driver); +if (ret <0 ) { + printk(KERN_ERR IPATH_DRV_NAME ": Unable to create driver " + "sysfs entries: error %d\n", -ret); + goto bail_pci; + } + + ret =ipath_init_ipathfs (); +if (ret <0 ) { + printk(KERN_ERR IPATH_DRV_NAME ": Unable to create " + "ipathfs: error %d\n", -ret); + goto bail_group; + } + + goto bail; + +bail_group: +ipath_driver_remove_group (&ipath_driver.driver); + +bail_pci: +pci_unregister_driver (&ipath_driver); + +bail_unit: +idr_destroy (&unit_table); + +bail: +return ret; +} + +static void cleanup_device (struct ipath_devdata *dd) +{ +int port; + +ipath_shutdown_device (dd); + +if (*dd->ipath_statusp &IPATH_STATUS_CHIP_PRESENT ) { +/* can't do anything more with chip; needs re-init */ + *dd->ipath_statusp &= ~IPATH_STATUS_CHIP_PRESENT ; +if (dd->ipath_kregbase) { +/* +* if we haven't already cleaned up before these are +* to ensure any register reads/writes "fail" until +* re-init +*/ + dd->ipath_kregbase =NULL ; + dd->ipath_kregvirt =NULL ; + dd->ipath_uregbase =0 ; + dd->ipath_sregbase =0 ; + dd->ipath_cregbase =0 ; + dd->ipath_kregsize =0 ; + } +ipath_disable_wc (dd); + } + +if (dd->ipath_pioavailregs_dma) { +dma_free_coherent (&dd->pcidev->dev,PAGE_SIZE , + (void *) dd->ipath_pioavailregs_dma, + dd->ipath_pioavailregs_phys); + dd->ipath_pioavailregs_dma =NULL ; + } + +if (dd->ipath_pageshadow) { +struct page **tmpp = dd->ipath_pageshadow; +int i, cnt =0 ; + +ipath_cdbg (VERBOSE ,"Unlocking any expTID pages still " +"locked\n" ); +for (port =0 ; port < dd->ipath_cfgports; port++) { +int port_tidbase = port * dd->ipath_rcvtidcnt; +int maxtid = port_tidbase + dd->ipath_rcvtidcnt; +for (i = port_tidbase; i < maxtid; i++) { +if (!tmpp[i]) +continue ; +ipath_release_user_pages (&tmpp[i],1 ); + tmpp[i] =NULL ; + cnt++; + } + } +if (cnt) { + ipath_stats.sps_pageunlocks += cnt; +ipath_cdbg (VERBOSE ,"There were still %u expTID " +"entries locked\n" , cnt); + } +if (ipath_stats.sps_pagelocks || + ipath_stats.sps_pageunlocks) +ipath_cdbg (VERBOSE ,"%llu pages locked, %llu " +"unlocked via ipath_m{un}lock\n" , + (unsigned long long ) + ipath_stats.sps_pagelocks, + (unsigned long long ) + ipath_stats.sps_pageunlocks); + +ipath_cdbg (VERBOSE ,"Free shadow page tid array at %p\n" , + dd->ipath_pageshadow); +vfree (dd->ipath_pageshadow); + dd->ipath_pageshadow =NULL ; + } + +/* +* free any resources still in use (usually just kernel ports) +* at unload +*/ +for (port =0 ; port < dd->ipath_cfgports; port++) +ipath_free_pddata (dd, port,1 ); +kfree (dd->ipath_pd); +/* +* debuggability, in case some cleanup path tries to use it +* after this +*/ + dd->ipath_pd =NULL ; +} + +static void __exit infinipath_cleanup(void) +{ +struct ipath_devdata *dd, *tmp; +unsigned long flags; + +ipath_exit_ipathfs (); + +ipath_driver_remove_group (&ipath_driver.driver); + +spin_lock_irqsave (&ipath_devs_lock, flags); + +/* +* turn off rcv, send, and interrupts for all ports, all drivers +* should also hard reset the chip here? +* free up port 0 (kernel) rcvhdr, egr bufs, and eventually tid bufs +* for all versions of the driver, if they were allocated +*/ +list_for_each_entry_safe (dd, tmp, &ipath_dev_list, ipath_list) { +spin_unlock_irqrestore (&ipath_devs_lock, flags); + +if (dd->ipath_kregbase) +cleanup_device (dd); + +if (dd->pcidev) { +if (dd->pcidev->irq) { +ipath_cdbg (VERBOSE , +"unit %u free_irq of irq %x\n" , + dd->ipath_unit, dd->pcidev->irq); +free_irq (dd->pcidev->irq, dd); + }else +ipath_dbg ("irq is 0, not doing free_irq " +"for unit %u\n" , dd->ipath_unit); + dd->pcidev =NULL ; + } + +/* +* we check for NULL here, because it's outside the kregbase +* check, and we need to call it after the free_irq. Thus +* it's possible that the function pointers were never +* initialized. +*/ +if (dd->ipath_f_cleanup) +/* clean up chip-specific stuff */ + dd->ipath_f_cleanup (dd); + +spin_lock_irqsave (&ipath_devs_lock, flags); + } + +spin_unlock_irqrestore (&ipath_devs_lock, flags); + +ipath_cdbg (VERBOSE ,"Unregistering pci driver\n" ); +pci_unregister_driver (&ipath_driver); + +idr_destroy (&unit_table); +} + +/** +* ipath_reset_device - reset the chip if possible +* @unit: the device to reset +* +* Whether or not reset is successful, we attempt to re-initialize the chip +* (that is, much like a driver unload/reload). We clear the INITTED flag +* so that the various entry points will fail until we reinitialize. For +* now, we only allow this if no user ports are open that use chip resources +*/ +int ipath_reset_device (int unit) +{ +int ret, i; +struct ipath_devdata *dd =ipath_lookup (unit); + +if (!dd) +return -ENODEV ; + +dev_info (&dd->pcidev->dev,"Reset on unit %u requested\n" , unit); + +if (!dd->ipath_kregbase || !(dd->ipath_flags &IPATH_PRESENT )) { +dev_info (&dd->pcidev->dev,"Invalid unit number %u or " +"not initialized or not present\n" , unit); +return -ENXIO ; + } + +if (dd->ipath_pd) +for (i =1 ; i < dd->ipath_portcnt; i++) { +if (dd->ipath_pd[i] && dd->ipath_pd[i]->port_cnt) { +ipath_dbg ("unit %u port %d is in use " +"(PID %u cmd %s), can't reset\n" , + unit, i, + dd->ipath_pd[i]->port_pid, + dd->ipath_pd[i]->port_comm); +return -EBUSY ; + } + } + + dd->ipath_flags &= ~IPATH_INITTED ; + ret = dd->ipath_f_reset (dd); +if (ret !=1 ) +ipath_dbg ("reset was not successful\n" ); +ipath_dbg ("Trying to reinitialize unit %u after reset attempt\n" , + unit); + ret =ipath_init_chip (dd,1 ); +if (ret) +ipath_dev_err (dd,"Reinitialize unit %u after " +"reset failed with %d\n" , unit, ret); +else +dev_info (&dd->pcidev->dev,"Reinitialized unit %u after " +"resetting\n" , unit); +return ret; +} + +module_init (infinipath_init); +module_exit (infinipath_cleanup);
PATCH 1/1 ipath - infiniband UC and UD protocol support
In reply to Bryan O'SullivanSigned-off-by Bryan O'Sullivan <[email protected]>
drivers/infiniband/hw/ipath/ipath_uc.c new file +639 -0
@@ -0,0 +1,639 @@ +/* +* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. +* +* This software is available to you under a choice of one of two +* licenses. You may choose to be licensed under the terms of the GNU +* General Public License (GPL) Version 2, available from the file +* COPYING in the main directory of this source tree, or the +* OpenIB.org BSD license below: +* +* Redistribution and use in source and binary forms, with or +* without modification, are permitted provided that the following +* conditions are met: +* +* - Redistributions of source code must retain the above +* copyright notice, this list of conditions and the following +* disclaimer. +* +* - Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials +* provided with the distribution. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ + +#include "ipath_verbs.h" +#include "ips_common.h" + +/* cut down ridiculously long IB macro names */ +#define OP (x) IB_OPCODE_UC_##x + +static void complete_last_send (struct ipath_qp *qp,struct ipath_swqe *wqe, +struct ib_wc *wc) +{ +if (++qp->s_last == qp->s_size) + qp->s_last =0 ; +if (!test_bit (IPATH_S_SIGNAL_REQ_WR , &qp->s_flags) || + (wqe->wr.send_flags &IB_SEND_SIGNALED )) { + wc->wr_id = wqe->wr.wr_id; + wc->status =IB_WC_SUCCESS ; + wc->opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; + wc->vendor_err =0 ; + wc->byte_len = wqe->length; + wc->qp_num = qp->ibqp.qp_num; + wc->src_qp = qp->remote_qpn; + wc->pkey_index =0 ; + wc->slid = qp->remote_ah_attr.dlid; + wc->sl = qp->remote_ah_attr.sl; + wc->dlid_path_bits =0 ; + wc->port_num =0 ; +ipath_cq_enter (to_icq (qp->ibqp.send_cq), wc,0 ); + } + wqe =get_swqe_ptr (qp, qp->s_last); +} + +/** +* ipath_do_uc_send - do a send on a UC queue +* @data: contains a pointer to the QP to send on +* +* Process entries in the send work queue until the queue is exhausted. +* Only allow one CPU to send a packet per QP (tasklet). +* Otherwise, after we drop the QP lock, two threads could send +* packets out of order. +* This is similar to ipath_do_rc_send() below except we don't have +* timeouts or resends. +*/ +void ipath_do_uc_send (unsigned long data) +{ +struct ipath_qp *qp = (struct ipath_qp *)data; +struct ipath_ibdev *dev =to_idev (qp->ibqp.device); +struct ipath_swqe *wqe; +unsigned long flags; +u16 lrh0; +u32 hwords; +u32 nwords; +u32 extra_bytes; +u32 bth0; +u32 bth2; +u32 pmtu =ib_mtu_enum_to_int (qp->path_mtu); +u32 len; +struct ipath_other_headers *ohdr; +struct ib_wc wc; + +if (test_and_set_bit (IPATH_S_BUSY , &qp->s_flags)) +return ; + +if (unlikely (qp->remote_ah_attr.dlid == +ipath_layer_get_lid (dev->dd))) { +/* Pass in an uninitialized ib_wc to save stack space. */ +ipath_ruc_loopback (qp, &wc); +clear_bit (IPATH_S_BUSY , &qp->s_flags); +return ; + } + + ohdr = &qp->s_hdr.u.oth; +if (qp->remote_ah_attr.ah_flags &IB_AH_GRH ) + ohdr = &qp->s_hdr.u.l.oth; + +again: +/* Check for a constructed packet to be sent. */ +if (qp->s_hdrwords !=0 ) { +/* +* If no PIO bufs are available, return. +* An interrupt will call ipath_ib_piobufavail() +* when one is available. +*/ +if (ipath_verbs_send (dev->dd, qp->s_hdrwords, + (u32 *) &qp->s_hdr, + qp->s_cur_size, + qp->s_cur_sge)) { +ipath_no_bufs_available (qp, dev); +return ; + } + dev->n_unicast_xmit++; +/* Record that we sent the packet and s_hdr is empty. */ + qp->s_hdrwords =0 ; + } + + lrh0 =IPS_LRH_BTH ; +/* header size in 32-bit words LRH+BTH = (8+12)/4. */ + hwords =5 ; + +/* +* The lock is needed to synchronize between +* setting qp->s_ack_state and post_send(). +*/ +spin_lock_irqsave (&qp->s_lock, flags); + +if (!(ib_ipath_state_ops[qp->state] &IPATH_PROCESS_SEND_OK )) + goto done; + + bth0 =ipath_layer_get_pkey (dev->dd, qp->s_pkey_index); + +/* Send a request. */ + wqe =get_swqe_ptr (qp, qp->s_last); +switch (qp->s_state) { +default : +/* +* Signal the completion of the last send (if there is +* one). +*/ +if (qp->s_last != qp->s_tail) +complete_last_send (qp, wqe, &wc); + +/* Check if send work queue is empty. */ +if (qp->s_tail == qp->s_head) + goto done; +/* +* Start a new request. +*/ + qp->s_psn = wqe->psn = qp->s_next_psn; + qp->s_sge.sge = wqe->sg_list[0 ]; + qp->s_sge.sg_list = wqe->sg_list +1 ; + qp->s_sge.num_sge = wqe->wr.num_sge; + qp->s_len = len = wqe->length; +switch (wqe->wr.opcode) { +case IB_WR_SEND : +case IB_WR_SEND_WITH_IMM : +if (len > pmtu) { + qp->s_state =OP (SEND_FIRST ); + len = pmtu; +break ; + } +if (wqe->wr.opcode ==IB_WR_SEND ) + qp->s_state =OP (SEND_ONLY ); +else { + qp->s_state = +OP (SEND_ONLY_WITH_IMMEDIATE ); +/* Immediate data comes after the BTH */ + ohdr->u.imm_data = wqe->wr.imm_data; + hwords +=1 ; + } +if (wqe->wr.send_flags &IB_SEND_SOLICITED ) + bth0 |=1 <<23 ; +break ; + +case IB_WR_RDMA_WRITE : +case IB_WR_RDMA_WRITE_WITH_IMM : + ohdr->u.rc.reth.vaddr = +cpu_to_be64 (wqe->wr.wr.rdma.remote_addr); + ohdr->u.rc.reth.rkey = +cpu_to_be32 (wqe->wr.wr.rdma.rkey); + ohdr->u.rc.reth.length =cpu_to_be32 (len); + hwords +=sizeof (struct ib_reth ) /4 ; +if (len > pmtu) { + qp->s_state =OP (RDMA_WRITE_FIRST ); + len = pmtu; +break ; + } +if (wqe->wr.opcode ==IB_WR_RDMA_WRITE ) + qp->s_state =OP (RDMA_WRITE_ONLY ); +else { + qp->s_state = +OP (RDMA_WRITE_ONLY_WITH_IMMEDIATE ); +/* Immediate data comes after the RETH */ + ohdr->u.rc.imm_data = wqe->wr.imm_data; + hwords +=1 ; +if (wqe->wr.send_flags &IB_SEND_SOLICITED ) + bth0 |=1 <<23 ; + } +break ; + +default : + goto done; + } +if (++qp->s_tail >= qp->s_size) + qp->s_tail =0 ; +break ; + +case OP (SEND_FIRST ): + qp->s_state =OP (SEND_MIDDLE ); +/* FALLTHROUGH */ +case OP (SEND_MIDDLE ): + len = qp->s_len; +if (len > pmtu) { + len = pmtu; +break ; + } +if (wqe->wr.opcode ==IB_WR_SEND ) + qp->s_state =OP (SEND_LAST ); +else { + qp->s_state =OP (SEND_LAST_WITH_IMMEDIATE ); +/* Immediate data comes after the BTH */ + ohdr->u.imm_data = wqe->wr.imm_data; + hwords +=1 ; + } +if (wqe->wr.send_flags &IB_SEND_SOLICITED ) + bth0 |=1 <<23 ; +break ; + +case OP (RDMA_WRITE_FIRST ): + qp->s_state =OP (RDMA_WRITE_MIDDLE ); +/* FALLTHROUGH */ +case OP (RDMA_WRITE_MIDDLE ): + len = qp->s_len; +if (len > pmtu) { + len = pmtu; +break ; + } +if (wqe->wr.opcode ==IB_WR_RDMA_WRITE ) + qp->s_state =OP (RDMA_WRITE_LAST ); +else { + qp->s_state = +OP (RDMA_WRITE_LAST_WITH_IMMEDIATE ); +/* Immediate data comes after the BTH */ + ohdr->u.imm_data = wqe->wr.imm_data; + hwords +=1 ; +if (wqe->wr.send_flags &IB_SEND_SOLICITED ) + bth0 |=1 <<23 ; + } +break ; + } + bth2 = qp->s_next_psn++ &IPS_PSN_MASK ; + qp->s_len -= len; + bth0 |= qp->s_state <<24 ; + +spin_unlock_irqrestore (&qp->s_lock, flags); + +/* Construct the header. */ + extra_bytes = (4 - len) &3 ; + nwords = (len + extra_bytes) >>2 ; +if (unlikely (qp->remote_ah_attr.ah_flags &IB_AH_GRH )) { +/* Header size in 32-bit words. */ + hwords +=10 ; + lrh0 =IPS_LRH_GRH ; + qp->s_hdr.u.l.grh.version_tclass_flow = +cpu_to_be32 ((6 <<28 ) | + (qp->remote_ah_attr.grh.traffic_class + <<20 ) | + qp->remote_ah_attr.grh.flow_label); + qp->s_hdr.u.l.grh.paylen = +cpu_to_be16 (((hwords -12 ) + nwords + +SIZE_OF_CRC ) <<2 ); +/* next_hdr is defined by C8-7 in ch. 8.4.1 */ + qp->s_hdr.u.l.grh.next_hdr =0x1B ; + qp->s_hdr.u.l.grh.hop_limit = + qp->remote_ah_attr.grh.hop_limit; +/* The SGID is 32-bit aligned. */ + qp->s_hdr.u.l.grh.sgid.global.subnet_prefix = + dev->gid_prefix; + qp->s_hdr.u.l.grh.sgid.global.interface_id = +ipath_layer_get_guid (dev->dd); + qp->s_hdr.u.l.grh.dgid = qp->remote_ah_attr.grh.dgid; + } + qp->s_hdrwords = hwords; + qp->s_cur_sge = &qp->s_sge; + qp->s_cur_size = len; + lrh0 |= qp->remote_ah_attr.sl <<4 ; + qp->s_hdr.lrh[0 ] =cpu_to_be16 (lrh0); +/* DEST LID */ + qp->s_hdr.lrh[1 ] =cpu_to_be16 (qp->remote_ah_attr.dlid); + qp->s_hdr.lrh[2 ] =cpu_to_be16 (hwords + nwords +SIZE_OF_CRC ); + qp->s_hdr.lrh[3 ] =cpu_to_be16 (ipath_layer_get_lid (dev->dd)); + bth0 |= extra_bytes <<20 ; + ohdr->bth[0 ] =cpu_to_be32 (bth0); + ohdr->bth[1 ] =cpu_to_be32 (qp->remote_qpn); + ohdr->bth[2 ] =cpu_to_be32 (bth2); + +/* Check for more work to do. */ + goto again; + +done: +spin_unlock_irqrestore (&qp->s_lock, flags); +clear_bit (IPATH_S_BUSY , &qp->s_flags); +} + +/** +* ipath_uc_rcv - handle an incoming UC packet +* @dev: the device the packet came in on +* @hdr: the header of the packet +* @has_grh: true if the packet has a GRH +* @data: the packet data +* @tlen: the length of the packet +* @qp: the QP for this packet. +* +* This is called from ipath_qp_rcv() to process an incoming UC packet +* for the given QP. +* Called at interrupt level. +*/ +void ipath_uc_rcv (struct ipath_ibdev *dev,struct ipath_ib_header *hdr, +int has_grh,void *data,u32 tlen,struct ipath_qp *qp) +{ +struct ipath_other_headers *ohdr; +int opcode; +u32 hdrsize; +u32 psn; +u32 pad; +unsigned long flags; +struct ib_wc wc; +u32 pmtu =ib_mtu_enum_to_int (qp->path_mtu); +struct ib_reth *reth; +int header_in_data; + +/* Check for GRH */ +if (!has_grh) { + ohdr = &hdr->u.oth; + hdrsize =8 +12 ;/* LRH + BTH */ + psn =be32_to_cpu (ohdr->bth[2 ]); + header_in_data =0 ; + }else { + ohdr = &hdr->u.l.oth; + hdrsize =8 +40 +12 ;/* LRH + GRH + BTH */ +/* +* The header with GRH is 60 bytes and the +* core driver sets the eager header buffer +* size to 56 bytes so the last 4 bytes of +* the BTH header (PSN) is in the data buffer. +*/ + header_in_data = +ipath_layer_get_rcvhdrentsize (dev->dd) ==16 ; +if (header_in_data) { + psn =be32_to_cpu (((u32 *) data)[0 ]); + data +=sizeof (u32); + }else + psn =be32_to_cpu (ohdr->bth[2 ]); + } +/* +* The opcode is in the low byte when its in network order +* (top byte when in host order). +*/ + opcode =be32_to_cpu (ohdr->bth[0 ]) >>24 ; + + wc.imm_data =0 ; + wc.wc_flags =0 ; + +spin_lock_irqsave (&qp->r_rq.lock, flags); + +/* Compare the PSN verses the expected PSN. */ +if (unlikely (ipath_cmp24 (psn, qp->r_psn) !=0 )) { +/* +* Handle a sequence error. +* Silently drop any current message. +*/ + qp->r_psn = psn; + inv: + qp->r_state =OP (SEND_LAST ); +switch (opcode) { +case OP (SEND_FIRST ): +case OP (SEND_ONLY ): +case OP (SEND_ONLY_WITH_IMMEDIATE ): + goto send_first; + +case OP (RDMA_WRITE_FIRST ): +case OP (RDMA_WRITE_ONLY ): +case OP (RDMA_WRITE_ONLY_WITH_IMMEDIATE ): + goto rdma_first; + +default : + dev->n_pkt_drops++; + goto done; + } + } + +/* Check for opcode sequence errors. */ +switch (qp->r_state) { +case OP (SEND_FIRST ): +case OP (SEND_MIDDLE ): +if (opcode ==OP (SEND_MIDDLE ) || + opcode ==OP (SEND_LAST ) || + opcode ==OP (SEND_LAST_WITH_IMMEDIATE )) +break ; + goto inv; + +case OP (RDMA_WRITE_FIRST ): +case OP (RDMA_WRITE_MIDDLE ): +if (opcode ==OP (RDMA_WRITE_MIDDLE ) || + opcode ==OP (RDMA_WRITE_LAST ) || + opcode ==OP (RDMA_WRITE_LAST_WITH_IMMEDIATE )) +break ; + goto inv; + +default : +if (opcode ==OP (SEND_FIRST ) || + opcode ==OP (SEND_ONLY ) || + opcode ==OP (SEND_ONLY_WITH_IMMEDIATE ) || + opcode ==OP (RDMA_WRITE_FIRST ) || + opcode ==OP (RDMA_WRITE_ONLY ) || + opcode ==OP (RDMA_WRITE_ONLY_WITH_IMMEDIATE )) +break ; + goto inv; + } + +/* OK, process the packet. */ +switch (opcode) { +case OP (SEND_FIRST ): +case OP (SEND_ONLY ): +case OP (SEND_ONLY_WITH_IMMEDIATE ): + send_first: +if (qp->r_reuse_sge) { + qp->r_reuse_sge =0 ; + qp->r_sge = qp->s_rdma_sge; + }else if (!ipath_get_rwqe (qp,0 )) { + dev->n_pkt_drops++; + goto done; + } +/* Save the WQE so we can reuse it in case of an error. */ + qp->s_rdma_sge = qp->r_sge; + qp->r_rcv_len =0 ; +if (opcode ==OP (SEND_ONLY )) + goto send_last; +else if (opcode ==OP (SEND_ONLY_WITH_IMMEDIATE )) + goto send_last_imm; +/* FALLTHROUGH */ +case OP (SEND_MIDDLE ): +/* Check for invalid length PMTU or posted rwqe len. */ +if (unlikely (tlen != (hdrsize + pmtu +4 ))) { + qp->r_reuse_sge =1 ; + dev->n_pkt_drops++; + goto done; + } + qp->r_rcv_len += pmtu; +if (unlikely (qp->r_rcv_len > qp->r_len)) { + qp->r_reuse_sge =1 ; + dev->n_pkt_drops++; + goto done; + } +ipath_copy_sge (&qp->r_sge, data, pmtu); +break ; + +case OP (SEND_LAST_WITH_IMMEDIATE ): + send_last_imm: +if (header_in_data) { + wc.imm_data = *(u32 *) data; + data +=sizeof (u32); + }else { +/* Immediate data comes after BTH */ + wc.imm_data = ohdr->u.imm_data; + } + hdrsize +=4 ; + wc.wc_flags =IB_WC_WITH_IMM ; +/* FALLTHROUGH */ +case OP (SEND_LAST ): + send_last: +/* Get the number of bytes the message was padded by. */ + pad = (be32_to_cpu (ohdr->bth[0 ]) >>20 ) &3 ; +/* Check for invalid length. */ +/* XXX LAST len should be >= 1 */ +if (unlikely (tlen < (hdrsize + pad +4 ))) { + qp->r_reuse_sge =1 ; + dev->n_pkt_drops++; + goto done; + } +/* Don't count the CRC. */ + tlen -= (hdrsize + pad +4 ); + wc.byte_len = tlen + qp->r_rcv_len; +if (unlikely (wc.byte_len > qp->r_len)) { + qp->r_reuse_sge =1 ; + dev->n_pkt_drops++; + goto done; + } +/* XXX Need to free SGEs */ + last_imm: +ipath_copy_sge (&qp->r_sge, data, tlen); + wc.wr_id = qp->r_wr_id; + wc.status =IB_WC_SUCCESS ; + wc.opcode =IB_WC_RECV ; + wc.vendor_err =0 ; + wc.qp_num = qp->ibqp.qp_num; + wc.src_qp = qp->remote_qpn; + wc.pkey_index =0 ; + wc.slid = qp->remote_ah_attr.dlid; + wc.sl = qp->remote_ah_attr.sl; + wc.dlid_path_bits =0 ; + wc.port_num =0 ; +/* Signal completion event if the solicited bit is set. */ +ipath_cq_enter (to_icq (qp->ibqp.recv_cq), &wc, + ohdr->bth[0 ] & +__constant_cpu_to_be32 (1 <<23 )); +break ; + +case OP (RDMA_WRITE_FIRST ): +case OP (RDMA_WRITE_ONLY ): +case OP (RDMA_WRITE_ONLY_WITH_IMMEDIATE ):/* consume RWQE */ + rdma_first: +/* RETH comes after BTH */ +if (!header_in_data) + reth = &ohdr->u.rc.reth; +else { + reth = (struct ib_reth *)data; + data +=sizeof (*reth); + } + hdrsize +=sizeof (*reth); + qp->r_len =be32_to_cpu (reth->length); + qp->r_rcv_len =0 ; +if (qp->r_len !=0 ) { +u32 rkey =be32_to_cpu (reth->rkey); +u64 vaddr =be64_to_cpu (reth->vaddr); + +/* Check rkey */ +if (unlikely (!ipath_rkey_ok ( + dev, &qp->r_sge, qp->r_len, + vaddr, rkey, +IB_ACCESS_REMOTE_WRITE ))) { + dev->n_pkt_drops++; + goto done; + } + }else { + qp->r_sge.sg_list =NULL ; + qp->r_sge.sge.mr =NULL ; + qp->r_sge.sge.vaddr =NULL ; + qp->r_sge.sge.length =0 ; + qp->r_sge.sge.sge_length =0 ; + } +if (unlikely (!(qp->qp_access_flags & +IB_ACCESS_REMOTE_WRITE ))) { + dev->n_pkt_drops++; + goto done; + } +if (opcode ==OP (RDMA_WRITE_ONLY )) + goto rdma_last; +else if (opcode == +OP (RDMA_WRITE_ONLY_WITH_IMMEDIATE )) + goto rdma_last_imm; +/* FALLTHROUGH */ +case OP (RDMA_WRITE_MIDDLE ): +/* Check for invalid length PMTU or posted rwqe len. */ +if (unlikely (tlen != (hdrsize + pmtu +4 ))) { + dev->n_pkt_drops++; + goto done; + } + qp->r_rcv_len += pmtu; +if (unlikely (qp->r_rcv_len > qp->r_len)) { + dev->n_pkt_drops++; + goto done; + } +ipath_copy_sge (&qp->r_sge, data, pmtu); +break ; + +case OP (RDMA_WRITE_LAST_WITH_IMMEDIATE ): + rdma_last_imm: +/* Get the number of bytes the message was padded by. */ + pad = (be32_to_cpu (ohdr->bth[0 ]) >>20 ) &3 ; +/* Check for invalid length. */ +/* XXX LAST len should be >= 1 */ +if (unlikely (tlen < (hdrsize + pad +4 ))) { + dev->n_pkt_drops++; + goto done; + } +/* Don't count the CRC. */ + tlen -= (hdrsize + pad +4 ); +if (unlikely (tlen + qp->r_rcv_len != qp->r_len)) { + dev->n_pkt_drops++; + goto done; + } +if (qp->r_reuse_sge) { + qp->r_reuse_sge =0 ; + }else if (!ipath_get_rwqe (qp,1 )) { + dev->n_pkt_drops++; + goto done; + } +if (header_in_data) { + wc.imm_data = *(u32 *) data; + data +=sizeof (u32); + }else { +/* Immediate data comes after BTH */ + wc.imm_data = ohdr->u.imm_data; + } + hdrsize +=4 ; + wc.wc_flags =IB_WC_WITH_IMM ; + wc.byte_len =0 ; + goto last_imm; + +case OP (RDMA_WRITE_LAST ): + rdma_last: +/* Get the number of bytes the message was padded by. */ + pad = (be32_to_cpu (ohdr->bth[0 ]) >>20 ) &3 ; +/* Check for invalid length. */ +/* XXX LAST len should be >= 1 */ +if (unlikely (tlen < (hdrsize + pad +4 ))) { + dev->n_pkt_drops++; + goto done; + } +/* Don't count the CRC. */ + tlen -= (hdrsize + pad +4 ); +if (unlikely (tlen + qp->r_rcv_len != qp->r_len)) { + dev->n_pkt_drops++; + goto done; + } +ipath_copy_sge (&qp->r_sge, data, tlen); +break ; + +default : +/* Drop packet for unknown opcodes. */ +spin_unlock_irqrestore (&qp->r_rq.lock, flags); + dev->n_pkt_drops++; +return ; + } + qp->r_psn++; + qp->r_state = opcode; +done: +spin_unlock_irqrestore (&qp->r_rq.lock, flags); +}
drivers/infiniband/hw/ipath/ipath_ud.c new file +600 -0
600 lines not on this page · raw message
PATCH 1/1 ipath - infiniband RC protocol support
In reply to Bryan O'SullivanSigned-off-by Bryan O'Sullivan <[email protected]>
drivers/infiniband/hw/ipath/ipath_rc.c new file +1769 -0
@@ -0,0 +1,1769 @@ +/* +* Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. +* +* This software is available to you under a choice of one of two +* licenses. You may choose to be licensed under the terms of the GNU +* General Public License (GPL) Version 2, available from the file +* COPYING in the main directory of this source tree, or the +* OpenIB.org BSD license below: +* +* Redistribution and use in source and binary forms, with or +* without modification, are permitted provided that the following +* conditions are met: +* +* - Redistributions of source code must retain the above +* copyright notice, this list of conditions and the following +* disclaimer. +* +* - Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following +* disclaimer in the documentation and/or other materials +* provided with the distribution. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ + +#include "ipath_verbs.h" +#include "ips_common.h" + +/* cut down ridiculously long IB macro names */ +#define OP (x) IB_OPCODE_RC_##x + +/** +* ipath_init_restart- initialize the qp->s_sge after a restart +* @qp: the QP who's SGE we're restarting +* @wqe: the work queue to initialize the QP's SGE from +* +* The QP s_lock should be held. +*/ +static void ipath_init_restart (struct ipath_qp *qp,struct ipath_swqe *wqe) +{ +struct ipath_ibdev *dev; +u32 len; + + len = ((qp->s_psn - wqe->psn) &IPS_PSN_MASK ) * +ib_mtu_enum_to_int (qp->path_mtu); + qp->s_sge.sge = wqe->sg_list[0 ]; + qp->s_sge.sg_list = wqe->sg_list +1 ; + qp->s_sge.num_sge = wqe->wr.num_sge; +ipath_skip_sge (&qp->s_sge, len); + qp->s_len = wqe->length - len; + dev =to_idev (qp->ibqp.device); +spin_lock (&dev->pending_lock); +if (qp->timerwait.next ==LIST_POISON1 ) +list_add_tail (&qp->timerwait, + &dev->pending[dev->pending_index]); +spin_unlock (&dev->pending_lock); +} + +/** +* ipath_do_rc_send - perform a send on an RC QP +* @data: contains a pointer to the QP +* +* Process entries in the send work queue until credit or queue is +* exhausted. Only allow one CPU to send a packet per QP (tasklet). +* Otherwise, after we drop the QP s_lock, two threads could send +* packets out of order. +*/ +void ipath_do_rc_send (unsigned long data) +{ +struct ipath_qp *qp = (struct ipath_qp *)data; +struct ipath_ibdev *dev =to_idev (qp->ibqp.device); +struct ipath_swqe *wqe; +struct ipath_sge_state *ss; +unsigned long flags; +u16 lrh0; +u32 hwords; +u32 nwords; +u32 extra_bytes; +u32 bth0; +u32 bth2; +u32 pmtu =ib_mtu_enum_to_int (qp->path_mtu); +u32 len; +struct ipath_other_headers *ohdr; +char newreq; + +if (test_and_set_bit (IPATH_S_BUSY , &qp->s_flags)) +return ; + +if (unlikely (qp->remote_ah_attr.dlid == +ipath_layer_get_lid (dev->dd))) { +struct ib_wc wc; + +/* +* Pass in an uninitialized ib_wc to be consistent with +* other places where ipath_ruc_loopback() is called. +*/ +ipath_ruc_loopback (qp, &wc); +clear_bit (IPATH_S_BUSY , &qp->s_flags); +return ; + } + + ohdr = &qp->s_hdr.u.oth; +if (qp->remote_ah_attr.ah_flags &IB_AH_GRH ) + ohdr = &qp->s_hdr.u.l.oth; + +again: +/* Check for a constructed packet to be sent. */ +if (qp->s_hdrwords !=0 ) { +/* +* If no PIO bufs are available, return. +* An interrupt will call ipath_ib_piobufavail() +* when one is available. +*/ +if (ipath_verbs_send (dev->dd, qp->s_hdrwords, + (u32 *) &qp->s_hdr, + qp->s_cur_size, + qp->s_cur_sge)) { +ipath_no_bufs_available (qp, dev); +return ; + } + dev->n_unicast_xmit++; +/* Record that we sent the packet and s_hdr is empty. */ + qp->s_hdrwords =0 ; + } + + lrh0 =IPS_LRH_BTH ; +/* header size in 32-bit words LRH+BTH = (8+12)/4. */ + hwords =5 ; + +/* +* The lock is needed to synchronize between +* setting qp->s_ack_state, resend timer, and post_send(). +*/ +spin_lock_irqsave (&qp->s_lock, flags); + + bth0 =ipath_layer_get_pkey (dev->dd, qp->s_pkey_index); + +/* Sending responses has higher priority over sending requests. */ +if (qp->s_ack_state !=OP (ACKNOWLEDGE )) { +/* +* Send a response. Note that we are in the responder's +* side of the QP context. +*/ +switch (qp->s_ack_state) { +case OP (RDMA_READ_REQUEST ): + ss = &qp->s_rdma_sge; + len = qp->s_rdma_len; +if (len > pmtu) { + len = pmtu; + qp->s_ack_state = +OP (RDMA_READ_RESPONSE_FIRST ); + }else + qp->s_ack_state = +OP (RDMA_READ_RESPONSE_ONLY ); + qp->s_rdma_len -= len; + bth0 |= qp->s_ack_state <<24 ; + ohdr->u.aeth =ipath_compute_aeth (qp); + hwords++; +break ; + +case OP (RDMA_READ_RESPONSE_FIRST ): + qp->s_ack_state = +OP (RDMA_READ_RESPONSE_MIDDLE ); +/* FALLTHROUGH */ +case OP (RDMA_READ_RESPONSE_MIDDLE ): + ss = &qp->s_rdma_sge; + len = qp->s_rdma_len; +if (len > pmtu) + len = pmtu; +else { + ohdr->u.aeth =ipath_compute_aeth (qp); + hwords++; + qp->s_ack_state = +OP (RDMA_READ_RESPONSE_LAST ); + } + qp->s_rdma_len -= len; + bth0 |= qp->s_ack_state <<24 ; +break ; + +case OP (RDMA_READ_RESPONSE_LAST ): +case OP (RDMA_READ_RESPONSE_ONLY ): +/* +* We have to prevent new requests from changing +* the r_sge state while a ipath_verbs_send() +* is in progress. +* Changing r_state allows the receiver +* to continue processing new packets. +* We do it here now instead of above so +* that we are sure the packet was sent before +* changing the state. +*/ + qp->r_state =OP (RDMA_READ_RESPONSE_LAST ); + qp->s_ack_state =OP (ACKNOWLEDGE ); + goto send_req; + +case OP (COMPARE_SWAP ): +case OP (FETCH_ADD ): + ss =NULL ; + len =0 ; + qp->r_state =OP (SEND_LAST ); + qp->s_ack_state =OP (ACKNOWLEDGE ); + bth0 |=IB_OPCODE_ATOMIC_ACKNOWLEDGE <<24 ; + ohdr->u.at.aeth =ipath_compute_aeth (qp); + ohdr->u.at.atomic_ack_eth = +cpu_to_be64 (qp->s_ack_atomic); + hwords +=sizeof (ohdr->u.at) /4 ; +break ; + +default : +/* Send a regular ACK. */ + ss =NULL ; + len =0 ; + qp->s_ack_state =OP (ACKNOWLEDGE ); + bth0 |= qp->s_ack_state <<24 ; + ohdr->u.aeth =ipath_compute_aeth (qp); + hwords++; + } + bth2 = qp->s_ack_psn++ &IPS_PSN_MASK ; + }else { + send_req: +if (!(ib_ipath_state_ops[qp->state] & +IPATH_PROCESS_SEND_OK ) || qp->s_rnr_timeout) + goto done; + +/* Send a request. */ + wqe =get_swqe_ptr (qp, qp->s_cur); +switch (qp->s_state) { +default : +/* +* Resend an old request or start a new one. +* +* We keep track of the current SWQE so that +* we don't reset the "furthest progress" state +* if we need to back up. +*/ + newreq =0 ; +if (qp->s_cur == qp->s_tail) { +/* Check if send work queue is empty. */ +if (qp->s_tail == qp->s_head) + goto done; + qp->s_psn = wqe->psn = qp->s_next_psn; + newreq =1 ; + } +/* +* Note that we have to be careful not to modify the +* original work request since we may need to resend +* it. +*/ + qp->s_sge.sge = wqe->sg_list[0 ]; + qp->s_sge.sg_list = wqe->sg_list +1 ; + qp->s_sge.num_sge = wqe->wr.num_sge; + qp->s_len = len = wqe->length; + ss = &qp->s_sge; + bth2 =0 ; +switch (wqe->wr.opcode) { +case IB_WR_SEND : +case IB_WR_SEND_WITH_IMM : +/* If no credit, return. */ +if (qp->s_lsn != (u32) -1 && +ipath_cmp24 (wqe->ssn, + qp->s_lsn +1 ) >0 ) + goto done; + wqe->lpsn = wqe->psn; +if (len > pmtu) { + wqe->lpsn += (len -1 ) / pmtu; + qp->s_state = +OP (SEND_FIRST ); + len = pmtu; +break ; + } +if (wqe->wr.opcode ==IB_WR_SEND ) + qp->s_state = +OP (SEND_ONLY ); +else { + qp->s_state = +OP (SEND_ONLY_WITH_IMMEDIATE ); +/* +* Immediate data comes after the +* BTH +*/ + ohdr->u.imm_data = wqe->wr.imm_data; + hwords +=1 ; + } +if (wqe->wr.send_flags &IB_SEND_SOLICITED ) + bth0 |=1 <<23 ; + bth2 =1 <<31 ;/* Request ACK. */ +if (++qp->s_cur == qp->s_size) + qp->s_cur =0 ; +break ; + +case IB_WR_RDMA_WRITE : +if (newreq) + qp->s_lsn++; +/* FALLTHROUGH */ +case IB_WR_RDMA_WRITE_WITH_IMM : +/* If no credit, return. */ +if (qp->s_lsn != (u32) -1 && +ipath_cmp24 (wqe->ssn, + qp->s_lsn +1 ) >0 ) + goto done; + ohdr->u.rc.reth.vaddr = +cpu_to_be64 (wqe->wr.wr.rdma.remote_addr); + ohdr->u.rc.reth.rkey = +cpu_to_be32 (wqe->wr.wr.rdma.rkey); + ohdr->u.rc.reth.length =cpu_to_be32 (len); + hwords +=sizeof (struct ib_reth ) /4 ; + wqe->lpsn = wqe->psn; +if (len > pmtu) { + wqe->lpsn += (len -1 ) / pmtu; + qp->s_state = +OP (RDMA_WRITE_FIRST ); + len = pmtu; +break ; + } +if (wqe->wr.opcode ==IB_WR_RDMA_WRITE ) + qp->s_state = +OP (RDMA_WRITE_ONLY ); +else { + qp->s_state = +OP (RDMA_WRITE_ONLY_WITH_IMMEDIATE ); +/* Immediate data comes +* after RETH */ + ohdr->u.rc.imm_data = + wqe->wr.imm_data; + hwords +=1 ; +if (wqe->wr.send_flags & +IB_SEND_SOLICITED ) + bth0 |=1 <<23 ; + } + bth2 =1 <<31 ;/* Request ACK. */ +if (++qp->s_cur == qp->s_size) + qp->s_cur =0 ; +break ; + +case IB_WR_RDMA_READ : + ohdr->u.rc.reth.vaddr = +cpu_to_be64 (wqe->wr.wr.rdma.remote_addr); + ohdr->u.rc.reth.rkey = +cpu_to_be32 (wqe->wr.wr.rdma.rkey); + ohdr->u.rc.reth.length =cpu_to_be32 (len); + qp->s_state =OP (RDMA_READ_REQUEST ); + hwords +=sizeof (ohdr->u.rc.reth) /4 ; +if (newreq) { + qp->s_lsn++; +/* +* Adjust s_next_psn to count the +* expected number of responses. +*/ +if (len > pmtu) + qp->s_next_psn += + (len -1 ) / pmtu; + wqe->lpsn = qp->s_next_psn++; + } + ss =NULL ; + len =0 ; +if (++qp->s_cur == qp->s_size) + qp->s_cur =0 ; +break ; + +case IB_WR_ATOMIC_CMP_AND_SWP : +case IB_WR_ATOMIC_FETCH_AND_ADD : + qp->s_state = (wqe->wr.opcode == +IB_WR_ATOMIC_CMP_AND_SWP ) + ?OP (COMPARE_SWAP ) + :OP (FETCH_ADD ); + ohdr->u.atomic_eth.vaddr = +cpu_to_be64 (wqe->wr.wr.atomic.remote_addr); + ohdr->u.atomic_eth.rkey = +cpu_to_be32 (wqe->wr.wr.atomic.rkey); + ohdr->u.atomic_eth.swap_data = +cpu_to_be64 (wqe->wr.wr.atomic.swap); + ohdr->u.atomic_eth.compare_data = +cpu_to_be64 (wqe->wr.wr.atomic.compare_add); + hwords +=sizeof (struct ib_atomic_eth ) /4 ; +if (newreq) { + qp->s_lsn++; + wqe->lpsn = wqe->psn; + } +if (++qp->s_cur == qp->s_size) + qp->s_cur =0 ; + ss =NULL ; + len =0 ; +break ; + +default : + goto done; + } +if (newreq) { + qp->s_tail++; +if (qp->s_tail >= qp->s_size) + qp->s_tail =0 ; + } + bth2 |= qp->s_psn++ &IPS_PSN_MASK ; +if ((int )(qp->s_psn - qp->s_next_psn) >0 ) + qp->s_next_psn = qp->s_psn; +spin_lock (&dev->pending_lock); +if (qp->timerwait.next ==LIST_POISON1 ) +list_add_tail (&qp->timerwait, + &dev->pending[dev-> + pending_index]); +spin_unlock (&dev->pending_lock); +break ; + +case OP (RDMA_READ_RESPONSE_FIRST ): +/* +* This case can only happen if a send is +* restarted. See ipath_restart_rc(). +*/ +ipath_init_restart (qp, wqe); +/* FALLTHROUGH */ +case OP (SEND_FIRST ): + qp->s_state =OP (SEND_MIDDLE ); +/* FALLTHROUGH */ +case OP (SEND_MIDDLE ): + bth2 = qp->s_psn++ &IPS_PSN_MASK ; +if ((int )(qp->s_psn - qp->s_next_psn) >0 ) + qp->s_next_psn = qp->s_psn; + ss = &qp->s_sge; + len = qp->s_len; +if (len > pmtu) { +/* +* Request an ACK every 1/2 MB to avoid +* retransmit timeouts. +*/ +if (((wqe->length - len) % + (512 *1024 )) ==0 ) + bth2 |=1 <<31 ; + len = pmtu; +break ; + } +if (wqe->wr.opcode ==IB_WR_SEND ) + qp->s_state =OP (SEND_LAST ); +else { + qp->s_state = +OP (SEND_LAST_WITH_IMMEDIATE ); +/* Immediate data comes after the BTH */ + ohdr->u.imm_data = wqe->wr.imm_data; + hwords +=1 ; + } +if (wqe->wr.send_flags &IB_SEND_SOLICITED ) + bth0 |=1 <<23 ; + bth2 |=1 <<31 ;/* Request ACK. */ + qp->s_cur++; +if (qp->s_cur >= qp->s_size) + qp->s_cur =0 ; +break ; + +case OP (RDMA_READ_RESPONSE_LAST ): +/* +* This case can only happen if a RDMA write is +* restarted. See ipath_restart_rc(). +*/ +ipath_init_restart (qp, wqe); +/* FALLTHROUGH */ +case OP (RDMA_WRITE_FIRST ): + qp->s_state =OP (RDMA_WRITE_MIDDLE ); +/* FALLTHROUGH */ +case OP (RDMA_WRITE_MIDDLE ): + bth2 = qp->s_psn++ &IPS_PSN_MASK ; +if ((int )(qp->s_psn - qp->s_next_psn) >0 ) + qp->s_next_psn = qp->s_psn; + ss = &qp->s_sge; + len = qp->s_len; +if (len > pmtu) { +/* +* Request an ACK every 1/2 MB to avoid +* retransmit timeouts. +*/ +if (((wqe->length - len) % + (512 *1024 )) ==0 ) + bth2 |=1 <<31 ; + len = pmtu; +break ; + } +if (wqe->wr.opcode ==IB_WR_RDMA_WRITE ) + qp->s_state =OP (RDMA_WRITE_LAST ); +else { + qp->s_state = +OP (RDMA_WRITE_LAST_WITH_IMMEDIATE ); +/* Immediate data comes after the BTH */ + ohdr->u.imm_data = wqe->wr.imm_data; + hwords +=1 ; +if (wqe->wr.send_flags &IB_SEND_SOLICITED ) + bth0 |=1 <<23 ; + } + bth2 |=1 <<31 ;/* Request ACK. */ + qp->s_cur++; +if (qp->s_cur >= qp->s_size) + qp->s_cur =0 ; +break ; + +case OP (RDMA_READ_RESPONSE_MIDDLE ): +/* +* This case can only happen if a RDMA read is +* restarted. See ipath_restart_rc(). +*/ +ipath_init_restart (qp, wqe); + len = ((qp->s_psn - wqe->psn) & +IPS_PSN_MASK ) * pmtu; + ohdr->u.rc.reth.vaddr = +cpu_to_be64 (wqe->wr.wr.rdma.remote_addr + + len); + ohdr->u.rc.reth.rkey = +cpu_to_be32 (wqe->wr.wr.rdma.rkey); + ohdr->u.rc.reth.length =cpu_to_be32 (qp->s_len); + qp->s_state =OP (RDMA_READ_REQUEST ); + hwords +=sizeof (ohdr->u.rc.reth) /4 ; + bth2 = qp->s_psn++ &IPS_PSN_MASK ; +if ((int )(qp->s_psn - qp->s_next_psn) >0 ) + qp->s_next_psn = qp->s_psn; + ss =NULL ; + len =0 ; + qp->s_cur++; +if (qp->s_cur == qp->s_size) + qp->s_cur =0 ; +break ; + +case OP (RDMA_READ_REQUEST ): +case OP (COMPARE_SWAP ): +case OP (FETCH_ADD ): +/* +* We shouldn't start anything new until this +* request is finished. The ACK will handle +* rescheduling us. XXX The number of outstanding +* ones is negotiated at connection setup time (see +* pg. 258,289)? XXX Also, if we support multiple +* outstanding requests, we need to check the WQE +* IB_SEND_FENCE flag and not send a new request if +* a RDMA read or atomic is pending. +*/ + goto done; + } + qp->s_len -= len; + bth0 |= qp->s_state <<24 ; +/* XXX queue resend timeout. */ + } +/* Make sure it is non-zero before dropping the lock. */ + qp->s_hdrwords = hwords; +spin_unlock_irqrestore (&qp->s_lock, flags); + +/* Construct the header. */ + extra_bytes = (4 - len) &3 ; + nwords = (len + extra_bytes) >>2 ; +if (unlikely (qp->remote_ah_attr.ah_flags &IB_AH_GRH )) { +/* Header size in 32-bit words. */ + hwords +=10 ; + lrh0 =IPS_LRH_GRH ; + qp->s_hdr.u.l.grh.version_tclass_flow = +cpu_to_be32 ((6 <<28 ) | + (qp->remote_ah_attr.grh.traffic_class + <<20 ) | + qp->remote_ah_attr.grh.flow_label); + qp->s_hdr.u.l.grh.paylen = +cpu_to_be16 (((hwords -12 ) + nwords + +SIZE_OF_CRC ) <<2 ); +/* next_hdr is defined by C8-7 in ch. 8.4.1 */ + qp->s_hdr.u.l.grh.next_hdr =0x1B ; + qp->s_hdr.u.l.grh.hop_limit = + qp->remote_ah_attr.grh.hop_limit; +/* The SGID is 32-bit aligned. */ + qp->s_hdr.u.l.grh.sgid.global.subnet_prefix = + dev->gid_prefix; + qp->s_hdr.u.l.grh.sgid.global.interface_id = +ipath_layer_get_guid (dev->dd); + qp->s_hdr.u.l.grh.dgid = qp->remote_ah_attr.grh.dgid; + qp->s_hdrwords = hwords; + } + qp->s_cur_sge = ss; + qp->s_cur_size = len; + lrh0 |= qp->remote_ah_attr.sl <<4 ; + qp->s_hdr.lrh[0 ] =cpu_to_be16 (lrh0); +/* DEST LID */ + qp->s_hdr.lrh[1 ] =cpu_to_be16 (qp->remote_ah_attr.dlid); + qp->s_hdr.lrh[2 ] =cpu_to_be16 (hwords + nwords +SIZE_OF_CRC ); + qp->s_hdr.lrh[3 ] =cpu_to_be16 (ipath_layer_get_lid (dev->dd)); + bth0 |= extra_bytes <<20 ; + ohdr->bth[0 ] =cpu_to_be32 (bth0); + ohdr->bth[1 ] =cpu_to_be32 (qp->remote_qpn); + ohdr->bth[2 ] =cpu_to_be32 (bth2); + +/* Check for more work to do. */ + goto again; + +done: +spin_unlock_irqrestore (&qp->s_lock, flags); +clear_bit (IPATH_S_BUSY , &qp->s_flags); +} + +static void send_rc_ack (struct ipath_qp *qp) +{ +struct ipath_ibdev *dev =to_idev (qp->ibqp.device); +u16 lrh0; +u32 bth0; +u32 hwords; +struct ipath_other_headers *ohdr; + +/* Construct the header. */ + ohdr = &qp->s_hdr.u.oth; + lrh0 =IPS_LRH_BTH ; +/* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4. */ + hwords =6 ; +if (unlikely (qp->remote_ah_attr.ah_flags &IB_AH_GRH )) { + ohdr = &qp->s_hdr.u.l.oth; +/* Header size in 32-bit words. */ + hwords +=10 ; + lrh0 =IPS_LRH_GRH ; + qp->s_hdr.u.l.grh.version_tclass_flow = +cpu_to_be32 ( + (6 <<28 ) | + (qp->remote_ah_attr.grh.traffic_class + <<20 ) | + qp->remote_ah_attr.grh.flow_label); + qp->s_hdr.u.l.grh.paylen = +cpu_to_be16 (((hwords -12 ) +SIZE_OF_CRC ) <<2 ); +/* next_hdr is defined by C8-7 in ch. 8.4.1 */ + qp->s_hdr.u.l.grh.next_hdr =0x1B ; + qp->s_hdr.u.l.grh.hop_limit = + qp->remote_ah_attr.grh.hop_limit; +/* The SGID is 32-bit aligned. */ + qp->s_hdr.u.l.grh.sgid.global.subnet_prefix = + dev->gid_prefix; + qp->s_hdr.u.l.grh.sgid.global.interface_id = +ipath_layer_get_guid (dev->dd); + qp->s_hdr.u.l.grh.dgid = qp->remote_ah_attr.grh.dgid; + } + bth0 =ipath_layer_get_pkey (dev->dd, qp->s_pkey_index); + ohdr->u.aeth =ipath_compute_aeth (qp); +if (qp->s_ack_state >=OP (COMPARE_SWAP )) { + bth0 |=IB_OPCODE_ATOMIC_ACKNOWLEDGE <<24 ; + ohdr->u.at.atomic_ack_eth =cpu_to_be64 (qp->s_ack_atomic); + hwords +=sizeof (ohdr->u.at.atomic_ack_eth) /4 ; + } +else + bth0 |=OP (ACKNOWLEDGE ) <<24 ; + lrh0 |= qp->remote_ah_attr.sl <<4 ; + qp->s_hdr.lrh[0 ] =cpu_to_be16 (lrh0); +/* DEST LID */ + qp->s_hdr.lrh[1 ] =cpu_to_be16 (qp->remote_ah_attr.dlid); + qp->s_hdr.lrh[2 ] =cpu_to_be16 (hwords +SIZE_OF_CRC ); + qp->s_hdr.lrh[3 ] =cpu_to_be16 (ipath_layer_get_lid (dev->dd)); + ohdr->bth[0 ] =cpu_to_be32 (bth0); + ohdr->bth[1 ] =cpu_to_be32 (qp->remote_qpn); + ohdr->bth[2 ] =cpu_to_be32 (qp->s_ack_psn &IPS_PSN_MASK ); + +/* +* If we can send the ACK, clear the ACK state. +*/ +if (ipath_verbs_send (dev->dd, hwords, (u32 *) &qp->s_hdr, +0 ,NULL ) ==0 ) { + qp->s_ack_state =OP (ACKNOWLEDGE ); + dev->n_rc_qacks++; + dev->n_unicast_xmit++; + } +} + +/** +* ipath_restart_rc - back up requester to resend the last un-ACKed request +* @qp: the QP to restart +* @psn: packet sequence number for the request +* @wc: the work completion request +* +* The QP s_lock should be held. +*/ +void ipath_restart_rc (struct ipath_qp *qp,u32 psn,struct ib_wc *wc) +{ +struct ipath_swqe *wqe =get_swqe_ptr (qp, qp->s_last); +struct ipath_ibdev *dev; +u32 n; + +/* +* If there are no requests pending, we are done. +*/ +if (ipath_cmp24 (psn, qp->s_next_psn) >=0 || + qp->s_last == qp->s_tail) + goto done; + +if (qp->s_retry ==0 ) { + wc->wr_id = wqe->wr.wr_id; + wc->status =IB_WC_RETRY_EXC_ERR ; + wc->opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; + wc->vendor_err =0 ; + wc->byte_len =0 ; + wc->qp_num = qp->ibqp.qp_num; + wc->src_qp = qp->remote_qpn; + wc->pkey_index =0 ; + wc->slid = qp->remote_ah_attr.dlid; + wc->sl = qp->remote_ah_attr.sl; + wc->dlid_path_bits =0 ; + wc->port_num =0 ; +ipath_sqerror_qp (qp, wc); +return ; + } + qp->s_retry--; + +/* +* Remove the QP from the timeout queue. +* Note: it may already have been removed by ipath_ib_timer(). +*/ + dev =to_idev (qp->ibqp.device); +spin_lock (&dev->pending_lock); +if (qp->timerwait.next !=LIST_POISON1 ) +list_del (&qp->timerwait); +spin_unlock (&dev->pending_lock); + +if (wqe->wr.opcode ==IB_WR_RDMA_READ ) + dev->n_rc_resends++; +else + dev->n_rc_resends += (int )qp->s_psn - (int )psn; + +/* +* If we are starting the request from the beginning, let the +* normal send code handle initialization. +*/ + qp->s_cur = qp->s_last; +if (ipath_cmp24 (psn, wqe->psn) <=0 ) { + qp->s_state =OP (SEND_LAST ); + qp->s_psn = wqe->psn; + }else { + n = qp->s_cur; +for (;;) { +if (++n == qp->s_size) + n =0 ; +if (n == qp->s_tail) { +if (ipath_cmp24 (psn, qp->s_next_psn) >=0 ) { + qp->s_cur = n; + wqe =get_swqe_ptr (qp, n); + } +break ; + } + wqe =get_swqe_ptr (qp, n); +if (ipath_cmp24 (psn, wqe->psn) <0 ) +break ; + qp->s_cur = n; + } + qp->s_psn = psn; + +/* +* Reset the state to restart in the middle of a request. +* Don't change the s_sge, s_cur_sge, or s_cur_size. +* See ipath_do_rc_send(). +*/ +switch (wqe->wr.opcode) { +case IB_WR_SEND : +case IB_WR_SEND_WITH_IMM : + qp->s_state =OP (RDMA_READ_RESPONSE_FIRST ); +break ; + +case IB_WR_RDMA_WRITE : +case IB_WR_RDMA_WRITE_WITH_IMM : + qp->s_state =OP (RDMA_READ_RESPONSE_LAST ); +break ; + +case IB_WR_RDMA_READ : + qp->s_state = +OP (RDMA_READ_RESPONSE_MIDDLE ); +break ; + +default : +/* +* This case shouldn't happen since its only +* one PSN per req. +*/ + qp->s_state =OP (SEND_LAST ); + } + } + +done: +tasklet_hi_schedule (&qp->s_task); +} + +/** +* do_rc_ack - process an incoming RC ACK +* @qp: the QP the ACK came in on +* @psn: the packet sequence number of the ACK +* @opcode: the opcode of the request that resulted in the ACK +* +* This is called from ipath_rc_rcv() to process an incoming RC ACK +* for the given QP. +* Called at interrupt level with the QP s_lock held. +* Returns 1 if OK, 0 if current operation should be aborted (NAK). +*/ +static int do_rc_ack (struct ipath_qp *qp,u32 aeth,u32 psn,int opcode) +{ +struct ipath_ibdev *dev =to_idev (qp->ibqp.device); +struct ib_wc wc; +struct ipath_swqe *wqe; + +/* +* Remove the QP from the timeout queue (or RNR timeout queue). +* If ipath_ib_timer() has already removed it, +* it's OK since we hold the QP s_lock and ipath_restart_rc() +* just won't find anything to restart if we ACK everything. +*/ +spin_lock (&dev->pending_lock); +if (qp->timerwait.next !=LIST_POISON1 ) +list_del (&qp->timerwait); +spin_unlock (&dev->pending_lock); + +/* +* Note that NAKs implicitly ACK outstanding SEND and +* RDMA write requests and implicitly NAK RDMA read and +* atomic requests issued before the NAK'ed request. +* The MSN won't include the NAK'ed request but will include +* an ACK'ed request(s). +*/ + wqe =get_swqe_ptr (qp, qp->s_last); + +/* Nothing is pending to ACK/NAK. */ +if (qp->s_last == qp->s_tail) +return 0 ; + +/* +* The MSN might be for a later WQE than the PSN indicates so +* only complete WQEs that the PSN finishes. +*/ +while (ipath_cmp24 (psn, wqe->lpsn) >=0 ) { +/* If we are ACKing a WQE, the MSN should be >= the SSN. */ +if (ipath_cmp24 (aeth, wqe->ssn) <0 ) +break ; +/* +* If this request is a RDMA read or atomic, and the ACK is +* for a later operation, this ACK NAKs the RDMA read or +* atomic. In other words, only a RDMA_READ_LAST or ONLY +* can ACK a RDMA read and likewise for atomic ops. Note +* that the NAK case can only happen if relaxed ordering is +* used and requests are sent after an RDMA read or atomic +* is sent but before the response is received. +*/ +if ((wqe->wr.opcode ==IB_WR_RDMA_READ && + opcode !=OP (RDMA_READ_RESPONSE_LAST )) || + ((wqe->wr.opcode ==IB_WR_ATOMIC_CMP_AND_SWP || + wqe->wr.opcode ==IB_WR_ATOMIC_FETCH_AND_ADD ) && + (opcode !=OP (ATOMIC_ACKNOWLEDGE ) || +ipath_cmp24 (wqe->psn, psn) !=0 ))) { +/* +* The last valid PSN seen is the previous +* request's. +*/ + qp->s_last_psn = wqe->psn -1 ; +/* Retry this request. */ +ipath_restart_rc (qp, wqe->psn, &wc); +/* +* No need to process the ACK/NAK since we are +* restarting an earlier request. +*/ +return 0 ; + } +/* Post a send completion queue entry if requested. */ +if (!test_bit (IPATH_S_SIGNAL_REQ_WR , &qp->s_flags) || + (wqe->wr.send_flags &IB_SEND_SIGNALED )) { + wc.wr_id = wqe->wr.wr_id; + wc.status =IB_WC_SUCCESS ; + wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; + wc.vendor_err =0 ; + wc.byte_len = wqe->length; + wc.qp_num = qp->ibqp.qp_num; + wc.src_qp = qp->remote_qpn; + wc.pkey_index =0 ; + wc.slid = qp->remote_ah_attr.dlid; + wc.sl = qp->remote_ah_attr.sl; + wc.dlid_path_bits =0 ; + wc.port_num =0 ; +ipath_cq_enter (to_icq (qp->ibqp.send_cq), &wc,0 ); + } + qp->s_retry = qp->s_retry_cnt; +/* +* If we are completing a request which is in the process +* of being resent, we can stop resending it since we know +* the responder has already seen it. +*/ +if (qp->s_last == qp->s_cur) { +if (++qp->s_cur >= qp->s_size) + qp->s_cur =0 ; + wqe =get_swqe_ptr (qp, qp->s_cur); + qp->s_state =OP (SEND_LAST ); + qp->s_psn = wqe->psn; + } +if (++qp->s_last >= qp->s_size) + qp->s_last =0 ; + wqe =get_swqe_ptr (qp, qp->s_last); +if (qp->s_last == qp->s_tail) +break ; + } + +switch (aeth >>29 ) { +case 0 :/* ACK */ + dev->n_rc_acks++; +/* If this is a partial ACK, reset the retransmit timer. */ +if (qp->s_last != qp->s_tail) { +spin_lock (&dev->pending_lock); +list_add_tail (&qp->timerwait, + &dev->pending[dev->pending_index]); +spin_unlock (&dev->pending_lock); + } +ipath_get_credit (qp, aeth); + qp->s_rnr_retry = qp->s_rnr_retry_cnt; + qp->s_retry = qp->s_retry_cnt; + qp->s_last_psn = psn; +return 1 ; + +case 1 :/* RNR NAK */ + dev->n_rnr_naks++; +if (qp->s_rnr_retry ==0 ) { +if (qp->s_last == qp->s_tail) +return 0 ; + + wc.status =IB_WC_RNR_RETRY_EXC_ERR ; + goto class_b; + } +if (qp->s_rnr_retry_cnt <7 ) + qp->s_rnr_retry--; +if (qp->s_last == qp->s_tail) +return 0 ; + +/* The last valid PSN seen is the previous request's. */ + qp->s_last_psn = wqe->psn -1 ; + +/* Restart this request after the RNR timeout. */ + wqe =get_swqe_ptr (qp, qp->s_last); + + dev->n_rc_resends += (int )qp->s_psn - (int )psn; + +/* +* If we are starting the request from the beginning, let +* the normal send code handle initialization. +*/ + qp->s_cur = qp->s_last; +if (ipath_cmp24 (psn, wqe->psn) <=0 ) { + qp->s_state =OP (SEND_LAST ); + qp->s_psn = wqe->psn; + }else { +u32 n; + + n = qp->s_cur; +for (;;) { +if (++n == qp->s_size) + n =0 ; +if (n == qp->s_tail) { +if (ipath_cmp24 (psn, + qp->s_next_psn) + >=0 ) { + qp->s_cur = n; + wqe =get_swqe_ptr (qp, n); + } +break ; + } + wqe =get_swqe_ptr (qp, n); +if (ipath_cmp24 (psn, wqe->psn) <0 ) +break ; + qp->s_cur = n; + } + qp->s_psn = psn; + +/* +* Set the state to restart in the middle of a +* request. Don't change the s_sge, s_cur_sge, or +* s_cur_size. See ipath_do_rc_send(). +*/ +switch (wqe->wr.opcode) { +case IB_WR_SEND : +case IB_WR_SEND_WITH_IMM : + qp->s_state = +OP (RDMA_READ_RESPONSE_FIRST ); +break ; + +case IB_WR_RDMA_WRITE : +case IB_WR_RDMA_WRITE_WITH_IMM : + qp->s_state = +OP (RDMA_READ_RESPONSE_LAST ); +break ; + +case IB_WR_RDMA_READ : + qp->s_state = +OP (RDMA_READ_RESPONSE_MIDDLE ); +break ; + +default : +/* +* This case shouldn't happen since its only +* one PSN per req. +*/ + qp->s_state =OP (SEND_LAST ); + } + } + + qp->s_rnr_timeout = + ib_ipath_rnr_table[(aeth >>IPS_AETH_CREDIT_SHIFT ) & +IPS_AETH_CREDIT_MASK ]; +ipath_insert_rnr_queue (qp); +return 0 ; + +case 3 :/* NAK */ +/* The last valid PSN seen is the previous request's. */ +if (qp->s_last != qp->s_tail) + qp->s_last_psn = wqe->psn -1 ; +switch ((aeth >>IPS_AETH_CREDIT_SHIFT ) & +IPS_AETH_CREDIT_MASK ) { +case 0 :/* PSN sequence error */ + dev->n_seq_naks++; +/* +* Back up to the responder's expected PSN. +* XXX Note that we might get a NAK in the +* middle of an RDMA READ response which +* terminates the RDMA READ. +*/ +if (qp->s_last == qp->s_tail) +break ; + +if (ipath_cmp24 (psn, wqe->psn) <0 ) +break ; + +/* Retry the request. */ +ipath_restart_rc (qp, psn, &wc); +break ; + +case 1 :/* Invalid Request */ + wc.status =IB_WC_REM_INV_REQ_ERR ; + dev->n_other_naks++; + goto class_b; + +case 2 :/* Remote Access Error */ + wc.status =IB_WC_REM_ACCESS_ERR ; + dev->n_other_naks++; + goto class_b; + +case 3 :/* Remote Operation Error */ + wc.status =IB_WC_REM_OP_ERR ; + dev->n_other_naks++; + class_b: + wc.wr_id = wqe->wr.wr_id; + wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode]; + wc.vendor_err =0 ; + wc.byte_len =0 ; + wc.qp_num = qp->ibqp.qp_num; + wc.src_qp = qp->remote_qpn; + wc.pkey_index =0 ; + wc.slid = qp->remote_ah_attr.dlid; + wc.sl = qp->remote_ah_attr.sl; + wc.dlid_path_bits =0 ; + wc.port_num =0 ; +ipath_sqerror_qp (qp, &wc); +break ; + +default : +/* Ignore other reserved NAK error codes */ + goto reserved; + } + qp->s_rnr_retry = qp->s_rnr_retry_cnt; +return 0 ; + +default :/* 2: reserved */ + reserved: +/* Ignore reserved NAK codes. */ +return 0 ; + } +} + +/** +* ipath_rc_rcv - process an incoming RC packet +* @dev: the device this packet came in on +* @hdr: the header of this packet +* @has_grh: true if the header has a GRH +* @data: the packet data +* @tlen: the packet length +* @qp: the QP for this packet +* +* This is called from ipath_qp_rcv() to process an incoming RC packet +* for the given QP. +* Called at interrupt level. +*/ +void ipath_rc_rcv (struct ipath_ibdev *dev,struct ipath_ib_header *hdr, +int has_grh,void *data,u32 tlen,struct ipath_qp *qp) +{ +struct ipath_other_headers *ohdr; +int opcode; +u32 hdrsize; +u32 psn; +u32 pad; +unsigned long flags; +struct ib_wc wc; +u32 pmtu =ib_mtu_enum_to_int (qp->path_mtu); +int diff; +struct ib_reth *reth; +int header_in_data; + +/* Check for GRH */ +if (!has_grh) { + ohdr = &hdr->u.oth; + hdrsize =8 +12 ;/* LRH + BTH */ + psn =be32_to_cpu (ohdr->bth[2 ]); + header_in_data =0 ; + }else { + ohdr = &hdr->u.l.oth; + hdrsize =8 +40 +12 ;/* LRH + GRH + BTH */ +/* +* The header with GRH is 60 bytes and the +* core driver sets the eager header buffer +* size to 56 bytes so the last 4 bytes of +* the BTH header (PSN) is in the data buffer. +*/ + header_in_data = +ipath_layer_get_rcvhdrentsize (dev->dd) ==16 ; +if (header_in_data) { + psn =be32_to_cpu (((u32 *) data)[0 ]); + data +=sizeof (u32); + }else + psn =be32_to_cpu (ohdr->bth[2 ]); + } +/* +* The opcode is in the low byte when its in network order +* (top byte when in host order). +*/ + opcode =be32_to_cpu (ohdr->bth[0 ]) >>24 ; + +/* +* Process responses (ACKs) before anything else. +* Note that the packet sequence number will be for something +* in the send work queue rather than the expected receive +* packet sequence number. In other words, this QP is the +* requester. +*/ +if (opcode >=OP (RDMA_READ_RESPONSE_FIRST ) && + opcode <=OP (ATOMIC_ACKNOWLEDGE )) { + +spin_lock_irqsave (&qp->s_lock, flags); + +/* Ignore invalid responses. */ +if (ipath_cmp24 (psn, qp->s_next_psn) >=0 ) + goto ack_done; + +/* Ignore duplicate responses. */ + diff =ipath_cmp24 (psn, qp->s_last_psn); +if (unlikely (diff <=0 )) { +/* Update credits for "ghost" ACKs */ +if (diff ==0 && + opcode ==OP (ACKNOWLEDGE )) { +if (!header_in_data) { + pad =be32_to_cpu (ohdr->u.aeth); + }else { + pad =be32_to_cpu ( + ((u32 *) data)[0 ]); + data +=sizeof (u32); + } +if ((pad >>29 ) ==0 ) +ipath_get_credit (qp, pad); + } + goto ack_done; + } + +switch (opcode) { +case OP (ACKNOWLEDGE ): +case OP (ATOMIC_ACKNOWLEDGE ): +case OP (RDMA_READ_RESPONSE_FIRST ): +if (!header_in_data) + pad =be32_to_cpu (ohdr->u.aeth); +else { + pad =be32_to_cpu (((u32 *) data)[0 ]); + data +=sizeof (u32); + } +if (opcode ==OP (ATOMIC_ACKNOWLEDGE )) + *(u64 *) qp->s_sge.sge.vaddr = + *(u64 *) data; +if (!do_rc_ack (qp, pad, psn, opcode) || + opcode !=OP (RDMA_READ_RESPONSE_FIRST )) + goto ack_done; + hdrsize +=4 ; +/* +* do_rc_ack() has already checked the PSN so skip +* the sequence check. +*/ + goto rdma_read; + +case OP (RDMA_READ_RESPONSE_MIDDLE ): +/* no AETH, no ACK */ +if (unlikely (ipath_cmp24 (psn, + qp->s_last_psn +1 ))) { + dev->n_rdma_seq++; +ipath_restart_rc (qp, qp->s_last_psn +1 , + &wc); + goto ack_done; + } + rdma_read: +if (unlikely (qp->s_state != +OP (RDMA_READ_REQUEST ))) + goto ack_done; +if (unlikely (tlen != (hdrsize + pmtu +4 ))) + goto ack_done; +if (unlikely (pmtu >= qp->s_len)) + goto ack_done; +/* We got a response so update the timeout. */ +if (unlikely (qp->s_last == qp->s_tail || +get_swqe_ptr (qp, qp->s_last)->wr.opcode != +IB_WR_RDMA_READ )) + goto ack_done; +spin_lock (&dev->pending_lock); +if (qp->s_rnr_timeout ==0 && + qp->timerwait.next !=LIST_POISON1 ) { +list_move_tail ( + &qp->timerwait, + &dev->pending[dev->pending_index]); + } +spin_unlock (&dev->pending_lock); +/* +* Update the RDMA receive state but do the copy w/o +* holding the locks and blocking interrupts. +* XXX Yet another place that affects relaxed +* RDMA order since we don't want s_sge modified. +*/ + qp->s_len -= pmtu; + qp->s_last_psn = psn; +spin_unlock_irqrestore (&qp->s_lock, flags); +ipath_copy_sge (&qp->s_sge, data, pmtu); +return ; + +case OP (RDMA_READ_RESPONSE_LAST ): +/* ACKs READ req. */ +if (unlikely (ipath_cmp24 (psn, + qp->s_last_psn +1 ))) { + dev->n_rdma_seq++; +ipath_restart_rc (qp, qp->s_last_psn +1 , + &wc); + goto ack_done; + } +/* FALLTHROUGH */ +case OP (RDMA_READ_RESPONSE_ONLY ): +if (unlikely (qp->s_state != +OP (RDMA_READ_REQUEST ))) + goto ack_done; +/* +* Get the number of bytes the message was padded +* by. +*/ + pad = (be32_to_cpu (ohdr->bth[0 ]) >>20 ) &3 ; +/* +* Check that the data size is >= 1 && <= pmtu. +* Remember to account for the AETH header (4) and +* ICRC (4). +*/ +if (unlikely (tlen <= (hdrsize + pad +8 ))) { +/* +* XXX Need to generate an error CQ +* entry. +*/ + goto ack_done; + } + tlen -= hdrsize + pad +8 ; +if (unlikely (tlen != qp->s_len)) { +/* +* XXX Need to generate an error CQ +* entry. +*/ + goto ack_done; + } +if (!header_in_data) { + pad =be32_to_cpu (ohdr->u.aeth); + }else { + pad =be32_to_cpu (((u32 *) data)[0 ]); + data +=sizeof (u32); + } +ipath_copy_sge (&qp->s_sge, data, tlen); +if (do_rc_ack ( + qp, pad, psn, +OP (RDMA_READ_RESPONSE_LAST ))) { +/* +* Change the state so we contimue +* processing new requests. +*/ + qp->s_state =OP (SEND_LAST ); + } + goto ack_done; + } + ack_done: +spin_unlock_irqrestore (&qp->s_lock, flags); +return ; + } + +spin_lock_irqsave (&qp->r_rq.lock, flags); + +/* Compute 24 bits worth of difference. */ + diff =ipath_cmp24 (psn, qp->r_psn); +if (unlikely (diff)) { +if (diff >0 ) { +/* +* Packet sequence error. +* A NAK will ACK earlier sends and RDMA writes. +* Don't queue the NAK if a RDMA read, atomic, or +* NAK is pending though. +*/ +spin_lock (&qp->s_lock); +if ((qp->s_ack_state >= +OP (RDMA_READ_REQUEST ) && + qp->s_ack_state !=IB_OPCODE_ACKNOWLEDGE ) || + qp->s_nak_state !=0 ) { +spin_unlock (&qp->s_lock); + goto done; + } + qp->s_ack_state =OP (SEND_ONLY ); + qp->s_nak_state =IB_NAK_PSN_ERROR ; +/* Use the expected PSN. */ + qp->s_ack_psn = qp->r_psn; + goto resched; + } + +/* +* Handle a duplicate request. Don't re-execute SEND, RDMA +* write or atomic op. Don't NAK errors, just silently drop +* the duplicate request. Note that r_sge, r_len, and +* r_rcv_len may be in use so don't modify them. +* +* We are supposed to ACK the earliest duplicate PSN but we +* can coalesce an outstanding duplicate ACK. We have to +* send the earliest so that RDMA reads can be restarted at +* the requester's expected PSN. +*/ +spin_lock (&qp->s_lock); +if (qp->s_ack_state !=IB_OPCODE_ACKNOWLEDGE && +ipath_cmp24 (psn, qp->s_ack_psn) >=0 ) { +if (qp->s_ack_state <IB_OPCODE_RDMA_READ_REQUEST ) + qp->s_ack_psn = psn; +spin_unlock (&qp->s_lock); + goto done; + } +switch (opcode) { +case OP (RDMA_READ_REQUEST ): +/* +* We have to be careful to not change s_rdma_sge +* while ipath_do_rc_send() is using it and not +* holding the s_lock. +*/ +if (qp->s_ack_state !=OP (ACKNOWLEDGE ) && + qp->s_ack_state >= +IB_OPCODE_RDMA_READ_REQUEST ) { +spin_unlock (&qp->s_lock); + dev->n_rdma_dup_busy++; + goto done; + } +/* RETH comes after BTH */ +if (!header_in_data) + reth = &ohdr->u.rc.reth; +else { + reth = (struct ib_reth *)data; + data +=sizeof (*reth); + } + qp->s_rdma_len =be32_to_cpu (reth->length); +if (qp->s_rdma_len !=0 ) { +u32 rkey =be32_to_cpu (reth->rkey); +u64 vaddr =be64_to_cpu (reth->vaddr); + +/* +* Address range must be a subset of the +* original request and start on pmtu +* boundaries. +*/ +if (unlikely (!ipath_rkey_ok ( + dev, + &qp->s_rdma_sge, + qp->s_rdma_len, + vaddr, rkey, +IB_ACCESS_REMOTE_READ ))) + goto done; + }else { + qp->s_rdma_sge.sg_list =NULL ; + qp->s_rdma_sge.num_sge =0 ; + qp->s_rdma_sge.sge.mr =NULL ; + qp->s_rdma_sge.sge.vaddr =NULL ; + qp->s_rdma_sge.sge.length =0 ; + qp->s_rdma_sge.sge.sge_length =0 ; + } +break ; + +case OP (COMPARE_SWAP ): +case OP (FETCH_ADD ): +/* +* Check for the PSN of the last atomic operations +* performed and resend the result if found. +*/ +if ((psn &IPS_PSN_MASK ) != qp->r_atomic_psn) { +spin_unlock (&qp->s_lock); + goto done; + } + qp->s_ack_atomic = qp->r_atomic_data; +break ; + } + qp->s_ack_state = opcode; + qp->s_nak_state =0 ; + qp->s_ack_psn = psn; + goto resched; + } + +/* Check for opcode sequence errors. */ +switch (qp->r_state) { +case OP (SEND_FIRST ): +case OP (SEND_MIDDLE ): +if (opcode ==OP (SEND_MIDDLE ) || + opcode ==OP (SEND_LAST ) || + opcode ==OP (SEND_LAST_WITH_IMMEDIATE )) +break ; + nack_inv: +/* +* A NAK will ACK earlier sends and RDMA writes. +* Don't queue the NAK if a RDMA read, atomic, or +* NAK is pending though. +*/ +spin_lock (&qp->s_lock); +if (qp->s_ack_state >=OP (RDMA_READ_REQUEST ) && + qp->s_ack_state !=IB_OPCODE_ACKNOWLEDGE ) { +spin_unlock (&qp->s_lock); + goto done; + } +/* XXX Flush WQEs */ + qp->state =IB_QPS_ERR ; + qp->s_ack_state =OP (SEND_ONLY ); + qp->s_nak_state =IB_NAK_INVALID_REQUEST ; + qp->s_ack_psn = qp->r_psn; + goto resched; + +case OP (RDMA_WRITE_FIRST ): +case OP (RDMA_WRITE_MIDDLE ): +if (opcode ==OP (RDMA_WRITE_MIDDLE ) || + opcode ==OP (RDMA_WRITE_LAST ) || + opcode ==OP (RDMA_WRITE_LAST_WITH_IMMEDIATE )) +break ; + goto nack_inv; + +case OP (RDMA_READ_REQUEST ): +case OP (COMPARE_SWAP ): +case OP (FETCH_ADD ): +/* +* Drop all new requests until a response has been sent. +* A new request then ACKs the RDMA response we sent. +* Relaxed ordering would allow new requests to be +* processed but we would need to keep a queue +* of rwqe's for all that are in progress. +* Note that we can't RNR NAK this request since the RDMA +* READ or atomic response is already queued to be sent +* (unless we implement a response send queue). +*/ + goto done; + +default : +if (opcode ==OP (SEND_MIDDLE ) || + opcode ==OP (SEND_LAST ) || + opcode ==OP (SEND_LAST_WITH_IMMEDIATE ) || + opcode ==OP (RDMA_WRITE_MIDDLE ) || + opcode ==OP (RDMA_WRITE_LAST ) || + opcode ==OP (RDMA_WRITE_LAST_WITH_IMMEDIATE )) + goto nack_inv; +break ; + } + + wc.imm_data =0 ; + wc.wc_flags =0 ; + +/* OK, process the packet. */ +switch (opcode) { +case OP (SEND_FIRST ): +if (!ipath_get_rwqe (qp,0 )) { + rnr_nak: +/* +* A RNR NAK will ACK earlier sends and RDMA writes. +* Don't queue the NAK if a RDMA read or atomic +* is pending though. +*/ +spin_lock (&qp->s_lock); +if (qp->s_ack_state >= +OP (RDMA_READ_REQUEST ) && + qp->s_ack_state !=IB_OPCODE_ACKNOWLEDGE ) { +spin_unlock (&qp->s_lock); + goto done; + } + qp->s_ack_state =OP (SEND_ONLY ); + qp->s_nak_state =IB_RNR_NAK | qp->s_min_rnr_timer; + qp->s_ack_psn = qp->r_psn; + goto resched; + } + qp->r_rcv_len =0 ; +/* FALLTHROUGH */ +case OP (SEND_MIDDLE ): +case OP (RDMA_WRITE_MIDDLE ): + send_middle: +/* Check for invalid length PMTU or posted rwqe len. */ +if (unlikely (tlen != (hdrsize + pmtu +4 ))) + goto nack_inv; + qp->r_rcv_len += pmtu; +if (unlikely (qp->r_rcv_len > qp->r_len)) + goto nack_inv; +ipath_copy_sge (&qp->r_sge, data, pmtu); +break ; + +case OP (RDMA_WRITE_LAST_WITH_IMMEDIATE ): +/* consume RWQE */ +if (!ipath_get_rwqe (qp,1 )) + goto rnr_nak; + goto send_last_imm; + +case OP (SEND_ONLY ): +case OP (SEND_ONLY_WITH_IMMEDIATE ): +if (!ipath_get_rwqe (qp,0 )) + goto rnr_nak; + qp->r_rcv_len =0 ; +if (opcode ==OP (SEND_ONLY )) + goto send_last; +/* FALLTHROUGH */ +case OP (SEND_LAST_WITH_IMMEDIATE ): + send_last_imm: +if (header_in_data) { + wc.imm_data = *(u32 *) data; + data +=sizeof (u32); + }else { +/* Immediate data comes after BTH */ + wc.imm_data = ohdr->u.imm_data; + } + hdrsize +=4 ; + wc.wc_flags =IB_WC_WITH_IMM ; +/* FALLTHROUGH */ +case OP (SEND_LAST ): +case OP (RDMA_WRITE_LAST ): + send_last: +/* Get the number of bytes the message was padded by. */ + pad = (be32_to_cpu (ohdr->bth[0 ]) >>20 ) &3 ; +/* Check for invalid length. */ +/* XXX LAST len should be >= 1 */ +if (unlikely (tlen < (hdrsize + pad +4 ))) + goto nack_inv; +/* Don't count the CRC. */ + tlen -= (hdrsize + pad +4 ); + wc.byte_len = tlen + qp->r_rcv_len; +if (unlikely (wc.byte_len > qp->r_len)) { + goto nack_inv; + } +/* XXX Need to free SGEs */ +ipath_copy_sge (&qp->r_sge, data, tlen); +atomic_inc (&qp->msn); +if (opcode ==OP (RDMA_WRITE_LAST ) || + opcode ==OP (RDMA_WRITE_ONLY )) +break ; + wc.wr_id = qp->r_wr_id; + wc.status =IB_WC_SUCCESS ; + wc.opcode =IB_WC_RECV ; + wc.vendor_err =0 ; + wc.qp_num = qp->ibqp.qp_num; + wc.src_qp = qp->remote_qpn; + wc.pkey_index =0 ; + wc.slid = qp->remote_ah_attr.dlid; + wc.sl = qp->remote_ah_attr.sl; + wc.dlid_path_bits =0 ; + wc.port_num =0 ; +/* Signal completion event if the solicited bit is set. */ +ipath_cq_enter (to_icq (qp->ibqp.recv_cq), &wc, + ohdr->bth[0 ] & +__constant_cpu_to_be32 (1 <<23 )); +break ; + +case OP (RDMA_WRITE_FIRST ): +case OP (RDMA_WRITE_ONLY ): +case OP (RDMA_WRITE_ONLY_WITH_IMMEDIATE ): +/* consume RWQE */ +/* RETH comes after BTH */ +if (!header_in_data) + reth = &ohdr->u.rc.reth; +else { + reth = (struct ib_reth *)data; + data +=sizeof (*reth); + } + hdrsize +=sizeof (*reth); + qp->r_len =be32_to_cpu (reth->length); + qp->r_rcv_len =0 ; +if (qp->r_len !=0 ) { +u32 rkey =be32_to_cpu (reth->rkey); +u64 vaddr =be64_to_cpu (reth->vaddr); + +/* Check rkey & NAK */ +if (unlikely (!ipath_rkey_ok ( + dev, &qp->r_sge, + qp->r_len, + vaddr, rkey, +IB_ACCESS_REMOTE_WRITE ))) { + nack_acc: +/* +* A NAK will ACK earlier sends and RDMA +* writes. Don't queue the NAK if a RDMA +* read, atomic, or NAK is pending though. +*/ +spin_lock (&qp->s_lock); +if (qp->s_ack_state >= +OP (RDMA_READ_REQUEST ) && + qp->s_ack_state != +IB_OPCODE_ACKNOWLEDGE ) { +spin_unlock (&qp->s_lock); + goto done; + } +/* XXX Flush WQEs */ + qp->state =IB_QPS_ERR ; + qp->s_ack_state =OP (RDMA_WRITE_ONLY ); + qp->s_nak_state = +IB_NAK_REMOTE_ACCESS_ERROR ; + qp->s_ack_psn = qp->r_psn; + goto resched; + } + }else { + qp->r_sge.sg_list =NULL ; + qp->r_sge.sge.mr =NULL ; + qp->r_sge.sge.vaddr =NULL ; + qp->r_sge.sge.length =0 ; + qp->r_sge.sge.sge_length =0 ; + } +if (unlikely (!(qp->qp_access_flags & +IB_ACCESS_REMOTE_WRITE ))) + goto nack_acc; +if (opcode ==OP (RDMA_WRITE_FIRST )) + goto send_middle; +else if (opcode ==OP (RDMA_WRITE_ONLY )) + goto send_last; +if (!ipath_get_rwqe (qp,1 )) + goto rnr_nak; + goto send_last_imm; + +case OP (RDMA_READ_REQUEST ): +/* RETH comes after BTH */ +if (!header_in_data) + reth = &ohdr->u.rc.reth; +else { + reth = (struct ib_reth *)data; + data +=sizeof (*reth); + } +spin_lock (&qp->s_lock); +if (qp->s_ack_state !=OP (ACKNOWLEDGE ) && + qp->s_ack_state >=IB_OPCODE_RDMA_READ_REQUEST ) { +spin_unlock (&qp->s_lock); + goto done; + } + qp->s_rdma_len =be32_to_cpu (reth->length); +if (qp->s_rdma_len !=0 ) { +u32 rkey =be32_to_cpu (reth->rkey); +u64 vaddr =be64_to_cpu (reth->vaddr); + +/* Check rkey & NAK */ +if (unlikely (!ipath_rkey_ok ( + dev, &qp->s_rdma_sge, + qp->s_rdma_len, + vaddr, rkey, +IB_ACCESS_REMOTE_READ ))) { +spin_unlock (&qp->s_lock); + goto nack_acc; + } +/* +* Update the next expected PSN. We add 1 later +* below, so only add the remainder here. +*/ +if (qp->s_rdma_len > pmtu) + qp->r_psn += (qp->s_rdma_len -1 ) / pmtu; + }else { + qp->s_rdma_sge.sg_list =NULL ; + qp->s_rdma_sge.num_sge =0 ; + qp->s_rdma_sge.sge.mr =NULL ; + qp->s_rdma_sge.sge.vaddr =NULL ; + qp->s_rdma_sge.sge.length =0 ; + qp->s_rdma_sge.sge.sge_length =0 ; + } +if (unlikely (!(qp->qp_access_flags & +IB_ACCESS_REMOTE_READ ))) + goto nack_acc; +/* +* We need to increment the MSN here instead of when we +* finish sending the result since a duplicate request would +* increment it more than once. +*/ +atomic_inc (&qp->msn); + qp->s_ack_state = opcode; + qp->s_nak_state =0 ; + qp->s_ack_psn = psn; + qp->r_psn++; + qp->r_state = opcode; + goto rdmadone; + +case OP (COMPARE_SWAP ): +case OP (FETCH_ADD ):{ +struct ib_atomic_eth *ateth; +u64 vaddr; +u64 sdata; +u32 rkey; + +if (!header_in_data) + ateth = &ohdr->u.atomic_eth; +else { + ateth = (struct ib_atomic_eth *)data; + data +=sizeof (*ateth); + } + vaddr =be64_to_cpu (ateth->vaddr); +if (unlikely (vaddr & (sizeof (u64) -1 ))) + goto nack_inv; + rkey =be32_to_cpu (ateth->rkey); +/* Check rkey & NAK */ +if (unlikely (!ipath_rkey_ok (dev, &qp->r_sge, +sizeof (u64), vaddr, rkey, +IB_ACCESS_REMOTE_ATOMIC ))) + goto nack_acc; +if (unlikely (!(qp->qp_access_flags & +IB_ACCESS_REMOTE_ATOMIC ))) + goto nack_acc; +/* Perform atomic OP and save result. */ + sdata =be64_to_cpu (ateth->swap_data); +spin_lock (&dev->pending_lock); + qp->r_atomic_data = *(u64 *) qp->r_sge.sge.vaddr; +if (opcode ==OP (FETCH_ADD )) + *(u64 *) qp->r_sge.sge.vaddr = + qp->r_atomic_data + sdata; +else if (qp->r_atomic_data == +be64_to_cpu (ateth->compare_data)) + *(u64 *) qp->r_sge.sge.vaddr = sdata; +spin_unlock (&dev->pending_lock); +atomic_inc (&qp->msn); + qp->r_atomic_psn = psn &IPS_PSN_MASK ; + psn |=1 <<31 ; +break ; + } + +default : +/* Drop packet for unknown opcodes. */ +spin_unlock_irqrestore (&qp->r_rq.lock, flags); +return ; + } + qp->r_psn++; + qp->r_state = opcode; +/* Send an ACK if requested or required. */ +if (psn & (1 <<31 )) { +/* +* Coalesce ACKs unless there is a RDMA READ or +* ATOMIC pending. +*/ +spin_lock (&qp->s_lock); +if (qp->s_ack_state ==OP (ACKNOWLEDGE ) || + qp->s_ack_state <IB_OPCODE_RDMA_READ_REQUEST ) { + qp->s_ack_state = opcode; + qp->s_nak_state =0 ; + qp->s_ack_psn = psn; + qp->s_ack_atomic = qp->r_atomic_data; + goto resched; + } +spin_unlock (&qp->s_lock); + } +done: +spin_unlock_irqrestore (&qp->r_rq.lock, flags); +return ; + +resched: +/* +* Try to send ACK right away but not if ipath_do_rc_send() is +* active. +*/ +if (qp->s_hdrwords ==0 && + (qp->s_ack_state <IB_OPCODE_RDMA_READ_REQUEST || + qp->s_ack_state >=IB_OPCODE_COMPARE_SWAP )) +send_rc_ack (qp); + +rdmadone: +spin_unlock (&qp->s_lock); +spin_unlock_irqrestore (&qp->r_rq.lock, flags); + +/* Call ipath_do_rc_send() in another thread. */ +tasklet_hi_schedule (&qp->s_task); +}