InfiniBand: updates for 2.6.11-rc1
Subsystems (2)
cover InfiniBand: updates for 2.6.11-rc1
Here are updates since the initial merge to drivers/infiniband taken from the OpenIB repository. This is a mix of cleanups, bug fixes and small features. There shouldn't be anything controversial or risky.
Thanks,
Roland
PATCH 1/18 InfiniBand/IPoIB: use correct static rate in IpoIB
Calculate static rate for IPoIB address handles based on local width/speed and path rate.
drivers/infiniband/include/ib_sa.h +28 -0
@@ -59,6 +59,34 @@ IB_SA_BEST = 3 }; +enum ib_sa_rate { +IB_SA_RATE_2_5_GBPS =2 , +IB_SA_RATE_5_GBPS =5 , +IB_SA_RATE_10_GBPS =3 , +IB_SA_RATE_20_GBPS =6 , +IB_SA_RATE_30_GBPS =4 , +IB_SA_RATE_40_GBPS =7 , +IB_SA_RATE_60_GBPS =8 , +IB_SA_RATE_80_GBPS =9 , +IB_SA_RATE_120_GBPS =10 +}; + +static inline int ib_sa_rate_enum_to_int (enum ib_sa_rate rate) +{ +switch (rate) { +case IB_SA_RATE_2_5_GBPS :return 1 ; +case IB_SA_RATE_5_GBPS :return 2 ; +case IB_SA_RATE_10_GBPS :return 4 ; +case IB_SA_RATE_20_GBPS :return 8 ; +case IB_SA_RATE_30_GBPS :return 12 ; +case IB_SA_RATE_40_GBPS :return 16 ; +case IB_SA_RATE_60_GBPS :return 24 ; +case IB_SA_RATE_80_GBPS :return 32 ; +case IB_SA_RATE_120_GBPS :return 48 ; +default :return -1 ; + } +} +typedef u64 __bitwise ib_sa_comp_mask ;#define IB_SA_COMP_MASK (n) ((__force ib_sa_comp_mask) cpu_to_be64(1ull << n))
drivers/infiniband/ulp/ipoib/ipoib_main.c +9 -9
@@ -283,21 +283,21 @@skb_queue_head_init (&skqueue);if (!status) { -/* -* For now we set static_rate to 0. This is not -* really correct: we should look at the rate -* component of the path member record, compare it -* with the rate of our local port (calculated from -* the active link speed and link width) and set an -* inter-packet delay appropriately. -*/ struct ib_ah_attr av = { .dlid =be16_to_cpu (pathrec->dlid), .sl = pathrec->sl, - .static_rate =0 , .port_num = priv->port }; +if (ib_sa_rate_enum_to_int (pathrec->rate) >0 ) + av.static_rate = (2 * priv->local_rate - +ib_sa_rate_enum_to_int (pathrec->rate) -1 ) / + (priv->local_rate ? priv->local_rate :1 ); + +ipoib_dbg (priv,"static_rate %d for local port %dX, path %dX\n" , + av.static_rate, priv->local_rate, +ib_sa_rate_enum_to_int (pathrec->rate)); + ah =ipoib_create_ah (dev, priv->pd, &av); }
drivers/infiniband/ulp/ipoib/ipoib_multicast.c +20 -18
@@ -238,19 +238,10 @@ } { -/* -* For now we set static_rate to 0. This is not -* really correct: we should look at the rate -* component of the MC member record, compare it with -* the rate of our local port (calculated from the -* active link speed and link width) and set an -* inter-packet delay appropriately. -*/ struct ib_ah_attr av = { .dlid =be16_to_cpu (mcast->mcmember.mlid), .port_num = priv->port, .sl = mcast->mcmember.sl, - .static_rate =0 , .ah_flags =IB_AH_GRH , .grh = { .flow_label =be32_to_cpu (mcast->mcmember.flow_label),
@@ -262,6 +253,15 @@ av.grh.dgid = mcast->mcmember.mgid; +if (ib_sa_rate_enum_to_int (mcast->mcmember.rate) >0 ) + av.static_rate = (2 * priv->local_rate - +ib_sa_rate_enum_to_int (mcast->mcmember.rate) -1 ) / + (priv->local_rate ? priv->local_rate :1 ); + +ipoib_dbg_mcast (priv,"static_rate %d for local port %dX, mcmember %dX\n" , + av.static_rate, priv->local_rate, +ib_sa_rate_enum_to_int (mcast->mcmember.rate)); + mcast->ah =ipoib_create_ah (dev, priv->pd, &av);if (!mcast->ah) {ipoib_warn (priv,"ib_address_create failed\n" );
@@ -506,6 +506,17 @@else memcpy (priv->dev->dev_addr +4 , priv->local_gid.raw,sizeof (union ib_gid )); + { +struct ib_port_attr attr; + +if (!ib_query_port (priv->ca, priv->port, &attr)) { + priv->local_lid = attr.lid; + priv->local_rate = attr.active_speed * +ib_width_enum_to_int (attr.active_width); + }else +ipoib_warn (priv,"ib_query_port failed\n" ); + } +if (!priv->broadcast) { priv->broadcast =ipoib_mcast_alloc (dev,1 );if (!priv->broadcast) {
@@ -554,15 +565,6 @@return ; } - { -struct ib_port_attr attr; - -if (!ib_query_port (priv->ca, priv->port, &attr)) - priv->local_lid = attr.lid; -else -ipoib_warn (priv,"ib_query_port failed\n" ); - } - priv->mcast_mtu =ib_mtu_enum_to_int (priv->broadcast->mcmember.mtu) -IPOIB_ENCAP_LEN ; dev->mtu =min (priv->mcast_mtu, priv->admin_mtu);
drivers/infiniband/ulp/ipoib/ipoib.h +1 -0
@@ -143,6 +143,7 @@union ib_gid local_gid;u16 local_lid; +u8 local_rate;unsigned int admin_mtu;unsigned int mcast_mtu;
PATCH 2/18 InfiniBand/mthca: trivial formatting fix
Trivial formatting fix for empty for loops.
drivers/infiniband/hw/mthca/mthca_mr.c +2 -2
@@ -197,7 +197,7 @@for (i = dev->limits.mtt_seg_size /8 , mr->order =0 ; i < list_len; i <<=1 , ++mr->order) -/* nothing */ ; + ;/* nothing */ mr->first_seg =mthca_alloc_mtt (dev, mr->order);if (mr->first_seg ==-1 )
@@ -337,7 +337,7 @@for (i =1 , dev->mr_table.max_mtt_order =0 ; i < dev->limits.num_mtt_segs; i <<=1 , ++dev->mr_table.max_mtt_order) -/* nothing */ ; + ;/* nothing */ dev->mr_table.mtt_buddy = kmalloc((dev->mr_table.max_mtt_order + 1) * sizeof (long *),
PATCH 3/18 InfiniBand/mthca: support RDMA/atomic attributes in QP modify
Implement setting of RDMA/atomic enable bits, initiator resources and responder resources for modify QP in low-level Mellanox HCA driver (should complete RDMA/atomic implementation).
drivers/infiniband/hw/mthca/mthca_dev.h +3 -1
@@ -75,6 +75,7 @@MTHCA_EQ_CONTEXT_SIZE =0x40 ,MTHCA_CQ_CONTEXT_SIZE =0x40 ,MTHCA_QP_CONTEXT_SIZE =0x200 , +MTHCA_RDB_ENTRY_SIZE =0x20 ,MTHCA_AV_SIZE =0x20 , MTHCA_MGM_ENTRY_SIZE = 0x40 };
@@ -121,7 +122,6 @@int mtt_seg_size;int reserved_mtts;int reserved_mrws; -int num_rdbs;int reserved_uars;int num_mgms;int num_amgms;
@@ -174,6 +174,8 @@struct mthca_qp_table {struct mthca_alloc alloc; +u32 rdb_base; +int rdb_shift;int sqp_start;spinlock_t lock;struct mthca_array qp;
drivers/infiniband/hw/mthca/mthca_provider.h +5 -2
@@ -162,9 +162,12 @@spinlock_t lock;atomic_t refcount;u32 qpn; -int transport; -enum ib_qp_state state;int is_direct; +u8 transport; +u8 state; +u8 atomic_rd_en; +u8 resp_depth; +struct mthca_mr mr;struct mthca_wq rq;
drivers/infiniband/hw/mthca/mthca_profile.c +7 -3
@@ -50,7 +50,6 @@ };enum { -MTHCA_RDB_ENTRY_SIZE =32 ,MTHCA_MTT_SEG_SIZE =64 };
@@ -181,8 +180,13 @@ init_hca->log_num_eqs = profile[i].log_num;break ;case MTHCA_RES_RDB : - dev->limits.num_rdbs = profile[i].num; - init_hca->rdb_base = profile[i].start; +for (dev->qp_table.rdb_shift =0 ; + profile[MTHCA_RES_QP ].num << dev->qp_table.rdb_shift < + profile[i].num; + ++dev->qp_table.rdb_shift) + ;/* nothing */ + dev->qp_table.rdb_base = (u32 ) profile[i].start; + init_hca->rdb_base = profile[i].start;break ;case MTHCA_RES_MCG : dev->limits.num_mgms = profile[i].num >>1 ;
drivers/infiniband/hw/mthca/mthca_qp.c +81 -4
115 lines not on this page · raw message
PATCH 4/18 InfiniBand/mthca: clean up allocation mapping of HCA context memory
Clean up the way we allocate and map memory for use as ICM ("InfiniHost Context Memory") when running in Arbel MemFree mode. This slightly improves the code for mapping the firmware area and will make future progress towards full MemFree support much easier.
drivers/infiniband/hw/mthca/mthca_dev.h +1 -2
@@ -40,7 +40,6 @@#include <linux/pci.h> #include <linux/dma-mapping.h> #include <asm/semaphore.h> -#include <asm/scatterlist.h> #include "mthca_provider.h" #include "mthca_doorbell.h"
@@ -214,7 +213,7 @@u64 clr_int_base;u64 eq_arm_base;u64 eq_set_ci_base; -struct scatterlist *mem; +struct mthca_icm *icm;u16 fw_pages; } arbel; } fw;
drivers/infiniband/hw/mthca/mthca_main.c +15 -75
146 lines not on this page · raw message
drivers/infiniband/hw/mthca/mthca_memfree.h new file +107 -0
107 lines not on this page · raw message
drivers/infiniband/hw/mthca/mthca_cmd.c +12 -11
@@ -40,6 +40,7 @@#include "mthca_dev.h" #include "mthca_config_reg.h" #include "mthca_cmd.h" +#include "mthca_memfree.h" #define CMD_POLL_TOKEN 0xffff
@@ -508,38 +509,38 @@return mthca_cmd (dev,0 ,0 ,0 ,CMD_SYS_DIS ,HZ , status); } -int mthca_MAP_FA (struct mthca_dev *dev,int count, -struct scatterlist *sglist,u8 *status) +int mthca_MAP_FA (struct mthca_dev *dev,struct mthca_icm *icm,u8 *status) {u32 *inbox;dma_addr_t indma; +struct mthca_icm_iter iter;int lg;int nent =0 ; -int i, j; +int i;int err =0 ;int ts =0 ; inbox =pci_alloc_consistent (dev->pdev,PAGE_SIZE , &indma);memset (inbox,0 ,PAGE_SIZE ); -for (i =0 ; i < count; ++i) { +for (mthca_icm_first (icm, &iter); !mthca_icm_last (&iter);mthca_icm_next (&iter)) {/* * We have to pass pages that are aligned to their * size, so find the least significant 1 in the * address or size and use that as our log2 size. */ - lg =ffs (sg_dma_address (sglist + i) |sg_dma_len (sglist + i)) -1 ; + lg =ffs (mthca_icm_addr (&iter) |mthca_icm_size (&iter)) -1 ;if (lg <12 ) { -mthca_warn (dev,"Got FW area not aligned to 4K (%llx/%x).\n" , - (unsigned long long )sg_dma_address (sglist + i), -sg_dma_len (sglist + i)); +mthca_warn (dev,"Got FW area not aligned to 4K (%llx/%lx).\n" , + (unsigned long long )mthca_icm_addr (&iter), +mthca_icm_size (&iter)); err = -EINVAL ; goto out; } -for (j =0 ; j <sg_dma_len (sglist + i) / (1 << lg); ++j, ++nent) { +for (i =0 ; i <mthca_icm_size (&iter) / (1 << lg); ++i, ++nent) { *((__be64 *) (inbox + nent *4 +2 )) = -cpu_to_be64 ((sg_dma_address (sglist + i) + - (j << lg)) | +cpu_to_be64 ((mthca_icm_addr (&iter) + + (i << lg)) | (lg -12 )); ts +=1 << (lg -10 );if (nent ==PAGE_SIZE /16 ) {
drivers/infiniband/hw/mthca/mthca_cmd.h +1 -2
@@ -219,8 +219,7 @@int mthca_SYS_EN (struct mthca_dev *dev,u8 *status);int mthca_SYS_DIS (struct mthca_dev *dev,u8 *status); -int mthca_MAP_FA (struct mthca_dev *dev,int count, -struct scatterlist *sglist,u8 *status); +int mthca_MAP_FA (struct mthca_dev *dev,struct mthca_icm *icm,u8 *status);int mthca_UNMAP_FA (struct mthca_dev *dev,u8 *status);int mthca_RUN_FW (struct mthca_dev *dev,u8 *status);int mthca_QUERY_FW (struct mthca_dev *dev,u8 *status);
drivers/infiniband/hw/mthca/Makefile +1 -1
@@ -9,4 +9,4 @@ ib_mthca-y := mthca_main.o mthca_cmd.o mthca_profile.o mthca_reset.o \ mthca_allocator.o mthca_eq.o mthca_pd.o mthca_cq.o \ mthca_mr.o mthca_qp.o mthca_av.o mthca_mcg.o mthca_mad.o \ - mthca_provider.o + mthca_provider.o mthca_memfree.o
drivers/infiniband/hw/mthca/mthca_memfree.c new file +133 -0
133 lines not on this page · raw message
PATCH 5/18 InfiniBand/mthca: add needed rmb() in event queue poll
Add an rmb() between checking the ownership bit of an event queue entry and reading the contents of the EQE. Without this barrier, the CPU could read stale contents of the EQE before HW writes the EQE but have the read of the ownership bit reordered until after HW finishes writing, which leads to the driver processing an incorrect event. This was actually observed to happen when multiple completion queues are in heavy use on an IBM JS20 PowerPC 970 system.
Also explain the existing rmb() in completion queue poll (there for the same reason) and slightly improve debugging output.
drivers/infiniband/hw/mthca/mthca_cq.c +7 -2
@@ -1,5 +1,5 @@/* -* Copyright (c) 2004 Topspin Communications. All rights reserved. +* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU
@@ -391,6 +391,10 @@if (!next_cqe_sw (cq))return -EAGAIN ; +/* +* Make sure we read CQ entry contents after we've checked the +* ownership bit. +*/ rmb (); cqe =get_cqe (cq, cq->cons_index);
@@ -768,7 +772,8 @@u32 *ctx =MAILBOX_ALIGN (mailbox);int j; -printk (KERN_ERR "context for CQN %x\n" , cq->cqn); +printk (KERN_ERR "context for CQN %x (cons index %x, next sw %d)\n" , + cq->cqn, cq->cons_index,next_cqe_sw (cq));for (j =0 ; j <16 ; ++j)printk (KERN_ERR "[%2x] %08x\n" , j *4 ,be32_to_cpu (ctx[j])); }
drivers/infiniband/hw/mthca/mthca_eq.c +7 -1
@@ -1,5 +1,5 @@/* -* Copyright (c) 2004 Topspin Communications. All rights reserved. +* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU
@@ -240,6 +240,12 @@int set_ci =0 ; eqe =get_eqe (eq, eq->cons_index); +/* +* Make sure we read EQ entry contents after we've +* checked the ownership bit. +*/ +rmb (); +switch (eqe->type) {case MTHCA_EVENT_TYPE_COMP : disarm_cqn =be32_to_cpu (eqe->event.comp.cqn) &0xffffff ;
PATCH 6/18 InfiniBand/core: remove debug printk
Remove debug printk accidentally included.
drivers/infiniband/core/sysfs.c +0 -2
@@ -188,8 +188,6 @@case 4 : speed =" QDR" ;break ; } -printk (KERN_ERR "width %d speed %d\n" , attr.active_width, attr.active_speed); - rate =25 *ib_width_enum_to_int (attr.active_width) * attr.active_speed;if (rate <0 )return -EINVAL ;
PATCH 7/18 InfiniBand: make more code static
Make needlessly global code static.
drivers/infiniband/ulp/ipoib/ipoib_main.c +1 -1
@@ -606,7 +606,7 @@return NETDEV_TX_OK ; } -struct net_device_stats *ipoib_get_stats (struct net_device *dev) +static struct net_device_stats *ipoib_get_stats (struct net_device *dev) {struct ipoib_dev_priv *priv =netdev_priv (dev);
drivers/infiniband/ulp/ipoib/ipoib_multicast.c +2 -2
@@ -44,7 +44,7 @@#include "ipoib.h" #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG -int mcast_debug_level; +static int mcast_debug_level;module_param (mcast_debug_level, int,0644 ); MODULE_PARM_DESC(mcast_debug_level,
@@ -623,7 +623,7 @@return 0 ; } -int ipoib_mcast_leave (struct net_device *dev,struct ipoib_mcast *mcast) +static int ipoib_mcast_leave (struct net_device *dev,struct ipoib_mcast *mcast) {struct ipoib_dev_priv *priv =netdev_priv (dev);struct ib_sa_mcmember_rec rec = {
drivers/infiniband/ulp/ipoib/ipoib_ib.c +1 -1
@@ -357,7 +357,7 @@ } } -void __ipoib_reap_ah (struct net_device *dev) +static void __ipoib_reap_ah (struct net_device *dev) {struct ipoib_dev_priv *priv =netdev_priv (dev);struct ipoib_ah *ah, *tah;
drivers/infiniband/core/cache.c +3 -3
@@ -252,7 +252,7 @@ } } -void ib_cache_setup_one (struct ib_device *device) +static void ib_cache_setup_one (struct ib_device *device) {int p;
@@ -295,7 +295,7 @@kfree (device->cache.gid_cache); } -void ib_cache_cleanup_one (struct ib_device *device) +static void ib_cache_cleanup_one (struct ib_device *device) {int p;
@@ -311,7 +311,7 @@kfree (device->cache.gid_cache); } -struct ib_client cache_client = { +static struct ib_client cache_client = { .name ="cache" , .add = ib_cache_setup_one, .remove = ib_cache_cleanup_one
PATCH 8/18 InfiniBand/core: set byte_cnt correctly in MAD completion
Integrate Michael Tsirkin's patch to local_completion to set the WC byte_cnt according to the IBA 1.1 spec (include the GRH size regardless of whether it is present or not).
drivers/infiniband/core/mad.c +3 -2
@@ -2018,9 +2018,10 @@ wc.status =IB_WC_SUCCESS ; wc.opcode =IB_WC_RECV ; wc.vendor_err =0 ; - wc.byte_len =sizeof (struct ib_mad ); + wc.byte_len =sizeof (struct ib_mad ) + +sizeof (struct ib_grh ); wc.src_qp =IB_QP0 ; - wc.wc_flags =0 ; + wc.wc_flags =0 ;/* No GRH */ wc.pkey_index =0 ; wc.slid =IB_LID_PERMISSIVE ; wc.sl =0 ;
PATCH 9/18 InfiniBand/core: add QP number to work completion struct
InfiniBand spec rev 1.2 compliance: add local qp number to work completion structure.
drivers/infiniband/include/ib_verbs.h +1 -0
@@ -352,6 +352,7 @@u32 vendor_err;u32 byte_len;__be32 imm_data; +u32 qp_num;u32 src_qp;int wc_flags;u16 pkey_index;
drivers/infiniband/core/mad.c +1 -0
@@ -2026,6 +2026,7 @@ wc.slid =IB_LID_PERMISSIVE ; wc.sl =0 ; wc.dlid_path_bits =0 ; + wc.qp_num =IB_QP0 ; local->mad_priv->header.recv_wc.wc = &wc; local->mad_priv->header.recv_wc.mad_len =sizeof (struct ib_mad );
drivers/infiniband/hw/mthca/mthca_cq.c +2 -0
@@ -444,6 +444,8 @@spin_lock (&(*cur_qp)->lock); } + entry->qp_num = (*cur_qp)->qpn; +if (is_send) { wq = &(*cur_qp)->sq; wqe_index = ((be32_to_cpu(cqe->wqe) - (*cur_qp)->send_wqe_offset)
PATCH 10/18 InfiniBand/core: add node_type and phys_state sysfs attrs
Add per-device "node_type" and per-port "phys_state" sysfs attributes for InfiniBand devices.
drivers/infiniband/include/ib_verbs.h +1 -0
@@ -212,6 +212,7 @@u8 init_type_reply;u8 active_width;u8 active_speed; +u8 phys_state; };enum ib_device_modify_flags {
drivers/infiniband/core/sysfs.c +39 -0
@@ -197,6 +197,29 @@ ib_width_enum_to_int(attr.active_width), speed); } +static ssize_t phys_state_show (struct ib_port *p,struct port_attribute *unused, +char *buf) +{ +struct ib_port_attr attr; + +ssize_t ret; + + ret =ib_query_port (p->ibdev, p->port_num, &attr); +if (ret) +return ret; + +switch (attr.phys_state) { +case 1 :return sprintf (buf,"1: Sleep\n" ); +case 2 :return sprintf (buf,"2: Polling\n" ); +case 3 :return sprintf (buf,"3: Disabled\n" ); +case 4 :return sprintf (buf,"4: PortConfigurationTraining\n" ); +case 5 :return sprintf (buf,"5: LinkUp\n" ); +case 6 :return sprintf (buf,"6: LinkErrorRecovery\n" ); +case 7 :return sprintf (buf,"7: Phy Test\n" ); +default :return sprintf (buf,"%d: <unknown>\n" , attr.phys_state); + } +} +static PORT_ATTR_RO (state);static PORT_ATTR_RO (lid);static PORT_ATTR_RO (lid_mask_count);
@@ -204,6 +227,7 @@static PORT_ATTR_RO (sm_sl);static PORT_ATTR_RO (cap_mask);static PORT_ATTR_RO (rate); +static PORT_ATTR_RO (phys_state);static struct attribute *port_default_attrs[] = { &port_attr_state.attr,
@@ -213,6 +237,7 @@ &port_attr_sm_sl.attr, &port_attr_cap_mask.attr, &port_attr_rate.attr, + &port_attr_phys_state.attr, NULL };
@@ -572,6 +597,18 @@return ret; } +static ssize_t show_node_type (struct class_device *cdev,char *buf) +{ + struct ib_device *dev = container_of(cdev, struct ib_device, class_dev); + +switch (dev->node_type) { +case IB_NODE_CA :return sprintf (buf,"%d: CA\n" , dev->node_type); +case IB_NODE_SWITCH :return sprintf (buf,"%d: switch\n" , dev->node_type); +case IB_NODE_ROUTER :return sprintf (buf,"%d: router\n" , dev->node_type); +default :return sprintf (buf,"%d: <unknown>\n" , dev->node_type); + } +} +static ssize_t show_sys_image_guid (struct class_device *cdev,char *buf) { struct ib_device *dev = container_of(cdev, struct ib_device, class_dev);
@@ -606,10 +643,12 @@ be16_to_cpu(((u16 *) &attr.node_guid)[3])); } +static CLASS_DEVICE_ATTR(node_type, S_IRUGO, show_node_type, NULL); static CLASS_DEVICE_ATTR(sys_image_guid, S_IRUGO, show_sys_image_guid, NULL); static CLASS_DEVICE_ATTR(node_guid, S_IRUGO, show_node_guid, NULL);static struct class_device_attribute *ib_class_attributes[] = { + &class_device_attr_node_type, &class_device_attr_sys_image_guid, &class_device_attr_node_guid };
drivers/infiniband/hw/mthca/mthca_provider.c +1 -0
@@ -119,6 +119,7 @@ props->sm_lid =be16_to_cpup ((u16 *) (out_mad->data +18 )); props->sm_sl = out_mad->data[36 ] &0xf ; props->state = out_mad->data[32 ] &0xf ; + props->phys_state = out_mad->data[33 ] >>4 ; props->port_cap_flags =be32_to_cpup ((u32 *) (out_mad->data +20 )); props->gid_tbl_len =to_mdev (ibdev)->limits.gid_table_len; props->pkey_tbl_len =to_mdev (ibdev)->limits.pkey_table_len;
Documentation/infiniband/sysfs.txt +2 -0
@@ -3,6 +3,7 @@ For each InfiniBand device, the InfiniBand drivers create the following files under /sys/class/linux/drivers/infiniband/<device name>: + node_type - Node type (CA, switch or router) node_guid - Node GUID sys_image_guid - System image GUID
@@ -25,6 +26,7 @@ sm_lid - Subnet manager LID for port's subnet sm_sl - Subnet manager SL for port's subnet state - Port state (DOWN, INIT, ARMED, ACTIVE or ACTIVE_DEFER) + phys_state - Port physical state (Sleep, Polling, LinkUp, etc) There is also a "counters" subdirectory, with files
PATCH 11/18 InfiniBand/mthca: clean up computation of HCA memory map
Clean up the computation of the HCA context memory map. This serves two purposes:
- make it easier to change the HCA "profile" (eg add more QPs)
- make it easier to implement mem-free Arbel support
drivers/infiniband/hw/mthca/mthca_dev.h +5 -2
@@ -70,13 +70,16 @@ };enum { -MTHCA_MPT_ENTRY_SIZE =0x40 ,MTHCA_EQ_CONTEXT_SIZE =0x40 ,MTHCA_CQ_CONTEXT_SIZE =0x40 ,MTHCA_QP_CONTEXT_SIZE =0x200 ,MTHCA_RDB_ENTRY_SIZE =0x20 ,MTHCA_AV_SIZE =0x20 , -MTHCA_MGM_ENTRY_SIZE =0x40 +MTHCA_MGM_ENTRY_SIZE =0x40 , + +/* Arbel FW gives us these, but we need them for Tavor */ +MTHCA_MPT_ENTRY_SIZE =0x40 , +MTHCA_MTT_SEG_SIZE =0x40 , };enum {
drivers/infiniband/hw/mthca/mthca_main.c +20 -1
@@ -76,6 +76,20 @@"ib_mthca: Mellanox InfiniBand HCA driver v" DRV_VERSION " (" DRV_RELDATE ")\n" ; +static struct mthca_profile default_profile = { + .num_qp =1 <<16 , + .rdb_per_qp =4 , + .num_cq =1 <<16 , + .num_mcg =1 <<13 , + .num_mpt =1 <<17 , + .num_mtt =1 <<20 +}; + +enum { +MTHCA_TAVOR_NUM_UDAV =1 <<15 , +MTHCA_ARBEL_UARC_SIZE =1 <<18 +}; +static int __devinit mthca_tune_pci (struct mthca_dev *mdev) {int cap;
@@ -175,6 +189,7 @@u8 status;int err;struct mthca_dev_lim dev_lim; +struct mthca_profile profile;struct mthca_init_hca_param init_hca;struct mthca_adapter adapter;
@@ -214,7 +229,11 @@ err =mthca_dev_lim (mdev, &dev_lim); - err =mthca_make_profile (mdev, &dev_lim, &init_hca); + profile = default_profile; + profile.num_uar = dev_lim.uar_size /PAGE_SIZE ; + profile.num_udav =MTHCA_TAVOR_NUM_UDAV ; + + err =mthca_make_profile (mdev, &profile, &dev_lim, &init_hca);if (err) goto err_out_disable;
drivers/infiniband/hw/mthca/mthca_profile.c +58 -33
160 lines not on this page · raw message
drivers/infiniband/hw/mthca/mthca_cmd.c +5 -3
@@ -1,5 +1,5 @@/* -* Copyright (c) 2004 Topspin Communications. All rights reserved. +* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU
@@ -959,9 +959,9 @@MTHCA_GET (field, outbox,QUERY_DEV_LIM_RSZ_SRQ_OFFSET ); dev_lim->hca.arbel.resize_srq = field &1 ;MTHCA_GET (size, outbox,QUERY_DEV_LIM_MTT_ENTRY_SZ_OFFSET ); - dev_lim->hca.arbel.mtt_entry_sz = size; + dev_lim->mtt_seg_sz = size;MTHCA_GET (size, outbox,QUERY_DEV_LIM_MPT_ENTRY_SZ_OFFSET ); - dev_lim->hca.arbel.mpt_entry_sz = size; + dev_lim->mpt_entry_sz = size;MTHCA_GET (field, outbox,QUERY_DEV_LIM_PBL_SZ_OFFSET ); dev_lim->hca.arbel.max_pbl_sz =1 << (field &0x3f ); MTHCA_GET(dev_lim->hca.arbel.bmme_flags, outbox,
@@ -987,6 +987,8 @@ }else {MTHCA_GET (field, outbox,QUERY_DEV_LIM_MAX_AV_OFFSET ); dev_lim->hca.tavor.max_avs =1 << (field &0x3f ); + dev_lim->mtt_seg_sz =MTHCA_MTT_SEG_SIZE ; + dev_lim->mpt_entry_sz =MTHCA_MPT_ENTRY_SIZE ; } out:
drivers/infiniband/hw/mthca/mthca_profile.h +11 -15
@@ -38,24 +38,20 @@#include "mthca_dev.h" #include "mthca_cmd.h" -enum { -MTHCA_RES_QP , -MTHCA_RES_EEC , -MTHCA_RES_SRQ , -MTHCA_RES_CQ , -MTHCA_RES_EQP , -MTHCA_RES_EEEC , -MTHCA_RES_EQ , -MTHCA_RES_RDB , -MTHCA_RES_MCG , -MTHCA_RES_MPT , -MTHCA_RES_MTT , -MTHCA_RES_UAR , -MTHCA_RES_UDAV , -MTHCA_RES_NUM +struct mthca_profile { +int num_qp; +int rdb_per_qp; +int num_cq; +int num_mcg; +int num_mpt; +int num_mtt; +int num_udav; +int num_uar; +int uarc_size; };int mthca_make_profile (struct mthca_dev *mdev, +struct mthca_profile *request,struct mthca_dev_lim *dev_lim,struct mthca_init_hca_param *init_hca);
drivers/infiniband/hw/mthca/mthca_cmd.h +3 -3
@@ -1,5 +1,5 @@/* -* Copyright (c) 2004 Topspin Communications. All rights reserved. +* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU
@@ -148,14 +148,14 @@int cqc_entry_sz;int srq_entry_sz;int uar_scratch_entry_sz; +int mtt_seg_sz; +int mpt_entry_sz;union {struct {int max_avs; } tavor;struct {int resize_srq; -int mtt_entry_sz; -int mpt_entry_sz;int max_pbl_sz;u8 bmme_flags;u32 reserved_lkey;
PATCH 12/18 InfiniBand/core: fix handling of 0-hop directed route MADs
Handle outgoing DR 0 hop SMPs properly when provider returns just SUCCESS to process_mad.
drivers/infiniband/core/mad.c +52 -12
@@ -60,6 +60,9 @@static int method_in_use (struct ib_mad_mgmt_method_table **method,struct ib_mad_reg_req *mad_reg_req);static void remove_mad_reg_req (struct ib_mad_agent_private *priv); +static struct ib_mad_agent_private *find_mad_agent ( +struct ib_mad_port_private *port_priv, +struct ib_mad *mad,int solicited);static int ib_mad_post_receive_mads (struct ib_mad_qp_info *qp_info,struct ib_mad_private *mad);static void cancel_mads (struct ib_mad_agent_private *mad_agent_priv);
@@ -623,10 +626,12 @@struct ib_smp *smp, struct ib_send_wr *send_wr) { - int ret, alloc_flags; + int ret, alloc_flags, solicited;unsigned long flags;struct ib_mad_local_private *local;struct ib_mad_private *mad_priv; +struct ib_mad_port_private *port_priv; +struct ib_mad_agent_private *recv_mad_agent =NULL ;struct ib_device *device = mad_agent_priv->agent.device;u8 port_num = mad_agent_priv->agent.port_num;
@@ -651,6 +656,7 @@ goto out; } local->mad_priv =NULL ; + local->recv_mad_agent =NULL ; mad_priv =kmem_cache_alloc (ib_mad_cache, alloc_flags);if (!mad_priv) { ret = -ENOMEM ;
@@ -669,19 +675,41 @@* there is a recv handler */ if (solicited_mad (&mad_priv->mad.mad) && - mad_agent_priv->agent.recv_handler) + mad_agent_priv->agent.recv_handler) { local->mad_priv = mad_priv; -else + local->recv_mad_agent = mad_agent_priv; +/* +* Reference MAD agent until receive +* side of local completion handled +*/ +atomic_inc (&mad_agent_priv->refcount); + }else kmem_cache_free (ib_mad_cache, mad_priv);break ;case IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_CONSUMED :kmem_cache_free (ib_mad_cache, mad_priv);break ;case IB_MAD_RESULT_SUCCESS : -kmem_cache_free (ib_mad_cache, mad_priv); -kfree (local); - ret =0 ; - goto out; +/* Treat like an incoming receive MAD */ + solicited =solicited_mad (&mad_priv->mad.mad); + port_priv =ib_get_mad_port (mad_agent_priv->agent.device, + mad_agent_priv->agent.port_num); +if (port_priv) { + mad_priv->mad.mad.mad_hdr.tid = + ((struct ib_mad *)smp)->mad_hdr.tid; + recv_mad_agent =find_mad_agent (port_priv, + &mad_priv->mad.mad, + solicited); + } +if (!port_priv || !recv_mad_agent) { +kmem_cache_free (ib_mad_cache, mad_priv); +kfree (local); + ret =0 ; + goto out; + } + local->mad_priv = mad_priv; + local->recv_mad_agent = recv_mad_agent; +break ;default :kmem_cache_free (ib_mad_cache, mad_priv);kfree (local);
@@ -696,7 +724,7 @@ local->send_wr.next =NULL ; local->tid = send_wr->wr.ud.mad_hdr->tid; local->wr_id = send_wr->wr_id; -/* Reference MAD agent until local completion handled */ +/* Reference MAD agent until send side of local completion handled */ atomic_inc (&mad_agent_priv->refcount);/* Queue local completion to local list */ spin_lock_irqsave (&mad_agent_priv->lock, flags);
@@ -1997,6 +2025,7 @@ {struct ib_mad_agent_private *mad_agent_priv;struct ib_mad_local_private *local; +struct ib_mad_agent_private *recv_mad_agent;unsigned long flags;struct ib_wc wc;struct ib_mad_send_wc mad_send_wc;
@@ -2010,6 +2039,13 @@ completion_list);spin_unlock_irqrestore (&mad_agent_priv->lock, flags);if (local->mad_priv) { + recv_mad_agent = local->recv_mad_agent; +if (!recv_mad_agent) { + printk(KERN_ERR PFX "No receive MAD agent for local completion\n"); +kmem_cache_free (ib_mad_cache, local->mad_priv); + goto local_send_completion; + } +/* * Defined behavior is to complete response * before request
@@ -2034,15 +2070,19 @@ local->mad_priv->header.recv_wc.recv_buf.grh =NULL ; local->mad_priv->header.recv_wc.recv_buf.mad = &local->mad_priv->mad.mad; -if (atomic_read (&mad_agent_priv->qp_info->snoop_count)) -snoop_recv (mad_agent_priv->qp_info, +if (atomic_read (&recv_mad_agent->qp_info->snoop_count)) +snoop_recv (recv_mad_agent->qp_info, &local->mad_priv->header.recv_wc,IB_MAD_SNOOP_RECVS ); - mad_agent_priv->agent.recv_handler ( - &mad_agent_priv->agent, + recv_mad_agent->agent.recv_handler ( + &recv_mad_agent->agent, &local->mad_priv->header.recv_wc); +spin_lock_irqsave (&recv_mad_agent->lock, flags); +atomic_dec (&recv_mad_agent->refcount); +spin_unlock_irqrestore (&recv_mad_agent->lock, flags); } +local_send_completion:/* Complete send */ mad_send_wc.status =IB_WC_SUCCESS ; mad_send_wc.vendor_err =0 ;
drivers/infiniband/core/mad_priv.h +1 -0
@@ -127,6 +127,7 @@struct ib_mad_local_private {struct list_head completion_list;struct ib_mad_private *mad_priv; +struct ib_mad_agent_private *recv_mad_agent;struct ib_send_wr send_wr;struct ib_sge sg_list[IB_MAD_SEND_REQ_MAX_SG ];u64 wr_id;/* client WR ID */
PATCH 13/18 InfiniBand/core: add more parameters to process_mad
Add parameters to process_mad device method to support full Mellanox firmware capabilities (pass sufficient information for baseboard management trap generation, etc).
drivers/infiniband/include/ib_verbs.h +6 -2
@@ -684,9 +684,12 @@ };struct ib_mad ; +struct ib_grh ;enum ib_process_mad_flags { -IB_MAD_IGNORE_MKEY =1 +IB_MAD_IGNORE_MKEY =1 , +IB_MAD_IGNORE_BKEY =2 , +IB_MAD_IGNORE_ALL =IB_MAD_IGNORE_MKEY |IB_MAD_IGNORE_BKEY };enum ib_mad_result {
@@ -812,7 +815,8 @@int (*process_mad)(struct ib_device *device,int process_mad_flags,u8 port_num, -u16 source_lid, +struct ib_wc *in_wc, +struct ib_grh *in_grh,struct ib_mad *in_mad,struct ib_mad *out_mad);
drivers/infiniband/core/mad.c +29 -15
74 lines not on this page · raw message
drivers/infiniband/core/sysfs.c +2 -2
@@ -315,8 +315,8 @@ in_mad->data[41 ] = p->port_num;/* PortSelect field */ -if ((p->ibdev->process_mad (p->ibdev,IB_MAD_IGNORE_MKEY , p->port_num,0xffff , - in_mad, out_mad) & +if ((p->ibdev->process_mad (p->ibdev,IB_MAD_IGNORE_MKEY , + p->port_num,NULL ,NULL , in_mad, out_mad) & (IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY )) != (IB_MAD_RESULT_SUCCESS |IB_MAD_RESULT_REPLY )) { ret = -EINVAL ;
drivers/infiniband/hw/mthca/mthca_dev.h +2 -1
@@ -381,7 +381,8 @@int mthca_process_mad (struct ib_device *ibdev,int mad_flags,u8 port_num, -u16 slid, +struct ib_wc *in_wc, +struct ib_grh *in_grh,struct ib_mad *in_mad,struct ib_mad *out_mad);int mthca_create_agents (struct mthca_dev *dev);
drivers/infiniband/hw/mthca/mthca_mad.c +3 -1
@@ -185,12 +185,14 @@int mthca_process_mad (struct ib_device *ibdev,int mad_flags,u8 port_num, -u16 slid, +struct ib_wc *in_wc, +struct ib_grh *in_grh,struct ib_mad *in_mad,struct ib_mad *out_mad) {int err;u8 status; +u16 slid = in_wc ? in_wc->slid :IB_LID_PERMISSIVE ;/* Forward locally generated traps to the SM */ if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP &&
PATCH 14/18 InfiniBand/core: add qp_type to struct ib_qp
Add qp_type to struct ib_qp.
Signed-off by: Sean Hefty <[email protected]>
drivers/infiniband/include/ib_verbs.h +1 -0
@@ -659,6 +659,7 @@void (*event_handler)(struct ib_event *,void *);void *qp_context;u32 qp_num; +enum ib_qp_type qp_type; };struct ib_mr {
drivers/infiniband/core/verbs.c +1 -0
@@ -132,6 +132,7 @@ qp->srq = qp_init_attr->srq; qp->event_handler = qp_init_attr->event_handler; qp->qp_context = qp_init_attr->qp_context; + qp->qp_type = qp_init_attr->qp_type;atomic_inc (&pd->usecnt);atomic_inc (&qp_init_attr->send_cq->usecnt);atomic_inc (&qp_init_attr->recv_cq->usecnt);
PATCH 15/18 InfiniBand/core: add ib_find_cached_gid function
Add a new function to find a port on a device given a GID by searching the cached GID tables. Document all cache functions in ib_cache.h. Rename existing functions to better match format of verb routines.
Signed-off by: Sean Hefty <[email protected]>
drivers/infiniband/ulp/ipoib/ipoib_verbs.c +2 -2
@@ -49,7 +49,7 @@if (!qp_attr) goto out; -if (ib_cached_pkey_find (priv->ca, priv->port, priv->pkey, &pkey_index)) { +if (ib_find_cached_pkey (priv->ca, priv->port, priv->pkey, &pkey_index)) {clear_bit (IPOIB_PKEY_ASSIGNED , &priv->flags); ret = -ENXIO ; goto out;
@@ -104,7 +104,7 @@* The port has to be assigned to the respective IB partition in * advance. */ - ret =ib_cached_pkey_find (priv->ca, priv->port, priv->pkey, &pkey_index); + ret =ib_find_cached_pkey (priv->ca, priv->port, priv->pkey, &pkey_index);if (ret) {clear_bit (IPOIB_PKEY_ASSIGNED , &priv->flags);return ret;
drivers/infiniband/ulp/ipoib/ipoib_ib.c +1 -1
@@ -630,7 +630,7 @@struct ipoib_dev_priv *priv =netdev_priv (dev);u16 pkey_index =0 ; -if (ib_cached_pkey_find (priv->ca, priv->port, priv->pkey, &pkey_index)) +if (ib_find_cached_pkey (priv->ca, priv->port, priv->pkey, &pkey_index))clear_bit (IPOIB_PKEY_ASSIGNED , &priv->flags);else set_bit (IPOIB_PKEY_ASSIGNED , &priv->flags);
drivers/infiniband/include/ib_cache.h +56 -6
72 lines not on this page · raw message
drivers/infiniband/core/cache.c +50 -15
121 lines not on this page · raw message
drivers/infiniband/hw/mthca/mthca_av.c +1 -1
@@ -159,7 +159,7 @@ (be32_to_cpu (ah->av->sl_tclass_flowlabel) >>20 ) &0xff ; header->grh.flow_label = ah->av->sl_tclass_flowlabel &cpu_to_be32 (0xfffff ); -ib_cached_gid_get (&dev->ib_dev, +ib_get_cached_gid (&dev->ib_dev,be32_to_cpu (ah->av->port_pd) >>24 , ah->av->gid_index, &header->grh.source_gid);
drivers/infiniband/hw/mthca/mthca_qp.c +2 -2
@@ -1190,11 +1190,11 @@ sqp->ud_header.lrh.source_lid =0xffff ; sqp->ud_header.bth.solicited_event = !!(wr->send_flags &IB_SEND_SOLICITED );if (!sqp->qp.ibqp.qp_num) -ib_cached_pkey_get (&dev->ib_dev, sqp->port, +ib_get_cached_pkey (&dev->ib_dev, sqp->port, sqp->pkey_index, &sqp->ud_header.bth.pkey);else -ib_cached_pkey_get (&dev->ib_dev, sqp->port, +ib_get_cached_pkey (&dev->ib_dev, sqp->port, wr->wr.ud.pkey_index, &sqp->ud_header.bth.pkey);cpu_to_be16s (&sqp->ud_header.bth.pkey);
PATCH 16/18 InfiniBand: update copyrights for new year
Update copyright line (files were modified in 2005).
drivers/infiniband/core/mad.c +1 -1
@@ -1,5 +1,5 @@/* -* Copyright (c) 2004, Voltaire, Inc. All rights reserved. +* Copyright (c) 2004, 2005 Voltaire, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU
drivers/infiniband/core/mad_priv.h +1 -1
@@ -1,5 +1,5 @@/* -* Copyright (c) 2004, Voltaire, Inc. All rights reserved. +* Copyright (c) 2004, 2005, Voltaire, Inc. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU
drivers/infiniband/core/sysfs.c +1 -1
@@ -1,5 +1,5 @@/* -* Copyright (c) 2004 Topspin Communications. All rights reserved. +* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU
drivers/infiniband/hw/mthca/mthca_provider.c +1 -1
@@ -1,5 +1,5 @@/* -* Copyright (c) 2004 Topspin Communications. All rights reserved. +* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU
PATCH 17/18 InfiniBand/ipoib: move structs from stack to device private struct
Move the gather list and work request used for posting sends from the stack in ipoib_send() to the private structure. This reduces the stack usage for the data path function ipoib_send() and may speed things up slightly because we don't need to initialize constant members of the structures.
drivers/infiniband/ulp/ipoib/ipoib_verbs.c +9 -2
@@ -1,5 +1,5 @@/* -* Copyright (c) 2004 Topspin Communications. All rights reserved. +* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU
@@ -187,7 +187,7 @@ priv->mr =ib_get_dma_mr (priv->pd,IB_ACCESS_LOCAL_WRITE );if (IS_ERR (priv->mr)) { -printk (KERN_WARNING "%s: ib_reg_phys_mr failed\n" , ca->name); +printk (KERN_WARNING "%s: ib_get_dma_mr failed\n" , ca->name); goto out_free_cq; }
@@ -204,6 +204,13 @@ priv->dev->dev_addr[2 ] = (priv->qp->qp_num >>8 ) &0xff ; priv->dev->dev_addr[3 ] = (priv->qp->qp_num ) &0xff ; + priv->tx_sge.lkey = priv->mr->lkey; + + priv->tx_wr.opcode =IB_WR_SEND ; + priv->tx_wr.sg_list = &priv->tx_sge; + priv->tx_wr.num_sge =1 ; + priv->tx_wr.send_flags =IB_SEND_SIGNALED ; +return 0 ; out_free_mr:
drivers/infiniband/ulp/ipoib/ipoib.h +6 -4
@@ -1,5 +1,5 @@/* -* Copyright (c) 2004 Topspin Communications. All rights reserved. +* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU
@@ -150,10 +150,12 @@struct ipoib_buf *rx_ring; -spinlock_t tx_lock; +spinlock_t tx_lock;struct ipoib_buf *tx_ring; -unsigned tx_head; -unsigned tx_tail; +unsigned tx_head; +unsigned tx_tail; +struct ib_sge tx_sge; +struct ib_send_wr tx_wr;struct ib_wc ibwc[IPOIB_NUM_WC ];
drivers/infiniband/ulp/ipoib/ipoib_multicast.c +4 -2
@@ -1,5 +1,5 @@/* -* Copyright (c) 2004 Topspin Communications. All rights reserved. +* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU
@@ -213,8 +213,10 @@/* Set the cached Q_Key before we attach if it's the broadcast group */ if (!memcmp (mcast->mcmember.mgid.raw, priv->dev->broadcast +4 , -sizeof (union ib_gid ))) +sizeof (union ib_gid ))) { priv->qkey =be32_to_cpu (priv->broadcast->mcmember.qkey); + priv->tx_wr.wr.ud.remote_qkey = priv->qkey; + }if (!test_bit (IPOIB_MCAST_FLAG_SENDONLY , &mcast->flags)) {if (test_and_set_bit (IPOIB_MCAST_FLAG_ATTACHED , &mcast->flags)) {
drivers/infiniband/ulp/ipoib/ipoib_ib.c +9 -21
@@ -1,5 +1,5 @@/* -* Copyright (c) 2004 Topspin Communications. All rights reserved. +* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU
@@ -281,28 +281,16 @@ struct ib_ah *address, u32 qpn, dma_addr_t addr, int len) { - struct ib_sge list = { - .addr = addr, - .length = len, - .lkey = priv->mr->lkey, - }; -struct ib_send_wr param = { - .wr_id = wr_id, - .opcode =IB_WR_SEND , - .sg_list = &list, - .num_sge =1 , - .wr = { - .ud = { - .remote_qpn = qpn, - .remote_qkey = priv->qkey, - .ah = address - }, - }, - .send_flags =IB_SEND_SIGNALED , - }; struct ib_send_wr *bad_wr; -return ib_post_send (priv->qp, ¶m, &bad_wr); + priv->tx_sge.addr = addr; + priv->tx_sge.length = len; + + priv->tx_wr.wr_id = wr_id; + priv->tx_wr.wr.ud.remote_qpn = qpn; + priv->tx_wr.wr.ud.ah = address; + +return ib_post_send (priv->qp, &priv->tx_wr, &bad_wr); } void ipoib_send(struct net_device *dev, struct sk_buff *skb,
PATCH 18/18 InfiniBand/core: rename handle_outgoing_smp
Change routine name from handle_outgoing_smp to handle_outgoing_dr_smp.
drivers/infiniband/core/mad.c +5 -4
@@ -619,9 +619,9 @@* Return 1 if SMP was consumed locally (whether or not solicited) * Return < 0 if error */ -static int handle_outgoing_smp (struct ib_mad_agent_private *mad_agent_priv, -struct ib_smp *smp, -struct ib_send_wr *send_wr) +static int handle_outgoing_dr_smp (struct ib_mad_agent_private *mad_agent_priv, +struct ib_smp *smp, +struct ib_send_wr *send_wr) {int ret, alloc_flags;unsigned long flags;
@@ -797,7 +797,8 @@ smp = (struct ib_smp *)send_wr->wr.ud.mad_hdr;if (smp->mgmt_class ==IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE ) { - ret =handle_outgoing_smp (mad_agent_priv, smp, send_wr); + ret =handle_outgoing_dr_smp (mad_agent_priv, smp, + send_wr);if (ret <0 )/* error */ goto error2;else if (ret ==1 )/* locally consumed */
Re: [openib-general] [PATCH][5/18] InfiniBand/mthca: add needed rmb() in event queue poll
In reply to Roland DreierHello!
Quoting r. Roland Dreier ([email protected]) "[openib-general] [PATCH][5/18] InfiniBand/mthca: add needed rmb() in event queue poll":
Quoted 55 lines
Add an
rmb()between checking the ownership bit of an event queue entry and reading the contents of the EQE. Without this barrier, the CPU could read stale contents of the EQE before HW writes the EQE but have the read of the ownership bit reordered until after HW finishes writing, which leads to the driver processing an incorrect event. This was actually observed to happen when multiple completion queues are in heavy use on an IBM JS20 PowerPC 970 system.Also explain the existing
rmb()in completion queue poll (there for the same reason) and slightly improve debugging output.Signed-off-by Roland Dreier <[email protected]>drivers/infiniband/hw/mthca/mthca_cq.c +7
-2@@ -1,5 +1,5 @@/* -* Copyright (c) 2004 Topspin Communications. All rights reserved. +* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -391,6 +391,10 @@if (!next_cqe_sw (cq))return -EAGAIN ; +/* +* Make sure we read CQ entry contents after we've checked the +* ownership bit. +*/ rmb (); cqe =get_cqe (cq, cq->cons_index);@@ -768,7 +772,8 @@u32 *ctx =MAILBOX_ALIGN (mailbox);int j; -printk (KERN_ERR "context for CQN %x\n" , cq->cqn); +printk (KERN_ERR "context for CQN %x (cons index %x, next sw %d)\n" , + cq->cqn, cq->cons_index,next_cqe_sw (cq));for (j =0 ; j <16 ; ++j)printk (KERN_ERR "[%2x] %08x\n" , j *4 ,be32_to_cpu (ctx[j])); }drivers/infiniband/hw/mthca/mthca_eq.c +1
-1@@ -1,5 +1,5 @@/* -* Copyright (c) 2004 Topspin Communications. All rights reserved. +* Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -240,6 +240,12 @@int set_ci =0 ; eqe =get_eqe (eq, eq->cons_index);
drivers/infiniband/hw/mthca/mthca_eq.c line 251
Quoted 10 lines
drivers/infiniband/hw/mthca/mthca_eq.c +6
-0+/* +* Make sure we read EQ entry contents after we've +* checked the ownership bit. +*/ +rmb (); +switch (eqe->type) {case MTHCA_EVENT_TYPE_COMP : disarm_cqn =be32_to_cpu (eqe->event.comp.cqn) &0xffffff ;
Since we are using the eqe here, it seems that read_barrier_depends shall be sufficient (as well as in the cq case)?
However, I see that read_barrier_depends is a nop on ppc, and the comment indicates that problems were seen on ppc 970. What gives? do I misunderstand what a dependency is?
MST
Re: [openib-general] [PATCH][5/18] InfiniBand/mthca: add needed rmb() in event queue poll
Michael> Since we are using the eqe here, it seems that Michael> read_barrier_depends shall be sufficient (as well as in Michael> the cq case)?
Michael> However, I see that read_barrier_depends is a nop on ppc, Michael> and the comment indicates that problems were seen on ppc Michael> 970. What gives? do I misunderstand what a dependency Michael> is?
There is no dependency between the EQE ownership field and the rest of the EQE, so read_barrier_depends() is not sufficient. I think you are misunderstanding what a dependency is. The comments in asm-i386/system.h or http://lse.sourceforge.net/locking/wmbdd.html may help clear things up.
- R.