InfiniBand: updates for 2.6.11-rc2

@@ -43,7 +43,6 @@
 #include <linux/poll.h>
 #include <linux/rwsem.h>
 #include <linux/kref.h>
-#include <linux/ioctl32.h>
 
 #include <asm/uaccess.h>
 

@@ -502,14 +501,14 @@
 }
 
 static struct file_operations umad_fops = {
-	.owner 	 = THIS_MODULE,
-	.read 	 = ib_umad_read,
-	.write 	 = ib_umad_write,
-	.poll 	 = ib_umad_poll,
+	.owner 	        = THIS_MODULE,
+	.read 	        = ib_umad_read,
+	.write 	        = ib_umad_write,
+	.poll 	        = ib_umad_poll,
 	.unlocked_ioctl = ib_umad_ioctl,
-	.compat_ioctl = ib_umad_ioctl,
-	.open 	 = ib_umad_open,
-	.release = ib_umad_close
+	.compat_ioctl   = ib_umad_ioctl,
+	.open 	        = ib_umad_open,
+	.release        = ib_umad_close
 };
 
 static struct ib_client umad_client = {

@@ -705,8 +704,6 @@
 
 	return 0;
 
-	ib_unregister_client(&umad_client);
-
 out_class:
 	class_unregister(&umad_class);
 

@@ -174,27 +174,30 @@
 
 struct mthca_init_hca_param {
 	u64 qpc_base;
-	u8  log_num_qps;
 	u64 eec_base;
-	u8  log_num_eecs;
 	u64 srqc_base;
-	u8  log_num_srqs;
 	u64 cqc_base;
-	u8  log_num_cqs;
 	u64 eqpc_base;
 	u64 eeec_base;
 	u64 eqc_base;
-	u8  log_num_eqs;
 	u64 rdb_base;
 	u64 mc_base;
+	u64 mpt_base;
+	u64 mtt_base;
+	u64 uar_scratch_base;
+	u64 uarc_base;
 	u16 log_mc_entry_sz;
 	u16 mc_hash_sz;
+	u8  log_num_qps;
+	u8  log_num_eecs;
+	u8  log_num_srqs;
+	u8  log_num_cqs;
+	u8  log_num_eqs;
 	u8  log_mc_table_sz;
-	u64 mpt_base;
 	u8  mtt_seg_sz;
 	u8  log_mpt_sz;
-	u64 mtt_base;
-	u64 uar_scratch_base;
+	u8  log_uar_sz;
+	u8  log_uarc_sz;
 };
 
 struct mthca_init_ib_param {

@@ -238,6 +241,13 @@
 		  int port, u8 *status);
 int mthca_CLOSE_IB(struct mthca_dev *dev, int port, u8 *status);
 int mthca_CLOSE_HCA(struct mthca_dev *dev, int panic, u8 *status);
+int mthca_MAP_ICM(struct mthca_dev *dev, struct mthca_icm *icm, u64 virt, u8 *status);
+int mthca_MAP_ICM_page(struct mthca_dev *dev, u64 dma_addr, u64 virt, u8 *status);
+int mthca_UNMAP_ICM(struct mthca_dev *dev, u64 virt, u32 page_count, u8 *status);
+int mthca_MAP_ICM_AUX(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status);
+int mthca_UNMAP_ICM_AUX(struct mthca_dev *dev, u8 *status);
+int mthca_SET_ICM_SIZE(struct mthca_dev *dev, u64 icm_size, u64 *aux_pages,
+		       u8 *status);
 int mthca_SW2HW_MPT(struct mthca_dev *dev, void *mpt_entry,
 		    int mpt_index, u8 *status);
 int mthca_HW2SW_MPT(struct mthca_dev *dev, void *mpt_entry,

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU

@@ -60,7 +60,7 @@
 	MTHCA_NUM_PDS = 1 << 15
 };
 
-int mthca_make_profile(struct mthca_dev *dev,
+u64 mthca_make_profile(struct mthca_dev *dev,
 		       struct mthca_profile *request,
 		       struct mthca_dev_lim *dev_lim,
 		       struct mthca_init_hca_param *init_hca)

@@ -116,6 +116,8 @@
 		profile[i].type     = i;
 		profile[i].log_num  = max(ffs(profile[i].num) - 1, 0);
 		profile[i].size    *= profile[i].num;
+		if (dev->hca_type == ARBEL_NATIVE)
+			profile[i].size = max(profile[i].size, (u64) PAGE_SIZE);
 	}
 
 	if (dev->hca_type == ARBEL_NATIVE) {

@@ -239,6 +241,10 @@
 		case MTHCA_RES_UDAV:
 			dev->av_table.ddr_av_base = profile[i].start;
 			dev->av_table.num_ddr_avs = profile[i].num;
+		case MTHCA_RES_UARC:
+			init_hca->uarc_base   = profile[i].start;
+			init_hca->log_uarc_sz = ffs(request->uarc_size) - 13;
+			init_hca->log_uar_sz  = ffs(request->num_uar) - 1;
 		default:
 			break;
 		}

@@ -251,5 +257,5 @@
 	dev->limits.num_pds = MTHCA_NUM_PDS;
 
 	kfree(profile);
-	return 0;
+	return total_size;
 }

104 lines not on this page · raw message

50 lines not on this page · raw message

668 lines not on this page · raw message

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU

@@ -50,7 +50,7 @@
 	int uarc_size;
 };
 
-int mthca_make_profile(struct mthca_dev *mdev,
+u64 mthca_make_profile(struct mthca_dev *mdev,
 		       struct mthca_profile *request,
 		       struct mthca_dev_lim *dev_lim,
 		       struct mthca_init_hca_param *init_hca);

191 lines not on this page · raw message

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU

@@ -38,8 +38,10 @@
 #include <linux/list.h>
 #include <linux/pci.h>
 
+#include <asm/semaphore.h>
+
 #define MTHCA_ICM_CHUNK_LEN \
-	((512 - sizeof (struct list_head) - 2 * sizeof (int)) /		\
+	((256 - sizeof (struct list_head) - 2 * sizeof (int)) /		\
 	 (sizeof (struct scatterlist)))
 
 struct mthca_icm_chunk {

@@ -53,6 +55,13 @@
 	struct list_head chunk_list;
 };
 
+struct mthca_icm_table {
+	u64               virt;
+	int               num_icm;
+	struct semaphore  sem;
+	struct mthca_icm *icm[0];
+};
+
 struct mthca_icm_iter {
 	struct mthca_icm       *icm;
 	struct mthca_icm_chunk *chunk;

@@ -65,6 +74,12 @@
 				  unsigned int gfp_mask);
 void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm);
 
+struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev,
+					      u64 virt, unsigned size,
+					      unsigned reserved,
+					      int use_lowmem);
+void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table);
+
 static inline void mthca_icm_first(struct mthca_icm *icm,
 				   struct mthca_icm_iter *iter)
 {

78 lines not on this page · raw message

@@ -215,6 +215,13 @@
 	u64 si_guid;
 };
 
+struct mthca_set_ib_param {
+	int set_si_guid;
+	int reset_qkey_viol;
+	u64 si_guid;
+	u32 cap_mask;
+};
+
 int mthca_cmd_use_events(struct mthca_dev *dev);
 void mthca_cmd_use_polling(struct mthca_dev *dev);
 void mthca_cmd_event(struct mthca_dev *dev, u16 token,

@@ -241,6 +248,8 @@
 		  int port, u8 *status);
 int mthca_CLOSE_IB(struct mthca_dev *dev, int port, u8 *status);
 int mthca_CLOSE_HCA(struct mthca_dev *dev, int panic, u8 *status);
+int mthca_SET_IB(struct mthca_dev *dev, struct mthca_set_ib_param *param,
+		 int port, u8 *status);
 int mthca_MAP_ICM(struct mthca_dev *dev, struct mthca_icm *icm, u64 virt, u8 *status);
 int mthca_MAP_ICM_page(struct mthca_dev *dev, u64 dma_addr, u64 virt, u8 *status);
 int mthca_UNMAP_ICM(struct mthca_dev *dev, u64 virt, u32 page_count, u8 *status);

@@ -137,7 +137,35 @@
 			     u8 port, int port_modify_mask,
 			     struct ib_port_modify *props)
 {
-	return 0;
+	struct mthca_set_ib_param set_ib;
+	struct ib_port_attr attr;
+	int err;
+	u8 status;
+
+	if (down_interruptible(&to_mdev(ibdev)->cap_mask_mutex))
+		return -ERESTARTSYS;
+
+	err = mthca_query_port(ibdev, port, &attr);
+	if (err)
+		goto out;
+
+	set_ib.set_si_guid     = 0;
+	set_ib.reset_qkey_viol = !!(port_modify_mask & IB_PORT_RESET_QKEY_CNTR);
+
+	set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) &
+		~props->clr_port_cap_mask;
+
+	err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port, &status);
+	if (err)
+		goto out;
+	if (status) {
+		err = -EINVAL;
+		goto out;
+	}
+
+out:
+	up(&to_mdev(ibdev)->cap_mask_mutex);
+	return err;
 }
 
 static int mthca_query_pkey(struct ib_device *ibdev,

@@ -619,6 +647,8 @@
 		}
 	}
 
+	init_MUTEX(&dev->cap_mask_mutex);
+
 	return 0;
 }
 

@@ -1238,6 +1238,41 @@
 	return mthca_cmd(dev, 0, 0, panic, CMD_CLOSE_HCA, HZ, status);
 }
 
+int mthca_SET_IB(struct mthca_dev *dev, struct mthca_set_ib_param *param,
+		 int port, u8 *status)
+{
+	u32 *inbox;
+	dma_addr_t indma;
+	int err;
+	u32 flags = 0;
+
+#define SET_IB_IN_SIZE         0x40
+#define SET_IB_FLAGS_OFFSET    0x00
+#define SET_IB_FLAG_SIG        (1 << 18)
+#define SET_IB_FLAG_RQK        (1 <<  0)
+#define SET_IB_CAP_MASK_OFFSET 0x04
+#define SET_IB_SI_GUID_OFFSET  0x08
+
+	inbox = pci_alloc_consistent(dev->pdev, SET_IB_IN_SIZE, &indma);
+	if (!inbox)
+		return -ENOMEM;
+
+	memset(inbox, 0, SET_IB_IN_SIZE);
+
+	flags |= param->set_si_guid     ? SET_IB_FLAG_SIG : 0;
+	flags |= param->reset_qkey_viol ? SET_IB_FLAG_RQK : 0;
+	MTHCA_PUT(inbox, flags, SET_IB_FLAGS_OFFSET);
+
+	MTHCA_PUT(inbox, param->cap_mask, SET_IB_CAP_MASK_OFFSET);
+	MTHCA_PUT(inbox, param->si_guid,  SET_IB_SI_GUID_OFFSET);
+
+	err = mthca_cmd(dev, indma, port, 0, CMD_SET_IB,
+			CMD_TIME_CLASS_B, status);
+
+	pci_free_consistent(dev->pdev, INIT_HCA_IN_SIZE, inbox, indma);
+	return err;
+}
+
 int mthca_MAP_ICM(struct mthca_dev *dev, struct mthca_icm *icm, u64 virt, u8 *status)
 {
 	return mthca_map_cmd(dev, CMD_MAP_ICM, icm, virt, status);

@@ -234,6 +234,7 @@
 	u64              ddr_end;
 
 	MTHCA_DECLARE_DOORBELL_LOCK(doorbell_lock)
+	struct semaphore cap_mask_mutex;
 
 	void __iomem    *hcr;
 	void __iomem    *clr_base;

@@ -154,25 +154,28 @@
 };
 
 enum ib_port_cap_flags {
-	IB_PORT_SM				= (1<<31),
-	IB_PORT_NOTICE_SUP			= (1<<30),
-	IB_PORT_TRAP_SUP			= (1<<29),
-	IB_PORT_AUTO_MIGR_SUP			= (1<<27),
-	IB_PORT_SL_MAP_SUP			= (1<<26),
-	IB_PORT_MKEY_NVRAM			= (1<<25),
-	IB_PORT_PKEY_NVRAM			= (1<<24),
-	IB_PORT_LED_INFO_SUP			= (1<<23),
-	IB_PORT_SM_DISABLED			= (1<<22),
-	IB_PORT_SYS_IMAGE_GUID_SUP		= (1<<21),
-	IB_PORT_PKEY_SW_EXT_PORT_TRAP_SUP	= (1<<20),
-	IB_PORT_CM_SUP				= (1<<16),
-	IB_PORT_SNMP_TUNNEL_SUP			= (1<<15),
-	IB_PORT_REINIT_SUP			= (1<<14),
-	IB_PORT_DEVICE_MGMT_SUP			= (1<<13),
-	IB_PORT_VENDOR_CLASS_SUP		= (1<<12),
-	IB_PORT_DR_NOTICE_SUP			= (1<<11),
-	IB_PORT_PORT_NOTICE_SUP			= (1<<10),
-	IB_PORT_BOOT_MGMT_SUP			= (1<<9)
+	IB_PORT_SM				= 1 <<  1,
+	IB_PORT_NOTICE_SUP			= 1 <<  2,
+	IB_PORT_TRAP_SUP			= 1 <<  3,
+	IB_PORT_OPT_IPD_SUP                     = 1 <<  4,
+	IB_PORT_AUTO_MIGR_SUP			= 1 <<  5,
+	IB_PORT_SL_MAP_SUP			= 1 <<  6,
+	IB_PORT_MKEY_NVRAM			= 1 <<  7,
+	IB_PORT_PKEY_NVRAM			= 1 <<  8,
+	IB_PORT_LED_INFO_SUP			= 1 <<  9,
+	IB_PORT_SM_DISABLED			= 1 << 10,
+	IB_PORT_SYS_IMAGE_GUID_SUP		= 1 << 11,
+	IB_PORT_PKEY_SW_EXT_PORT_TRAP_SUP	= 1 << 12,
+	IB_PORT_CM_SUP				= 1 << 16,
+	IB_PORT_SNMP_TUNNEL_SUP			= 1 << 17,
+	IB_PORT_REINIT_SUP			= 1 << 18,
+	IB_PORT_DEVICE_MGMT_SUP			= 1 << 19,
+	IB_PORT_VENDOR_CLASS_SUP		= 1 << 20,
+	IB_PORT_DR_NOTICE_SUP			= 1 << 21,
+	IB_PORT_CAP_MASK_NOTICE_SUP		= 1 << 22,
+	IB_PORT_BOOT_MGMT_SUP			= 1 << 23,
+	IB_PORT_LINK_LATENCY_SUP		= 1 << 24,
+	IB_PORT_CLIENT_REG_SUP			= 1 << 25
 };
 
 enum ib_port_width {

@@ -381,7 +381,6 @@
 	struct mthca_eq  *eq  = eq_ptr;
 	struct mthca_dev *dev = eq->dev;
 
-	writel(eq->ecr_mask, dev->hcr + MTHCA_ECR_CLR_OFFSET + 4);
 	mthca_eq_int(dev, eq);
 
 	/* MSI-X vectors always belong to us */

@@ -280,7 +280,8 @@
 		   void *qp_context, u8 *status);
 int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn,
 			  u8 *status);
-int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int port,
+int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey,
+		  int port, struct ib_wc* in_wc, struct ib_grh* in_grh,
 		  void *in_mad, void *response_mad, u8 *status);
 int mthca_READ_MGM(struct mthca_dev *dev, int index, void *mgm,
 		   u8 *status);

@@ -59,8 +59,8 @@
 	in_mad->method         	   = IB_MGMT_METHOD_GET;
 	in_mad->attr_id   	   = IB_SMP_ATTR_NODE_INFO;
 
-	err = mthca_MAD_IFC(to_mdev(ibdev), 1,
-			    1, in_mad, out_mad,
+	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
+			    1, NULL, NULL, in_mad, out_mad,
 			    &status);
 	if (err)
 		goto out;

@@ -104,8 +104,8 @@
 	in_mad->attr_id   	   = IB_SMP_ATTR_PORT_INFO;
 	in_mad->attr_mod           = cpu_to_be32(port);
 
-	err = mthca_MAD_IFC(to_mdev(ibdev), 1,
-			    port, in_mad, out_mad,
+	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
+			    port, NULL, NULL, in_mad, out_mad,
 			    &status);
 	if (err)
 		goto out;

@@ -189,8 +189,8 @@
 	in_mad->attr_id   	   = IB_SMP_ATTR_PKEY_TABLE;
 	in_mad->attr_mod           = cpu_to_be32(index / 32);
 
-	err = mthca_MAD_IFC(to_mdev(ibdev), 1,
-			    port, in_mad, out_mad,
+	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
+			    port, NULL, NULL, in_mad, out_mad,
 			    &status);
 	if (err)
 		goto out;

@@ -228,8 +228,8 @@
 	in_mad->attr_id   	   = IB_SMP_ATTR_PORT_INFO;
 	in_mad->attr_mod           = cpu_to_be32(port);
 
-	err = mthca_MAD_IFC(to_mdev(ibdev), 1,
-			    port, in_mad, out_mad,
+	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
+			    port, NULL, NULL, in_mad, out_mad,
 			    &status);
 	if (err)
 		goto out;

@@ -248,8 +248,8 @@
 	in_mad->attr_id   	   = IB_SMP_ATTR_GUID_INFO;
 	in_mad->attr_mod           = cpu_to_be32(index / 8);
 
-	err = mthca_MAD_IFC(to_mdev(ibdev), 1,
-			    port, in_mad, out_mad,
+	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
+			    port, NULL, NULL, in_mad, out_mad,
 			    &status);
 	if (err)
 		goto out;

82 lines not on this page · raw message

@@ -232,8 +232,9 @@
 		return IB_MAD_RESULT_SUCCESS;
 
 	err = mthca_MAD_IFC(to_mdev(ibdev),
-			    !!(mad_flags & IB_MAD_IGNORE_MKEY),
-			    port_num, in_mad, out_mad,
+			    mad_flags & IB_MAD_IGNORE_MKEY,
+			    mad_flags & IB_MAD_IGNORE_BKEY,
+			    port_num, in_wc, in_grh, in_mad, out_mad,
 			    &status);
 	if (err) {
 		mthca_err(to_mdev(ibdev), "MAD_IFC failed\n");

@@ -66,11 +66,11 @@
 	struct mthca_dev      *dev;
 	int                    eqn;
 	u32                    ecr_mask;
+	u32                    cons_index;
 	u16                    msi_x_vector;
 	u16                    msi_x_entry;
 	int                    have_irq;
 	int                    nent;
-	int                    cons_index;
 	struct mthca_buf_list *page_list;
 	struct mthca_mr        mr;
 };

101 lines not on this page · raw message

@@ -289,6 +289,7 @@
 		    u8 *status);
 int mthca_MGID_HASH(struct mthca_dev *dev, void *gid, u16 *hash,
 		    u8 *status);
+int mthca_NOP(struct mthca_dev *dev, u8 *status);
 
 #define MAILBOX_ALIGN(x) ((void *) ALIGN((unsigned long) (x), MTHCA_CMD_MAILBOX_ALIGN))
 

@@ -570,6 +570,7 @@
 static int __devinit mthca_setup_hca(struct mthca_dev *dev)
 {
 	int err;
+	u8 status;
 
 	MTHCA_INIT_DOORBELL_LOCK(&dev->doorbell_lock);
 

@@ -615,6 +616,18 @@
 		goto err_eq_table_free;
 	}
 
+	err = mthca_NOP(dev, &status);
+	if (err || status) {
+		mthca_err(dev, "NOP command failed to generate interrupt, aborting.\n");
+		if (dev->mthca_flags & (MTHCA_FLAG_MSI | MTHCA_FLAG_MSI_X))
+			mthca_err(dev, "Try again with MSI/MSI-X disabled.\n");
+		else
+			mthca_err(dev, "BIOS or ACPI interrupt routing problem?\n");
+
+		goto err_cmd_poll;
+	} else
+		mthca_dbg(dev, "NOP command IRQ test passed\n");
+
 	err = mthca_init_cq_table(dev);
 	if (err) {
 		mthca_err(dev, "Failed to initialize "

@@ -1757,3 +1757,8 @@
 	pci_unmap_single(dev->pdev, indma, 16, PCI_DMA_TODEVICE);
 	return err;
 }
+
+int mthca_NOP(struct mthca_dev *dev, u8 *status)
+{
+	return mthca_cmd(dev, 0, 0x1f, 0, CMD_NOP, msecs_to_jiffies(100), status);
+}

@@ -509,7 +509,7 @@
 			goto timeout;
 		}
 
-		yield();
+		msleep(1);
 	}
 
 	ipoib_dbg(priv, "All sends and receives done.\n");

@@ -535,7 +535,7 @@
 			break;
 		}
 
-		yield();
+		msleep(1);
 	}
 
 	return 0;

@@ -45,6 +45,7 @@
 #include <linux/kref.h>
 
 #include <asm/uaccess.h>
+#include <asm/semaphore.h>
 
 #include <ib_mad.h>
 #include <ib_user_mad.h>

@@ -54,7 +55,7 @@
 MODULE_LICENSE("Dual BSD/GPL");
 
 enum {
-	IB_UMAD_MAX_PORTS  = 256,
+	IB_UMAD_MAX_PORTS  = 64,
 	IB_UMAD_MAX_AGENTS = 32
 };
 

@@ -62,6 +63,12 @@
 	int                    devnum;
 	struct cdev            dev;
 	struct class_device    class_dev;
+
+	int                    sm_devnum;
+	struct cdev            sm_dev;
+	struct class_device    sm_class_dev;
+	struct semaphore       sm_sem;
+
 	struct ib_device      *ib_dev;
 	struct ib_umad_device *umad_dev;
 	u8                     port_num;

@@ -92,7 +99,7 @@
 
 static dev_t base_dev;
 static spinlock_t map_lock;
-static DECLARE_BITMAP(dev_map, IB_UMAD_MAX_PORTS);
+static DECLARE_BITMAP(dev_map, IB_UMAD_MAX_PORTS * 2);
 
 static void ib_umad_add_one(struct ib_device *device);
 static void ib_umad_remove_one(struct ib_device *device);

@@ -511,6 +518,54 @@
 	.release        = ib_umad_close
 };
 
+static int ib_umad_sm_open(struct inode *inode, struct file *filp)
+{
+	struct ib_umad_port *port =
+		container_of(inode->i_cdev, struct ib_umad_port, sm_dev);
+	struct ib_port_modify props = {
+		.set_port_cap_mask = IB_PORT_SM
+	};
+	int ret;
+
+	if (filp->f_flags & O_NONBLOCK) {
+		if (down_trylock(&port->sm_sem))
+			return -EAGAIN;
+	} else {
+		if (down_interruptible(&port->sm_sem))
+			return -ERESTARTSYS;
+	}
+
+	ret = ib_modify_port(port->ib_dev, port->port_num, 0, &props);
+	if (ret) {
+		up(&port->sm_sem);
+		return ret;
+	}
+
+	filp->private_data = port;
+
+	return 0;
+}
+
+static int ib_umad_sm_close(struct inode *inode, struct file *filp)
+{
+	struct ib_umad_port *port = filp->private_data;
+	struct ib_port_modify props = {
+		.clr_port_cap_mask = IB_PORT_SM
+	};
+	int ret;
+
+	ret = ib_modify_port(port->ib_dev, port->port_num, 0, &props);
+	up(&port->sm_sem);
+
+	return ret;
+}
+
+static struct file_operations umad_sm_fops = {
+	.owner 	 = THIS_MODULE,
+	.open 	 = ib_umad_sm_open,
+	.release = ib_umad_sm_close
+};
+
 static struct ib_client umad_client = {
 	.name   = "umad",
 	.add    = ib_umad_add_one,

@@ -519,17 +574,18 @@
 
 static ssize_t show_dev(struct class_device *class_dev, char *buf)
 {
-	struct ib_umad_port *port =
-		container_of(class_dev, struct ib_umad_port, class_dev);
+	struct ib_umad_port *port = class_get_devdata(class_dev);
 
-	return print_dev_t(buf, port->dev.dev);
+	if (class_dev == &port->class_dev)
+		return print_dev_t(buf, port->dev.dev);
+	else
+		return print_dev_t(buf, port->sm_dev.dev);
 }
 static CLASS_DEVICE_ATTR(dev, S_IRUGO, show_dev, NULL);
 
 static ssize_t show_ibdev(struct class_device *class_dev, char *buf)
 {
-	struct ib_umad_port *port =
-		container_of(class_dev, struct ib_umad_port, class_dev);
+	struct ib_umad_port *port = class_get_devdata(class_dev);
 
 	return sprintf(buf, "%s\n", port->ib_dev->name);
 }

@@ -537,8 +593,7 @@
 
 static ssize_t show_port(struct class_device *class_dev, char *buf)
 {
-	struct ib_umad_port *port =
-		container_of(class_dev, struct ib_umad_port, class_dev);
+	struct ib_umad_port *port = class_get_devdata(class_dev);
 
 	return sprintf(buf, "%d\n", port->port_num);
 }

@@ -554,11 +609,16 @@
 
 static void ib_umad_release_port(struct class_device *class_dev)
 {
-	struct ib_umad_port *port =
-		container_of(class_dev, struct ib_umad_port, class_dev);
+	struct ib_umad_port *port = class_get_devdata(class_dev);
+
+	if (class_dev == &port->class_dev) {
+		cdev_del(&port->dev);
+		clear_bit(port->devnum, dev_map);
+	} else {
+		cdev_del(&port->sm_dev);
+		clear_bit(port->sm_devnum, dev_map);
+	}
 
-	cdev_del(&port->dev);
-	clear_bit(port->devnum, dev_map);
 	kref_put(&port->umad_dev->ref, ib_umad_release_dev);
 }
 

@@ -573,6 +633,94 @@
 }
 static CLASS_ATTR(abi_version, S_IRUGO, show_abi_version, NULL);
 
+static int ib_umad_init_port(struct ib_device *device, int port_num,
+			     struct ib_umad_port *port)
+{
+	spin_lock(&map_lock);
+	port->devnum = find_first_zero_bit(dev_map, IB_UMAD_MAX_PORTS);
+	if (port->devnum >= IB_UMAD_MAX_PORTS) {
+		spin_unlock(&map_lock);
+		return -1;
+	}
+	port->sm_devnum = find_next_zero_bit(dev_map, IB_UMAD_MAX_PORTS * 2, IB_UMAD_MAX_PORTS);
+	if (port->sm_devnum >= IB_UMAD_MAX_PORTS * 2) {
+		spin_unlock(&map_lock);
+		return -1;
+	}
+	set_bit(port->devnum, dev_map);
+	set_bit(port->sm_devnum, dev_map);
+	spin_unlock(&map_lock);
+
+	port->ib_dev   = device;
+	port->port_num = port_num;
+	init_MUTEX(&port->sm_sem);
+
+	cdev_init(&port->dev, &umad_fops);
+	port->dev.owner = THIS_MODULE;
+	kobject_set_name(&port->dev.kobj, "umad%d", port->devnum);
+	if (cdev_add(&port->dev, base_dev + port->devnum, 1))
+		return -1;
+
+	port->class_dev.class = &umad_class;
+	port->class_dev.dev   = device->dma_device;
+
+	snprintf(port->class_dev.class_id, BUS_ID_SIZE, "umad%d", port->devnum);
+
+	if (class_device_register(&port->class_dev))
+		goto err_cdev;
+
+	class_set_devdata(&port->class_dev, port);
+	kref_get(&port->umad_dev->ref);
+
+	if (class_device_create_file(&port->class_dev, &class_device_attr_dev))
+		goto err_class;
+	if (class_device_create_file(&port->class_dev, &class_device_attr_ibdev))
+		goto err_class;
+	if (class_device_create_file(&port->class_dev, &class_device_attr_port))
+		goto err_class;
+
+	cdev_init(&port->sm_dev, &umad_sm_fops);
+	port->sm_dev.owner = THIS_MODULE;
+	kobject_set_name(&port->dev.kobj, "issm%d", port->sm_devnum - IB_UMAD_MAX_PORTS);
+	if (cdev_add(&port->sm_dev, base_dev + port->sm_devnum, 1))
+		return -1;
+
+	port->sm_class_dev.class = &umad_class;
+	port->sm_class_dev.dev   = device->dma_device;
+
+	snprintf(port->sm_class_dev.class_id, BUS_ID_SIZE, "issm%d", port->sm_devnum - IB_UMAD_MAX_PORTS);
+
+	if (class_device_register(&port->sm_class_dev))
+		goto err_sm_cdev;
+
+	class_set_devdata(&port->sm_class_dev, port);
+	kref_get(&port->umad_dev->ref);
+
+	if (class_device_create_file(&port->sm_class_dev, &class_device_attr_dev))
+		goto err_sm_class;
+	if (class_device_create_file(&port->sm_class_dev, &class_device_attr_ibdev))
+		goto err_sm_class;
+	if (class_device_create_file(&port->sm_class_dev, &class_device_attr_port))
+		goto err_sm_class;
+
+	return 0;
+
+err_sm_class:
+	class_device_unregister(&port->sm_class_dev);
+
+err_sm_cdev:
+	cdev_del(&port->sm_dev);
+
+err_class:
+	class_device_unregister(&port->class_dev);
+
+err_cdev:
+	cdev_del(&port->dev);
+	clear_bit(port->devnum, dev_map);
+
+	return -1;
+}
+
 static void ib_umad_add_one(struct ib_device *device)
 {
 	struct ib_umad_device *umad_dev;

@@ -601,58 +749,20 @@
 
 	for (i = s; i <= e; ++i) {
 		umad_dev->port[i - s].umad_dev = umad_dev;
-		kref_get(&umad_dev->ref);
-
-		spin_lock(&map_lock);
-		umad_dev->port[i - s].devnum =
-			find_first_zero_bit(dev_map, IB_UMAD_MAX_PORTS);
-		if (umad_dev->port[i - s].devnum >= IB_UMAD_MAX_PORTS) {
-			spin_unlock(&map_lock);
-			goto err;
-		}
-		set_bit(umad_dev->port[i - s].devnum, dev_map);
-		spin_unlock(&map_lock);
 
-		umad_dev->port[i - s].ib_dev   = device;
-		umad_dev->port[i - s].port_num = i;
-
-		cdev_init(&umad_dev->port[i - s].dev, &umad_fops);
-		umad_dev->port[i - s].dev.owner = THIS_MODULE;
-		kobject_set_name(&umad_dev->port[i - s].dev.kobj,
-				 "umad%d", umad_dev->port[i - s].devnum);
-		if (cdev_add(&umad_dev->port[i - s].dev, base_dev +
-			     umad_dev->port[i - s].devnum, 1))
+		if (ib_umad_init_port(device, i, &umad_dev->port[i - s]))
 			goto err;
-
-		umad_dev->port[i - s].class_dev.class = &umad_class;
-		umad_dev->port[i - s].class_dev.dev   = device->dma_device;
-		snprintf(umad_dev->port[i - s].class_dev.class_id,
-			 BUS_ID_SIZE, "umad%d", umad_dev->port[i - s].devnum);
-		if (class_device_register(&umad_dev->port[i - s].class_dev))
-			goto err_class;
-
-		if (class_device_create_file(&umad_dev->port[i - s].class_dev,
-					     &class_device_attr_dev))
-			goto err_class;
-		if (class_device_create_file(&umad_dev->port[i - s].class_dev,
-					     &class_device_attr_ibdev))
-			goto err_class;
-		if (class_device_create_file(&umad_dev->port[i - s].class_dev,
-					     &class_device_attr_port))
-			goto err_class;
 	}
 
 	ib_set_client_data(device, &umad_client, umad_dev);
 
 	return;
 
-err_class:
-	cdev_del(&umad_dev->port[i - s].dev);
-	clear_bit(umad_dev->port[i - s].devnum, dev_map);
-
 err:
-	while (--i >= s)
+	while (--i >= s) {
 		class_device_unregister(&umad_dev->port[i - s].class_dev);
+		class_device_unregister(&umad_dev->port[i - s].sm_class_dev);
+	}
 
 	kref_put(&umad_dev->ref, ib_umad_release_dev);
 }

@@ -665,8 +775,10 @@
 	if (!umad_dev)
 		return;
 
-	for (i = 0; i <= umad_dev->end_port - umad_dev->start_port; ++i)
+	for (i = 0; i <= umad_dev->end_port - umad_dev->start_port; ++i) {
 		class_device_unregister(&umad_dev->port[i].class_dev);
+		class_device_unregister(&umad_dev->port[i].sm_class_dev);
+	}
 
 	kref_put(&umad_dev->ref, ib_umad_release_dev);
 }

@@ -677,7 +789,7 @@
 
 	spin_lock_init(&map_lock);
 
-	ret = alloc_chrdev_region(&base_dev, 0, IB_UMAD_MAX_PORTS,
+	ret = alloc_chrdev_region(&base_dev, 0, IB_UMAD_MAX_PORTS * 2,
 				  "infiniband_mad");
 	if (ret) {
 		printk(KERN_ERR "user_mad: couldn't get device number\n");

@@ -708,7 +820,7 @@
 	class_unregister(&umad_class);
 
 out_chrdev:
-	unregister_chrdev_region(base_dev, IB_UMAD_MAX_PORTS);
+	unregister_chrdev_region(base_dev, IB_UMAD_MAX_PORTS * 2);
 
 out:
 	return ret;

@@ -718,7 +830,7 @@
 {
 	ib_unregister_client(&umad_client);
 	class_unregister(&umad_class);
-	unregister_chrdev_region(base_dev, IB_UMAD_MAX_PORTS);
+	unregister_chrdev_region(base_dev, IB_UMAD_MAX_PORTS * 2);
 }
 
 module_init(ib_umad_init);

@@ -366,10 +366,11 @@
 	if (dev->eq_table.clr_mask)
 		writel(dev->eq_table.clr_mask, dev->eq_table.clr_int);
 
-	if ((ecr = readl(dev->hcr + MTHCA_ECR_OFFSET + 4)) != 0) {
+	if ((ecr = readl(dev->ecr_base + 4)) != 0) {
 		work = 1;
 
-		writel(ecr, dev->hcr + MTHCA_ECR_CLR_OFFSET + 4);
+		writel(ecr, dev->ecr_base +
+		       MTHCA_ECR_CLR_BASE - MTHCA_ECR_BASE + 4);
 
 		for (i = 0; i < MTHCA_NUM_EQ; ++i)
 			if (ecr & dev->eq_table.eq[i].ecr_mask)

160 lines not on this page · raw message

@@ -43,13 +43,8 @@
 #define MTHCA_ECR_SIZE         0x00008
 #define MTHCA_ECR_CLR_BASE     0x80708
 #define MTHCA_ECR_CLR_SIZE     0x00008
-#define MTHCA_ECR_OFFSET       (MTHCA_ECR_BASE     - MTHCA_HCR_BASE)
-#define MTHCA_ECR_CLR_OFFSET   (MTHCA_ECR_CLR_BASE - MTHCA_HCR_BASE)
+#define MTHCA_MAP_ECR_SIZE     (MTHCA_ECR_SIZE + MTHCA_ECR_CLR_SIZE)
 #define MTHCA_CLR_INT_BASE     0xf00d8
 #define MTHCA_CLR_INT_SIZE     0x00008
 
-#define MTHCA_MAP_HCR_SIZE     (MTHCA_ECR_CLR_BASE   + \
-			        MTHCA_ECR_CLR_SIZE   - \
-			        MTHCA_HCR_BASE)
-
 #endif /* MTHCA_CONFIG_REG_H */

@@ -237,6 +237,7 @@
 	struct semaphore cap_mask_mutex;
 
 	void __iomem    *hcr;
+	void __iomem    *ecr_base;
 	void __iomem    *clr_base;
 	void __iomem    *kar;
 

@@ -40,10 +40,6 @@
 #include <linux/pci.h>
 #include <linux/interrupt.h>
 
-#ifdef CONFIG_INFINIBAND_MTHCA_SSE_DOORBELL
-#include <asm/cpufeature.h>
-#endif
-
 #include "mthca_dev.h"
 #include "mthca_config_reg.h"
 #include "mthca_cmd.h"

@@ -1117,22 +1113,6 @@
 {
 	int ret;
 
-	/*
-	 * TODO: measure whether dynamically choosing doorbell code at
-	 * runtime affects our performance.  Is there a "magic" way to
-	 * choose without having to follow a function pointer every
-	 * time we ring a doorbell?
-	 */
-#ifdef CONFIG_INFINIBAND_MTHCA_SSE_DOORBELL
-	if (!cpu_has_xmm) {
-		printk(KERN_ERR PFX "mthca was compiled with SSE doorbell code, but\n");
-		printk(KERN_ERR PFX "the current CPU does not support SSE.\n");
-		printk(KERN_ERR PFX "Turn off CONFIG_INFINIBAND_MTHCA_SSE_DOORBELL "
-		       "and recompile.\n");
-		return -ENODEV;
-	}
-#endif
-
 	ret = pci_register_driver(&mthca_driver);
 	return ret < 0 ? ret : 0;
 }

66 lines not on this page · raw message

@@ -634,6 +634,8 @@
 	dev->ib_dev.detach_mcast         = mthca_multicast_detach;
 	dev->ib_dev.process_mad          = mthca_process_mad;
 
+	init_MUTEX(&dev->cap_mask_mutex);
+
 	ret = ib_register_device(&dev->ib_dev);
 	if (ret)
 		return ret;

@@ -647,8 +649,6 @@
 		}
 	}
 
-	init_MUTEX(&dev->cap_mask_mutex);
-
 	return 0;
 }