InfiniBand merge

@@ -114,7 +114,7 @@
 		cache = device->cache.gid_cache[p];
 		for (i = 0; i < cache->table_len; ++i) {
 			if (!memcmp(gid, &cache->table[i], sizeof *gid)) {
-				*port_num = p;
+				*port_num = p + start_port(device);
 				if (index)
 					*index = i;
 				ret = 0;

@@ -147,20 +147,21 @@
 			+ (entry * MTHCA_CQ_ENTRY_SIZE) % PAGE_SIZE;
 }
 
-static inline int cqe_sw(struct mthca_cq *cq, int i)
+static inline struct mthca_cqe *cqe_sw(struct mthca_cq *cq, int i)
 {
-	return !(MTHCA_CQ_ENTRY_OWNER_HW &
-		 get_cqe(cq, i)->owner);
+	struct mthca_cqe *cqe;
+	cqe = get_cqe(cq, i);
+	return (MTHCA_CQ_ENTRY_OWNER_HW & cqe->owner) ? NULL : cqe;
 }
 
-static inline int next_cqe_sw(struct mthca_cq *cq)
+static inline struct mthca_cqe *next_cqe_sw(struct mthca_cq *cq)
 {
 	return cqe_sw(cq, cq->cons_index);
 }
 
-static inline void set_cqe_hw(struct mthca_cq *cq, int entry)
+static inline void set_cqe_hw(struct mthca_cqe *cqe)
 {
-	get_cqe(cq, entry)->owner = MTHCA_CQ_ENTRY_OWNER_HW;
+	cqe->owner = MTHCA_CQ_ENTRY_OWNER_HW;
 }
 
 static inline void inc_cons_index(struct mthca_dev *dev, struct mthca_cq *cq,

@@ -388,7 +389,8 @@
 	int free_cqe = 1;
 	int err = 0;
 
-	if (!next_cqe_sw(cq))
+	cqe = next_cqe_sw(cq);
+	if (!cqe)
 		return -EAGAIN;
 
 	/*

@@ -397,8 +399,6 @@
 	 */
 	rmb();
 
-	cqe = get_cqe(cq, cq->cons_index);
-
 	if (0) {
 		mthca_dbg(dev, "%x/%d: CQE -> QPN %06x, WQE @ %08x\n",
 			  cq->cqn, cq->cons_index, be32_to_cpu(cqe->my_qpn),

@@ -509,8 +509,8 @@
 	entry->status = IB_WC_SUCCESS;
 
  out:
-	if (free_cqe) {
-		set_cqe_hw(cq, cq->cons_index);
+	if (likely(free_cqe)) {
+		set_cqe_hw(cqe);
 		++(*freed);
 		cq->cons_index = (cq->cons_index + 1) & cq->ibcq.cqe;
 	}

@@ -655,7 +655,7 @@
 	}
 
 	for (i = 0; i < nent; ++i)
-		set_cqe_hw(cq, i);
+		set_cqe_hw(get_cqe(cq, i));
 
 	cq->cqn = mthca_alloc(&dev->cq_table.alloc);
 	if (cq->cqn == -1)

@@ -773,7 +773,7 @@
 		int j;
 
 		printk(KERN_ERR "context for CQN %x (cons index %x, next sw %d)\n",
-		       cq->cqn, cq->cons_index, next_cqe_sw(cq));
+		       cq->cqn, cq->cons_index, !!next_cqe_sw(cq));
 		for (j = 0; j < 16; ++j)
 			printk(KERN_ERR "[%2x] %08x\n", j * 4, be32_to_cpu(ctx[j]));
 	}

@@ -33,6 +33,7 @@
  */
 
 #include <linux/init.h>
+#include <linux/hardirq.h>
 
 #include <ib_pack.h>
 

@@ -181,11 +182,7 @@
 {
 	struct mthca_cq *cq;
 
-	spin_lock(&dev->cq_table.lock);
 	cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1));
-	if (cq)
-		atomic_inc(&cq->refcount);
-	spin_unlock(&dev->cq_table.lock);
 
 	if (!cq) {
 		mthca_warn(dev, "Completion event for bogus CQ %08x\n", cqn);

@@ -193,9 +190,6 @@
 	}
 
 	cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
-
-	if (atomic_dec_and_test(&cq->refcount))
-		wake_up(&cq->wait);
 }
 
 void mthca_cq_clean(struct mthca_dev *dev, u32 cqn, u32 qpn)

@@ -783,6 +777,11 @@
 			  cq->cqn & (dev->limits.num_cqs - 1));
 	spin_unlock_irq(&dev->cq_table.lock);
 
+	if (dev->mthca_flags & MTHCA_FLAG_MSI_X)
+		synchronize_irq(dev->eq_table.eq[MTHCA_EQ_COMP].msi_x_vector);
+	else
+		synchronize_irq(dev->pdev->irq);
+
 	atomic_dec(&cq->refcount);
 	wait_event(cq->wait, !atomic_read(&cq->refcount));
 

@@ -418,14 +418,14 @@
 			spin_unlock(&(*cur_qp)->lock);
 		}
 
-		spin_lock(&dev->qp_table.lock);
+		/*
+		 * We do not have to take the QP table lock here,
+		 * because CQs will be locked while QPs are removed
+		 * from the table.
+		 */
 		*cur_qp = mthca_array_get(&dev->qp_table.qp,
 					  be32_to_cpu(cqe->my_qpn) &
 					  (dev->limits.num_qps - 1));
-		if (*cur_qp)
-			atomic_inc(&(*cur_qp)->refcount);
-		spin_unlock(&dev->qp_table.lock);
-
 		if (!*cur_qp) {
 			mthca_warn(dev, "CQ entry for unknown QP %06x\n",
 				   be32_to_cpu(cqe->my_qpn) & 0xffffff);

@@ -537,12 +537,8 @@
 		inc_cons_index(dev, cq, freed);
 	}
 
-	if (qp) {
+	if (qp)
 		spin_unlock(&qp->lock);
-		if (atomic_dec_and_test(&qp->refcount))
-			wake_up(&qp->wait);
-	}
-
 
 	spin_unlock_irqrestore(&cq->lock, flags);
 

@@ -1083,9 +1083,21 @@
 	return 0;
 
  err_out_free:
-	spin_lock_irq(&dev->qp_table.lock);
+	/*
+	 * Lock CQs here, so that CQ polling code can do QP lookup
+	 * without taking a lock.
+	 */
+	spin_lock_irq(&send_cq->lock);
+	if (send_cq != recv_cq)
+		spin_lock(&recv_cq->lock);
+
+	spin_lock(&dev->qp_table.lock);
 	mthca_array_clear(&dev->qp_table.qp, mqpn);
-	spin_unlock_irq(&dev->qp_table.lock);
+	spin_unlock(&dev->qp_table.lock);
+
+	if (send_cq != recv_cq)
+		spin_unlock(&recv_cq->lock);
+	spin_unlock_irq(&send_cq->lock);
 
  err_out:
 	dma_free_coherent(&dev->pdev->dev, sqp->header_buf_size,

@@ -1100,11 +1112,28 @@
 	u8 status;
 	int size;
 	int i;
+	struct mthca_cq *send_cq;
+	struct mthca_cq *recv_cq;
+
+	send_cq = to_mcq(qp->ibqp.send_cq);
+	recv_cq = to_mcq(qp->ibqp.recv_cq);
 
-	spin_lock_irq(&dev->qp_table.lock);
+	/*
+	 * Lock CQs here, so that CQ polling code can do QP lookup
+	 * without taking a lock.
+	 */
+	spin_lock_irq(&send_cq->lock);
+	if (send_cq != recv_cq)
+		spin_lock(&recv_cq->lock);
+
+	spin_lock(&dev->qp_table.lock);
 	mthca_array_clear(&dev->qp_table.qp,
 			  qp->qpn & (dev->limits.num_qps - 1));
-	spin_unlock_irq(&dev->qp_table.lock);
+	spin_unlock(&dev->qp_table.lock);
+
+	if (send_cq != recv_cq)
+		spin_unlock(&recv_cq->lock);
+	spin_unlock_irq(&send_cq->lock);
 
 	atomic_dec(&qp->refcount);
 	wait_event(qp->wait, !atomic_read(&qp->refcount));

@@ -150,9 +150,8 @@
 
 static inline struct mthca_cqe *cqe_sw(struct mthca_cq *cq, int i)
 {
-	struct mthca_cqe *cqe;
-	cqe = get_cqe(cq, i);
-	return (MTHCA_CQ_ENTRY_OWNER_HW & cqe->owner) ? NULL : cqe;
+	struct mthca_cqe *cqe = get_cqe(cq, i);
+	return MTHCA_CQ_ENTRY_OWNER_HW & cqe->owner ? NULL : cqe;
 }
 
 static inline struct mthca_cqe *next_cqe_sw(struct mthca_cq *cq)

@@ -378,7 +377,7 @@
 	struct mthca_wq *wq;
 	struct mthca_cqe *cqe;
 	int wqe_index;
-	int is_error = 0;
+	int is_error;
 	int is_send;
 	int free_cqe = 1;
 	int err = 0;

@@ -401,12 +400,9 @@
 		dump_cqe(cqe);
 	}
 
-	if ((cqe->opcode & MTHCA_ERROR_CQE_OPCODE_MASK) ==
-	    MTHCA_ERROR_CQE_OPCODE_MASK) {
-		is_error = 1;
-		is_send = cqe->opcode & 1;
-	} else
-		is_send = cqe->is_send & 0x80;
+	is_error = (cqe->opcode & MTHCA_ERROR_CQE_OPCODE_MASK) ==
+		MTHCA_ERROR_CQE_OPCODE_MASK;
+	is_send  = is_error ? cqe->opcode & 0x01 : cqe->is_send & 0x80;
 
 	if (!*cur_qp || be32_to_cpu(cqe->my_qpn) != (*cur_qp)->qpn) {
 		if (*cur_qp) {

@@ -2191,7 +2191,6 @@
 	recv_wr.next = NULL;
 	recv_wr.sg_list = &sg_list;
 	recv_wr.num_sge = 1;
-	recv_wr.recv_flags = IB_RECV_SIGNALED;
 
 	do {
 		/* Allocate and map receive buffer */

@@ -2386,7 +2385,6 @@
 	qp_init_attr.send_cq = qp_info->port_priv->cq;
 	qp_init_attr.recv_cq = qp_info->port_priv->cq;
 	qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
-	qp_init_attr.rq_sig_type = IB_SIGNAL_ALL_WR;
 	qp_init_attr.cap.max_send_wr = IB_MAD_QP_SEND_SIZE;
 	qp_init_attr.cap.max_recv_wr = IB_MAD_QP_RECV_SIZE;
 	qp_init_attr.cap.max_send_sge = IB_MAD_SEND_REQ_MAX_SG;

@@ -369,14 +369,12 @@
 		   struct mthca_cq *recv_cq,
 		   enum ib_qp_type type,
 		   enum ib_sig_type send_policy,
-		   enum ib_sig_type recv_policy,
 		   struct mthca_qp *qp);
 int mthca_alloc_sqp(struct mthca_dev *dev,
 		    struct mthca_pd *pd,
 		    struct mthca_cq *send_cq,
 		    struct mthca_cq *recv_cq,
 		    enum ib_sig_type send_policy,
-		    enum ib_sig_type recv_policy,
 		    int qpn,
 		    int port,
 		    struct mthca_sqp *sqp);

@@ -343,7 +343,7 @@
 				     to_mcq(init_attr->send_cq),
 				     to_mcq(init_attr->recv_cq),
 				     init_attr->qp_type, init_attr->sq_sig_type,
-				     init_attr->rq_sig_type, qp);
+				     qp);
 		qp->ibqp.qp_num = qp->qpn;
 		break;
 	}

@@ -364,7 +364,7 @@
 		err = mthca_alloc_sqp(to_mdev(pd->device), to_mpd(pd),
 				      to_mcq(init_attr->send_cq),
 				      to_mcq(init_attr->recv_cq),
-				      init_attr->sq_sig_type, init_attr->rq_sig_type,
+				      init_attr->sq_sig_type,
 				      qp->ibqp.qp_num, init_attr->port_num,
 				      to_msqp(qp));
 		break;

@@ -154,7 +154,6 @@
 	void *last;
 	int   max_gs;
 	int   wqe_shift;
-	enum ib_sig_type policy;
 };
 
 struct mthca_qp {

@@ -172,6 +171,7 @@
 
 	struct mthca_wq        rq;
 	struct mthca_wq        sq;
+	enum ib_sig_type       sq_policy;
 	int                    send_wqe_offset;
 
 	u64                   *wrid;

@@ -690,7 +690,7 @@
 					     MTHCA_QP_BIT_SRE           |
 					     MTHCA_QP_BIT_SWE           |
 					     MTHCA_QP_BIT_SAE);
-	if (qp->sq.policy == IB_SIGNAL_ALL_WR)
+	if (qp->sq_policy == IB_SIGNAL_ALL_WR)
 		qp_context->params1 |= cpu_to_be32(MTHCA_QP_BIT_SSC);
 	if (attr_mask & IB_QP_RETRY_CNT) {
 		qp_context->params1 |= cpu_to_be32(attr->retry_cnt << 16);

@@ -778,8 +778,8 @@
 		qp->resp_depth = attr->max_rd_atomic;
 	}
 
-	if (qp->rq.policy == IB_SIGNAL_ALL_WR)
-		qp_context->params2 |= cpu_to_be32(MTHCA_QP_BIT_RSC);
+	qp_context->params2 |= cpu_to_be32(MTHCA_QP_BIT_RSC);
+
 	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
 		qp_context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
 		qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RNR_TIMEOUT);

@@ -977,7 +977,6 @@
 				 struct mthca_cq *send_cq,
 				 struct mthca_cq *recv_cq,
 				 enum ib_sig_type send_policy,
-				 enum ib_sig_type recv_policy,
 				 struct mthca_qp *qp)
 {
 	int err;

@@ -987,8 +986,7 @@
 	qp->state    	 = IB_QPS_RESET;
 	qp->atomic_rd_en = 0;
 	qp->resp_depth   = 0;
-	qp->sq.policy    = send_policy;
-	qp->rq.policy    = recv_policy;
+	qp->sq_policy    = send_policy;
 	qp->rq.cur       = 0;
 	qp->sq.cur       = 0;
 	qp->rq.next      = 0;

@@ -1008,7 +1006,6 @@
 		   struct mthca_cq *recv_cq,
 		   enum ib_qp_type type,
 		   enum ib_sig_type send_policy,
-		   enum ib_sig_type recv_policy,
 		   struct mthca_qp *qp)
 {
 	int err;

@@ -1025,7 +1022,7 @@
 		return -ENOMEM;
 
 	err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq,
-				    send_policy, recv_policy, qp);
+				    send_policy, qp);
 	if (err) {
 		mthca_free(&dev->qp_table.alloc, qp->qpn);
 		return err;

@@ -1044,7 +1041,6 @@
 		    struct mthca_cq *send_cq,
 		    struct mthca_cq *recv_cq,
 		    enum ib_sig_type send_policy,
-		    enum ib_sig_type recv_policy,
 		    int qpn,
 		    int port,
 		    struct mthca_sqp *sqp)

@@ -1073,8 +1069,7 @@
 	sqp->qp.transport = MLX;
 
 	err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq,
-				    send_policy, recv_policy,
-				    &sqp->qp);
+				    send_policy, &sqp->qp);
 	if (err)
 		goto err_out_free;
 

@@ -1495,9 +1490,7 @@
 		((struct mthca_next_seg *) wqe)->nda_op = 0;
 		((struct mthca_next_seg *) wqe)->ee_nds =
 			cpu_to_be32(MTHCA_NEXT_DBD);
-		((struct mthca_next_seg *) wqe)->flags =
-			(wr->recv_flags & IB_RECV_SIGNALED) ?
-			cpu_to_be32(MTHCA_NEXT_CQ_UPDATE) : 0;
+		((struct mthca_next_seg *) wqe)->flags = 0;
 
 		wqe += sizeof (struct mthca_next_seg);
 		size = sizeof (struct mthca_next_seg) / 16;

@@ -73,7 +73,6 @@
 	IB_DEVICE_RC_RNR_NAK_GEN	= (1<<12),
 	IB_DEVICE_SRQ_RESIZE		= (1<<13),
 	IB_DEVICE_N_NOTIFY_CQ		= (1<<14),
-	IB_DEVICE_RQ_SIG_TYPE		= (1<<15)
 };
 
 enum ib_atomic_cap {

@@ -408,7 +407,6 @@
 	struct ib_srq	       *srq;
 	struct ib_qp_cap	cap;
 	enum ib_sig_type	sq_sig_type;
-	enum ib_sig_type	rq_sig_type;
 	enum ib_qp_type		qp_type;
 	u8			port_num; /* special QP types only */
 };

@@ -533,10 +531,6 @@
 	IB_SEND_INLINE		= (1<<3)
 };
 
-enum ib_recv_flags {
-	IB_RECV_SIGNALED	= 1
-};
-
 struct ib_sge {
 	u64	addr;
 	u32	length;

@@ -579,7 +573,6 @@
 	u64			wr_id;
 	struct ib_sge	       *sg_list;
 	int			num_sge;
-	int			recv_flags;
 };
 
 enum ib_access_flags {

@@ -105,7 +105,6 @@
 		.wr_id 	    = wr_id | IPOIB_OP_RECV,
 		.sg_list    = &list,
 		.num_sge    = 1,
-		.recv_flags = IB_RECV_SIGNALED
 	};
 	struct ib_recv_wr *bad_wr;
 

@@ -165,7 +165,6 @@
 			.max_recv_sge = 1
 		},
 		.sq_sig_type = IB_SIGNAL_ALL_WR,
-		.rq_sig_type = IB_SIGNAL_ALL_WR,
 		.qp_type     = IB_QPT_UD
 	};
 

@@ -46,5 +46,6 @@
 #define MTHCA_MAP_ECR_SIZE     (MTHCA_ECR_SIZE + MTHCA_ECR_CLR_SIZE)
 #define MTHCA_CLR_INT_BASE     0xf00d8
 #define MTHCA_CLR_INT_SIZE     0x00008
+#define MTHCA_EQ_SET_CI_SIZE   (8 * 32)
 
 #endif /* MTHCA_CONFIG_REG_H */

@@ -237,9 +237,17 @@
 	struct semaphore cap_mask_mutex;
 
 	void __iomem    *hcr;
-	void __iomem    *ecr_base;
-	void __iomem    *clr_base;
 	void __iomem    *kar;
+	void __iomem    *clr_base;
+	union {
+		struct {
+			void __iomem *ecr_base;
+		} tavor;
+		struct {
+			void __iomem *eq_arm;
+			void __iomem *eq_set_ci_base;
+		} arbel;
+	} eq_regs;
 
 	struct mthca_cmd    cmd;
 	struct mthca_limits limits;

175 lines not on this page · raw message

144 lines not on this page · raw message

@@ -666,7 +666,7 @@
 						  MTHCA_CQ_FLAG_TR);
 	cq_context->start           = cpu_to_be64(0);
 	cq_context->logsize_usrpage = cpu_to_be32((ffs(nent) - 1) << 24 |
-						  MTHCA_KAR_PAGE);
+						  dev->driver_uar.index);
 	cq_context->error_eqn       = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn);
 	cq_context->comp_eqn        = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_COMP].eqn);
 	cq_context->pd              = cpu_to_be32(dev->driver_pd.pd_num);

@@ -65,7 +65,6 @@
 };
 
 enum {
-	MTHCA_KAR_PAGE  = 1,
 	MTHCA_MAX_PORTS = 2
 };
 

@@ -108,6 +107,7 @@
 	int      gid_table_len;
 	int      pkey_table_len;
 	int      local_ca_ack_delay;
+	int      num_uars;
 	int      max_sg;
 	int      num_qps;
 	int      reserved_qps;

@@ -148,6 +148,12 @@
 	} *page_list;
 };
 
+struct mthca_uar_table {
+	struct mthca_alloc alloc;
+	u64                uarc_base;
+	int                uarc_size;
+};
+
 struct mthca_pd_table {
 	struct mthca_alloc alloc;
 };

@@ -252,6 +258,7 @@
 	struct mthca_cmd    cmd;
 	struct mthca_limits limits;
 
+	struct mthca_uar_table uar_table;
 	struct mthca_pd_table  pd_table;
 	struct mthca_mr_table  mr_table;
 	struct mthca_eq_table  eq_table;

@@ -260,6 +267,7 @@
 	struct mthca_av_table  av_table;
 	struct mthca_mcg_table mcg_table;
 
+	struct mthca_uar      driver_uar;
 	struct mthca_pd       driver_pd;
 	struct mthca_mr       driver_mr;
 

@@ -318,6 +326,7 @@
 int mthca_array_init(struct mthca_array *array, int nent);
 void mthca_array_cleanup(struct mthca_array *array, int nent);
 
+int mthca_init_uar_table(struct mthca_dev *dev);
 int mthca_init_pd_table(struct mthca_dev *dev);
 int mthca_init_mr_table(struct mthca_dev *dev);
 int mthca_init_eq_table(struct mthca_dev *dev);

@@ -326,6 +335,7 @@
 int mthca_init_av_table(struct mthca_dev *dev);
 int mthca_init_mcg_table(struct mthca_dev *dev);
 
+void mthca_cleanup_uar_table(struct mthca_dev *dev);
 void mthca_cleanup_pd_table(struct mthca_dev *dev);
 void mthca_cleanup_mr_table(struct mthca_dev *dev);
 void mthca_cleanup_eq_table(struct mthca_dev *dev);

@@ -337,6 +347,9 @@
 int mthca_register_device(struct mthca_dev *dev);
 void mthca_unregister_device(struct mthca_dev *dev);
 
+int mthca_uar_alloc(struct mthca_dev *dev, struct mthca_uar *uar);
+void mthca_uar_free(struct mthca_dev *dev, struct mthca_uar *uar);
+
 int mthca_pd_alloc(struct mthca_dev *dev, struct mthca_pd *pd);
 void mthca_pd_free(struct mthca_dev *dev, struct mthca_pd *pd);
 

@@ -469,7 +469,7 @@
 						  MTHCA_EQ_FLAG_TR);
 	eq_context->start           = cpu_to_be64(0);
 	eq_context->logsize_usrpage = cpu_to_be32((ffs(nent) - 1) << 24 |
-						  MTHCA_KAR_PAGE);
+						  dev->driver_uar.index);
 	eq_context->pd              = cpu_to_be32(dev->driver_pd.pd_num);
 	eq_context->intr            = intr;
 	eq_context->lkey            = cpu_to_be32(eq->mr.ibmr.lkey);

121 lines not on this page · raw message

@@ -236,6 +236,7 @@
 			init_hca->mtt_seg_sz     = ffs(dev_lim->mtt_seg_sz) - 7;
 			break;
 		case MTHCA_RES_UAR:
+			dev->limits.num_uars       = profile[i].num;
 			init_hca->uar_scratch_base = profile[i].start;
 			break;
 		case MTHCA_RES_UDAV:

@@ -49,6 +49,11 @@
 	DECLARE_PCI_UNMAP_ADDR(mapping)
 };
 
+struct mthca_uar {
+	unsigned long pfn;
+	int           index;
+};
+
 struct mthca_mr {
 	struct ib_mr ibmr;
 	int order;

@@ -625,7 +625,7 @@
 		qp_context->mtu_msgmax = cpu_to_be32((attr->path_mtu << 29) |
 						     (31 << 24));
 	}
-	qp_context->usr_page   = cpu_to_be32(MTHCA_KAR_PAGE);
+	qp_context->usr_page   = cpu_to_be32(dev->driver_uar.index);
 	qp_context->local_qpn  = cpu_to_be32(qp->qpn);
 	if (attr_mask & IB_QP_DEST_QPN) {
 		qp_context->remote_qpn = cpu_to_be32(attr->dest_qp_num);

69 lines not on this page · raw message

@@ -363,10 +363,9 @@
 	}
 
 	mdev->mr_table.mtt_table = mthca_alloc_icm_table(mdev, init_hca->mtt_base,
-							 mdev->limits.num_mtt_segs *
 							 init_hca->mtt_seg_sz,
-							 mdev->limits.reserved_mtts *
-							 init_hca->mtt_seg_sz, 1);
+							 mdev->limits.num_mtt_segs,
+							 mdev->limits.reserved_mtts, 1);
 	if (!mdev->mr_table.mtt_table) {
 		mthca_err(mdev, "Failed to map MTT context memory, aborting.\n");
 		err = -ENOMEM;

@@ -374,10 +373,9 @@
 	}
 
 	mdev->mr_table.mpt_table = mthca_alloc_icm_table(mdev, init_hca->mpt_base,
-							 mdev->limits.num_mpts *
 							 dev_lim->mpt_entry_sz,
-							 mdev->limits.reserved_mrws *
-							 dev_lim->mpt_entry_sz, 1);
+							 mdev->limits.num_mpts,
+							 mdev->limits.reserved_mrws, 1);
 	if (!mdev->mr_table.mpt_table) {
 		mthca_err(mdev, "Failed to map MPT context memory, aborting.\n");
 		err = -ENOMEM;

@@ -385,10 +383,9 @@
 	}
 
 	mdev->qp_table.qp_table = mthca_alloc_icm_table(mdev, init_hca->qpc_base,
-							mdev->limits.num_qps *
 							dev_lim->qpc_entry_sz,
-							mdev->limits.reserved_qps *
-							dev_lim->qpc_entry_sz, 1);
+							mdev->limits.num_qps,
+							mdev->limits.reserved_qps, 0);
 	if (!mdev->qp_table.qp_table) {
 		mthca_err(mdev, "Failed to map QP context memory, aborting.\n");
 		err = -ENOMEM;

@@ -396,10 +393,9 @@
 	}
 
 	mdev->qp_table.eqp_table = mthca_alloc_icm_table(mdev, init_hca->eqpc_base,
-							 mdev->limits.num_qps *
 							 dev_lim->eqpc_entry_sz,
-							 mdev->limits.reserved_qps *
-							 dev_lim->eqpc_entry_sz, 1);
+							 mdev->limits.num_qps,
+							 mdev->limits.reserved_qps, 0);
 	if (!mdev->qp_table.eqp_table) {
 		mthca_err(mdev, "Failed to map EQP context memory, aborting.\n");
 		err = -ENOMEM;

@@ -407,10 +403,9 @@
 	}
 
 	mdev->cq_table.table = mthca_alloc_icm_table(mdev, init_hca->cqc_base,
-						     mdev->limits.num_cqs *
 						     dev_lim->cqc_entry_sz,
-						     mdev->limits.reserved_cqs *
-						     dev_lim->cqc_entry_sz, 1);
+						     mdev->limits.num_cqs,
+						     mdev->limits.reserved_cqs, 0);
 	if (!mdev->cq_table.table) {
 		mthca_err(mdev, "Failed to map CQ context memory, aborting.\n");
 		err = -ENOMEM;

111 lines not on this page · raw message

@@ -53,12 +53,16 @@
 
 struct mthca_icm {
 	struct list_head chunk_list;
+	int              refcount;
 };
 
 struct mthca_icm_table {
 	u64               virt;
 	int               num_icm;
-	struct semaphore  sem;
+	int               num_obj;
+	int               obj_size;
+	int               lowmem;
+	struct semaphore  mutex;
 	struct mthca_icm *icm[0];
 };
 

@@ -75,10 +79,12 @@
 void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm);
 
 struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev,
-					      u64 virt, unsigned size,
-					      unsigned reserved,
+					      u64 virt, int obj_size,
+					      int nobj, int reserved,
 					      int use_lowmem);
 void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table);
+int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int obj);
+void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj);
 
 static inline void mthca_icm_first(struct mthca_icm *icm,
 				   struct mthca_icm_iter *iter)

@@ -53,7 +53,8 @@
 	u32 window_count;
 	u32 window_count_limit;
 	u64 mtt_seg;
-	u32 reserved[3];
+	u32 mtt_sz;		/* Arbel only */
+	u32 reserved[2];
 } __attribute__((packed));
 
 #define MTHCA_MPT_FLAG_SW_OWNS       (0xfUL << 28)

@@ -121,21 +122,38 @@
 	spin_unlock(&dev->mr_table.mpt_alloc.lock);
 }
 
+static inline u32 hw_index_to_key(struct mthca_dev *dev, u32 ind)
+{
+	if (dev->hca_type == ARBEL_NATIVE)
+		return (ind >> 24) | (ind << 8);
+	else
+		return ind;
+}
+
+static inline u32 key_to_hw_index(struct mthca_dev *dev, u32 key)
+{
+	if (dev->hca_type == ARBEL_NATIVE)
+		return (key << 24) | (key >> 8);
+	else
+		return key;
+}
+
 int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd,
 			   u32 access, struct mthca_mr *mr)
 {
 	void *mailbox;
 	struct mthca_mpt_entry *mpt_entry;
+	u32 key;
 	int err;
 	u8 status;
 
 	might_sleep();
 
 	mr->order = -1;
-	mr->ibmr.lkey = mthca_alloc(&dev->mr_table.mpt_alloc);
-	if (mr->ibmr.lkey == -1)
+	key = mthca_alloc(&dev->mr_table.mpt_alloc);
+	if (key == -1)
 		return -ENOMEM;
-	mr->ibmr.rkey = mr->ibmr.lkey;
+	mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
 
 	mailbox = kmalloc(sizeof *mpt_entry + MTHCA_CMD_MAILBOX_EXTRA,
 			  GFP_KERNEL);

@@ -151,7 +169,7 @@
 				       MTHCA_MPT_FLAG_REGION      |
 				       access);
 	mpt_entry->page_size = 0;
-	mpt_entry->key       = cpu_to_be32(mr->ibmr.lkey);
+	mpt_entry->key       = cpu_to_be32(key);
 	mpt_entry->pd        = cpu_to_be32(pd);
 	mpt_entry->start     = 0;
 	mpt_entry->length    = ~0ULL;

@@ -160,7 +178,7 @@
 	       sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey));
 
 	err = mthca_SW2HW_MPT(dev, mpt_entry,
-			      mr->ibmr.lkey & (dev->limits.num_mpts - 1),
+			      key & (dev->limits.num_mpts - 1),
 			      &status);
 	if (err)
 		mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);

@@ -182,6 +200,7 @@
 	void *mailbox;
 	u64 *mtt_entry;
 	struct mthca_mpt_entry *mpt_entry;
+	u32 key;
 	int err = -ENOMEM;
 	u8 status;
 	int i;

@@ -189,10 +208,10 @@
 	might_sleep();
 	WARN_ON(buffer_size_shift >= 32);
 
-	mr->ibmr.lkey = mthca_alloc(&dev->mr_table.mpt_alloc);
-	if (mr->ibmr.lkey == -1)
+	key = mthca_alloc(&dev->mr_table.mpt_alloc);
+	if (key == -1)
 		return -ENOMEM;
-	mr->ibmr.rkey = mr->ibmr.lkey;
+	mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
 
 	for (i = dev->limits.mtt_seg_size / 8, mr->order = 0;
 	     i < list_len;

@@ -254,7 +273,7 @@
 				       access);
 
 	mpt_entry->page_size = cpu_to_be32(buffer_size_shift - 12);
-	mpt_entry->key       = cpu_to_be32(mr->ibmr.lkey);
+	mpt_entry->key       = cpu_to_be32(key);
 	mpt_entry->pd        = cpu_to_be32(pd);
 	mpt_entry->start     = cpu_to_be64(iova);
 	mpt_entry->length    = cpu_to_be64(total_size);

@@ -275,7 +294,7 @@
 	}
 
 	err = mthca_SW2HW_MPT(dev, mpt_entry,
-			      mr->ibmr.lkey & (dev->limits.num_mpts - 1),
+			      key & (dev->limits.num_mpts - 1),
 			      &status);
 	if (err)
 		mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);

@@ -307,7 +326,8 @@
 	might_sleep();
 
 	err = mthca_HW2SW_MPT(dev, NULL,
-			      mr->ibmr.lkey & (dev->limits.num_mpts - 1),
+			      key_to_hw_index(dev, mr->ibmr.lkey) &
+			      (dev->limits.num_mpts - 1),
 			      &status);
 	if (err)
 		mthca_warn(dev, "HW2SW_MPT failed (%d)\n", err);

@@ -318,7 +338,7 @@
 	if (mr->order >= 0)
 		mthca_free_mtt(dev, mr->first_seg, mr->order);
 
-	mthca_free(&dev->mr_table.mpt_alloc, mr->ibmr.lkey);
+	mthca_free(&dev->mr_table.mpt_alloc, key_to_hw_index(dev, mr->ibmr.lkey));
 }
 
 int __devinit mthca_init_mr_table(struct mthca_dev *dev)

@@ -54,10 +54,10 @@
 	u32 flags;
 	u64 start;
 	u32 logsize_usrpage;
-	u32 pd;
+	u32 tavor_pd;		/* reserved for Arbel */
 	u8  reserved1[3];
 	u8  intr;
-	u32 lost_count;
+	u32 arbel_pd;		/* lost_count for Tavor */
 	u32 lkey;
 	u32 reserved2[2];
 	u32 consumer_index;

@@ -75,6 +75,7 @@
 #define MTHCA_EQ_STATE_ARMED        ( 1 <<  8)
 #define MTHCA_EQ_STATE_FIRED        ( 2 <<  8)
 #define MTHCA_EQ_STATE_ALWAYS_ARMED ( 3 <<  8)
+#define MTHCA_EQ_STATE_ARBEL        ( 8 <<  8)
 
 enum {
 	MTHCA_EVENT_TYPE_COMP       	    = 0x00,

@@ -467,10 +468,16 @@
 						  MTHCA_EQ_OWNER_HW    |
 						  MTHCA_EQ_STATE_ARMED |
 						  MTHCA_EQ_FLAG_TR);
-	eq_context->start           = cpu_to_be64(0);
-	eq_context->logsize_usrpage = cpu_to_be32((ffs(nent) - 1) << 24 |
-						  dev->driver_uar.index);
-	eq_context->pd              = cpu_to_be32(dev->driver_pd.pd_num);
+	if (dev->hca_type == ARBEL_NATIVE)
+		eq_context->flags  |= cpu_to_be32(MTHCA_EQ_STATE_ARBEL);
+
+	eq_context->logsize_usrpage = cpu_to_be32((ffs(nent) - 1) << 24);
+	if (dev->hca_type == ARBEL_NATIVE) {
+		eq_context->arbel_pd = cpu_to_be32(dev->driver_pd.pd_num);
+	} else {
+		eq_context->logsize_usrpage |= cpu_to_be32(dev->driver_uar.index);
+		eq_context->tavor_pd         = cpu_to_be32(dev->driver_pd.pd_num);
+	}
 	eq_context->intr            = intr;
 	eq_context->lkey            = cpu_to_be32(eq->mr.ibmr.lkey);
 

@@ -171,6 +171,7 @@
 	struct mthca_alloc alloc;
 	void __iomem      *clr_int;
 	u32                clr_mask;
+	u32                arm_mask;
 	struct mthca_eq    eq[MTHCA_NUM_EQ];
 	u64                icm_virt;
 	struct page       *icm_page;

272 lines not on this page · raw message

@@ -608,13 +608,6 @@
 		goto err_mr_table_free;
 	}
 
-	if (dev->hca_type == ARBEL_NATIVE) {
-		mthca_warn(dev, "Sorry, native MT25208 mode support is not done, "
-			   "aborting.\n");
-		err = -ENODEV;
-		goto err_pd_free;
-	}
-
 	err = mthca_init_eq_table(dev);
 	if (err) {
 		mthca_err(dev, "Failed to initialize "

@@ -638,8 +631,16 @@
 			mthca_err(dev, "BIOS or ACPI interrupt routing problem?\n");
 
 		goto err_cmd_poll;
-	} else
-		mthca_dbg(dev, "NOP command IRQ test passed\n");
+	}
+
+	mthca_dbg(dev, "NOP command IRQ test passed\n");
+
+	if (dev->hca_type == ARBEL_NATIVE) {
+		mthca_warn(dev, "Sorry, native MT25208 mode support is not complete, "
+			   "aborting.\n");
+		err = -ENODEV;
+		goto err_cmd_poll;
+	}
 
 	err = mthca_init_cq_table(dev);
 	if (err) {

@@ -70,7 +70,7 @@
 struct mthca_eq {
 	struct mthca_dev      *dev;
 	int                    eqn;
-	u32                    ecr_mask;
+	u32                    eqn_mask;
 	u32                    cons_index;
 	u16                    msi_x_vector;
 	u16                    msi_x_entry;

@@ -1305,6 +1305,9 @@
 
 int mthca_UNMAP_ICM(struct mthca_dev *dev, u64 virt, u32 page_count, u8 *status)
 {
+	mthca_dbg(dev, "Unmapping %d pages at %llx from ICM.\n",
+		  page_count, (unsigned long long) virt);
+
 	return mthca_cmd(dev, virt, page_count, 0, CMD_UNMAP_ICM, CMD_TIME_CLASS_B, status);
 }
 

@@ -1538,10 +1541,10 @@
 		if (0) {
 			int i;
 			mthca_dbg(dev, "Dumping QP context:\n");
-			printk(" %08x\n", be32_to_cpup(qp_context));
+			printk("  opt param mask: %08x\n", be32_to_cpup(qp_context));
 			for (i = 0; i < 0x100 / 4; ++i) {
 				if (i % 8 == 0)
-					printk("[%02x] ", i * 4);
+					printk("  [%02x] ", i * 4);
 				printk(" %08x", be32_to_cpu(((u32 *) qp_context)[i + 2]));
 				if ((i + 1) % 8 == 0)
 					printk("\n");

@@ -1290,7 +1290,7 @@
 		return -ENOMEM;
 
 	inbox[0] = cpu_to_be64(virt);
-	inbox[1] = cpu_to_be64(dma_addr | (PAGE_SHIFT - 12));
+	inbox[1] = cpu_to_be64(dma_addr);
 
 	err = mthca_cmd(dev, indma, 1, 0, CMD_MAP_ICM, CMD_TIME_CLASS_B, status);
 

@@ -268,9 +268,10 @@
 	struct mthca_av_table  av_table;
 	struct mthca_mcg_table mcg_table;
 
-	struct mthca_uar      driver_uar;
-	struct mthca_pd       driver_pd;
-	struct mthca_mr       driver_mr;
+	struct mthca_uar       driver_uar;
+	struct mthca_db_table *db_tab;
+	struct mthca_pd        driver_pd;
+	struct mthca_mr        driver_mr;
 
 	struct ib_mad_agent  *send_agent[MTHCA_MAX_PORTS][2];
 	struct ib_ah         *sm_ah[MTHCA_MAX_PORTS];

199 lines not on this page · raw message

@@ -125,4 +125,37 @@
 	return sg_dma_len(&iter->chunk->mem[iter->page_idx]);
 }
 
+enum {
+	MTHCA_DB_REC_PER_PAGE = 4096 / 8
+};
+
+struct mthca_db_page {
+	DECLARE_BITMAP(used, MTHCA_DB_REC_PER_PAGE);
+	u64       *db_rec;
+	dma_addr_t mapping;
+};
+
+struct mthca_db_table {
+	int 	       	      npages;
+	int 	       	      max_group1;
+	int 	       	      min_group2;
+	struct mthca_db_page *page;
+	struct semaphore      mutex;
+};
+
+enum {
+	MTHCA_DB_TYPE_INVALID   = 0x0,
+	MTHCA_DB_TYPE_CQ_SET_CI = 0x1,
+	MTHCA_DB_TYPE_CQ_ARM    = 0x2,
+	MTHCA_DB_TYPE_SQ        = 0x3,
+	MTHCA_DB_TYPE_RQ        = 0x4,
+	MTHCA_DB_TYPE_SRQ       = 0x5,
+	MTHCA_DB_TYPE_GROUP_SEP = 0x7
+};
+
+int mthca_init_db_tab(struct mthca_dev *dev);
+void mthca_cleanup_db_tab(struct mthca_dev *dev);
+int mthca_alloc_db(struct mthca_dev *dev, int type, u32 qn, u32 **db);
+void mthca_free_db(struct mthca_dev *dev, int type, int db_index);
+
 #endif /* MTHCA_MEMFREE_H */

@@ -244,9 +244,11 @@
 			dev->av_table.num_ddr_avs = profile[i].num;
 			break;
 		case MTHCA_RES_UARC:
-			init_hca->uarc_base   = profile[i].start;
-			init_hca->log_uarc_sz = ffs(request->uarc_size) - 13;
-			init_hca->log_uar_sz  = ffs(request->num_uar) - 1;
+			dev->uar_table.uarc_size = request->uarc_size;
+			dev->uar_table.uarc_base = profile[i].start;
+			init_hca->uarc_base   	 = profile[i].start;
+			init_hca->log_uarc_sz 	 = ffs(request->uarc_size) - 13;
+			init_hca->log_uar_sz  	 = ffs(request->num_uar) - 1;
 			break;
 		default:
 			break;

@@ -33,6 +33,7 @@
  */
 
 #include "mthca_dev.h"
+#include "mthca_memfree.h"
 
 int mthca_uar_alloc(struct mthca_dev *dev, struct mthca_uar *uar)
 {

@@ -58,12 +59,20 @@
 			       dev->limits.num_uars,
 			       dev->limits.num_uars - 1,
 			       dev->limits.reserved_uars);
+	if (ret)
+		return ret;
+
+	ret = mthca_init_db_tab(dev);
+	if (ret)
+		mthca_alloc_cleanup(&dev->uar_table.alloc);
 
 	return ret;
 }
 
 void mthca_cleanup_uar_table(struct mthca_dev *dev)
 {
+	mthca_cleanup_db_tab(dev);
+
 	/* XXX check if any UARs are still allocated? */
 	mthca_alloc_cleanup(&dev->uar_table.alloc);
 }

@@ -57,6 +57,11 @@
 	__raw_writeq(*(u64 *) val, dest);
 }
 
+static inline void mthca_write_db_rec(u32 val[2], u32 *db)
+{
+	*(u64 *) db = *(u64 *) val;
+}
+
 #else
 
 /*

@@ -80,4 +85,11 @@
 	spin_unlock_irqrestore(doorbell_lock, flags);
 }
 
+static inline void mthca_write_db_rec(u32 val[2], u32 *db)
+{
+	db[0] = val[0];
+	wmb();
+	db[1] = val[1];
+}
+
 #endif

@@ -557,32 +557,40 @@
 		      MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
 }
 
-int mthca_init_cq(struct mthca_dev *dev, int nent,
-		  struct mthca_cq *cq)
+static void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq *cq)
 {
-	int size = nent * MTHCA_CQ_ENTRY_SIZE;
-	dma_addr_t t;
-	void *mailbox = NULL;
-	int npages, shift;
-	u64 *dma_list = NULL;
-	struct mthca_cq_context *cq_context;
-	int err = -ENOMEM;
-	u8 status;
 	int i;
+	int size;
 
-	might_sleep();
+	if (cq->is_direct)
+		pci_free_consistent(dev->pdev,
+				    (cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE,
+				    cq->queue.direct.buf,
+				    pci_unmap_addr(&cq->queue.direct,
+						   mapping));
+	else {
+		size = (cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE;
+		for (i = 0; i < (size + PAGE_SIZE - 1) / PAGE_SIZE; ++i)
+			if (cq->queue.page_list[i].buf)
+				pci_free_consistent(dev->pdev, PAGE_SIZE,
+						    cq->queue.page_list[i].buf,
+						    pci_unmap_addr(&cq->queue.page_list[i],
+								   mapping));
 
-	mailbox = kmalloc(sizeof (struct mthca_cq_context) + MTHCA_CMD_MAILBOX_EXTRA,
-			  GFP_KERNEL);
-	if (!mailbox)
-		goto err_out;
+		kfree(cq->queue.page_list);
+	}
+}
 
-	cq_context = MAILBOX_ALIGN(mailbox);
+static int mthca_alloc_cq_buf(struct mthca_dev *dev, int size,
+			      struct mthca_cq *cq)
+{
+	int err = -ENOMEM;
+	int npages, shift;
+	u64 *dma_list = NULL;
+	dma_addr_t t;
+	int i;
 
 	if (size <= MTHCA_MAX_DIRECT_CQ_SIZE) {
-		if (0)
-			mthca_dbg(dev, "Creating direct CQ of size %d\n", size);
-
 		cq->is_direct = 1;
 		npages        = 1;
 		shift         = get_order(size) + PAGE_SHIFT;

@@ -590,7 +598,7 @@
 		cq->queue.direct.buf = pci_alloc_consistent(dev->pdev,
 							    size, &t);
 		if (!cq->queue.direct.buf)
-			goto err_out;
+			return -ENOMEM;
 
 		pci_unmap_addr_set(&cq->queue.direct, mapping, t);
 

@@ -603,7 +611,7 @@
 
 		dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
 		if (!dma_list)
-			goto err_out_free;
+			goto err_free;
 
 		for (i = 0; i < npages; ++i)
 			dma_list[i] = t + i * (1 << shift);

@@ -612,12 +620,9 @@
 		npages        = (size + PAGE_SIZE - 1) / PAGE_SIZE;
 		shift         = PAGE_SHIFT;
 
-		if (0)
-			mthca_dbg(dev, "Creating indirect CQ with %d pages\n", npages);
-
 		dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
 		if (!dma_list)
-			goto err_out;
+			return -ENOMEM;
 
 		cq->queue.page_list = kmalloc(npages * sizeof *cq->queue.page_list,
 					      GFP_KERNEL);

@@ -631,7 +636,7 @@
 			cq->queue.page_list[i].buf =
 				pci_alloc_consistent(dev->pdev, PAGE_SIZE, &t);
 			if (!cq->queue.page_list[i].buf)
-				goto err_out_free;
+				goto err_free;
 
 			dma_list[i] = t;
 			pci_unmap_addr_set(&cq->queue.page_list[i], mapping, t);

@@ -640,13 +645,6 @@
 		}
 	}
 
-	for (i = 0; i < nent; ++i)
-		set_cqe_hw(get_cqe(cq, i));
-
-	cq->cqn = mthca_alloc(&dev->cq_table.alloc);
-	if (cq->cqn == -1)
-		goto err_out_free;
-
 	err = mthca_mr_alloc_phys(dev, dev->driver_pd.pd_num,
 				  dma_list, shift, npages,
 				  0, size,

@@ -654,7 +652,52 @@
 				  MTHCA_MPT_FLAG_LOCAL_READ,
 				  &cq->mr);
 	if (err)
-		goto err_out_free_cq;
+		goto err_free;
+
+	kfree(dma_list);
+
+	return 0;
+
+err_free:
+	mthca_free_cq_buf(dev, cq);
+
+err_out:
+	kfree(dma_list);
+
+	return err;
+}
+
+int mthca_init_cq(struct mthca_dev *dev, int nent,
+		  struct mthca_cq *cq)
+{
+	int size = nent * MTHCA_CQ_ENTRY_SIZE;
+	void *mailbox = NULL;
+	struct mthca_cq_context *cq_context;
+	int err = -ENOMEM;
+	u8 status;
+	int i;
+
+	might_sleep();
+
+	cq->ibcq.cqe = nent - 1;
+
+	cq->cqn = mthca_alloc(&dev->cq_table.alloc);
+	if (cq->cqn == -1)
+		return -ENOMEM;
+
+	mailbox = kmalloc(sizeof (struct mthca_cq_context) + MTHCA_CMD_MAILBOX_EXTRA,
+			  GFP_KERNEL);
+	if (!mailbox)
+		goto err_out;
+
+	cq_context = MAILBOX_ALIGN(mailbox);
+
+	err = mthca_alloc_cq_buf(dev, size, cq);
+	if (err)
+		goto err_out_mailbox;
+
+	for (i = 0; i < nent; ++i)
+		set_cqe_hw(get_cqe(cq, i));
 
 	spin_lock_init(&cq->lock);
 	atomic_set(&cq->refcount, 1);

@@ -697,37 +740,20 @@
 
 	cq->cons_index = 0;
 
-	kfree(dma_list);
 	kfree(mailbox);
 
 	return 0;
 
- err_out_free_mr:
+err_out_free_mr:
 	mthca_free_mr(dev, &cq->mr);
+	mthca_free_cq_buf(dev, cq);
 
- err_out_free_cq:
-	mthca_free(&dev->cq_table.alloc, cq->cqn);
-
- err_out_free:
-	if (cq->is_direct)
-		pci_free_consistent(dev->pdev, size,
-				    cq->queue.direct.buf,
-				    pci_unmap_addr(&cq->queue.direct, mapping));
-	else {
-		for (i = 0; i < npages; ++i)
-			if (cq->queue.page_list[i].buf)
-				pci_free_consistent(dev->pdev, PAGE_SIZE,
-						    cq->queue.page_list[i].buf,
-						    pci_unmap_addr(&cq->queue.page_list[i],
-								   mapping));
-
-		kfree(cq->queue.page_list);
-	}
-
- err_out:
-	kfree(dma_list);
+err_out_mailbox:
 	kfree(mailbox);
 
+err_out:
+	mthca_free(&dev->cq_table.alloc, cq->cqn);
+
 	return err;
 }
 

@@ -778,27 +804,7 @@
 	wait_event(cq->wait, !atomic_read(&cq->refcount));
 
 	mthca_free_mr(dev, &cq->mr);
-
-	if (cq->is_direct)
-		pci_free_consistent(dev->pdev,
-				    (cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE,
-				    cq->queue.direct.buf,
-				    pci_unmap_addr(&cq->queue.direct,
-						   mapping));
-	else {
-		int i;
-
-		for (i = 0;
-		     i < ((cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE + PAGE_SIZE - 1) /
-			     PAGE_SIZE;
-		     ++i)
-			pci_free_consistent(dev->pdev, PAGE_SIZE,
-					    cq->queue.page_list[i].buf,
-					    pci_unmap_addr(&cq->queue.page_list[i],
-							   mapping));
-
-		kfree(cq->queue.page_list);
-	}
+	mthca_free_cq_buf(dev, cq);
 
 	mthca_free(&dev->cq_table.alloc, cq->cqn);
 	kfree(mailbox);

@@ -408,8 +408,7 @@
 	if (err) {
 		kfree(cq);
 		cq = ERR_PTR(err);
-	} else
-		cq->ibcq.cqe = nent - 1;
+	}
 
 	return &cq->ibcq;
 }

@@ -39,6 +39,7 @@
 
 #include "mthca_dev.h"
 #include "mthca_cmd.h"
+#include "mthca_memfree.h"
 
 enum {
 	MTHCA_MAX_DIRECT_CQ_SIZE = 4 * PAGE_SIZE

@@ -55,7 +56,7 @@
 	u32 flags;
 	u64 start;
 	u32 logsize_usrpage;
-	u32 error_eqn;
+	u32 error_eqn;		/* Tavor only */
 	u32 comp_eqn;
 	u32 pd;
 	u32 lkey;

@@ -64,7 +65,9 @@
 	u32 consumer_index;
 	u32 producer_index;
 	u32 cqn;
-	u32 reserved[3];
+	u32 ci_db;		/* Arbel only */
+	u32 state_db;		/* Arbel only */
+	u32 reserved;
 } __attribute__((packed));
 
 #define MTHCA_CQ_STATUS_OK          ( 0 << 28)

@@ -685,10 +688,30 @@
 	if (cq->cqn == -1)
 		return -ENOMEM;
 
+	if (dev->hca_type == ARBEL_NATIVE) {
+		cq->arm_sn = 1;
+
+		err = mthca_table_get(dev, dev->cq_table.table, cq->cqn);
+		if (err)
+			goto err_out;
+
+		err = -ENOMEM;
+
+		cq->set_ci_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_SET_CI,
+						     cq->cqn, &cq->set_ci_db);
+		if (cq->set_ci_db_index < 0)
+			goto err_out_icm;
+
+		cq->arm_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_ARM,
+						  cq->cqn, &cq->arm_db);
+		if (cq->arm_db_index < 0)
+			goto err_out_ci;
+	}
+
 	mailbox = kmalloc(sizeof (struct mthca_cq_context) + MTHCA_CMD_MAILBOX_EXTRA,
 			  GFP_KERNEL);
 	if (!mailbox)
-		goto err_out;
+		goto err_out_mailbox;
 
 	cq_context = MAILBOX_ALIGN(mailbox);
 

@@ -716,6 +739,11 @@
 	cq_context->lkey            = cpu_to_be32(cq->mr.ibmr.lkey);
 	cq_context->cqn             = cpu_to_be32(cq->cqn);
 
+	if (dev->hca_type == ARBEL_NATIVE) {
+		cq_context->ci_db    = cpu_to_be32(cq->set_ci_db_index);
+		cq_context->state_db = cpu_to_be32(cq->arm_db_index);
+	}
+
 	err = mthca_SW2HW_CQ(dev, cq_context, cq->cqn, &status);
 	if (err) {
 		mthca_warn(dev, "SW2HW_CQ failed (%d)\n", err);

@@ -751,6 +779,14 @@
 err_out_mailbox:
 	kfree(mailbox);
 
+	mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index);
+
+err_out_ci:
+	mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index);
+
+err_out_icm:
+	mthca_table_put(dev, dev->cq_table.table, cq->cqn);
+
 err_out:
 	mthca_free(&dev->cq_table.alloc, cq->cqn);
 

@@ -806,6 +842,12 @@
 	mthca_free_mr(dev, &cq->mr);
 	mthca_free_cq_buf(dev, cq);
 
+	if (dev->hca_type == ARBEL_NATIVE) {
+		mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM,    cq->arm_db_index);
+		mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index);
+		mthca_table_put(dev, dev->cq_table.table, cq->cqn);
+	}
+
 	mthca_free(&dev->cq_table.alloc, cq->cqn);
 	kfree(mailbox);
 }

@@ -143,6 +143,14 @@
 	int                    cqn;
 	int                    cons_index;
 	int                    is_direct;
+
+	/* Next fields are Arbel only */
+	int                    set_ci_db_index;
+	u32                   *set_ci_db;
+	int                    arm_db_index;
+	u32                   *arm_db;
+	int                    arm_sn;
+
 	union {
 		struct mthca_buf_list direct;
 		struct mthca_buf_list *page_list;

@@ -136,11 +136,15 @@
 #define MTHCA_CQ_ENTRY_OWNER_SW      (0 << 7)
 #define MTHCA_CQ_ENTRY_OWNER_HW      (1 << 7)
 
-#define MTHCA_CQ_DB_INC_CI       (1 << 24)
-#define MTHCA_CQ_DB_REQ_NOT      (2 << 24)
-#define MTHCA_CQ_DB_REQ_NOT_SOL  (3 << 24)
-#define MTHCA_CQ_DB_SET_CI       (4 << 24)
-#define MTHCA_CQ_DB_REQ_NOT_MULT (5 << 24)
+#define MTHCA_TAVOR_CQ_DB_INC_CI       (1 << 24)
+#define MTHCA_TAVOR_CQ_DB_REQ_NOT      (2 << 24)
+#define MTHCA_TAVOR_CQ_DB_REQ_NOT_SOL  (3 << 24)
+#define MTHCA_TAVOR_CQ_DB_SET_CI       (4 << 24)
+#define MTHCA_TAVOR_CQ_DB_REQ_NOT_MULT (5 << 24)
+
+#define MTHCA_ARBEL_CQ_DB_REQ_NOT_SOL  (1 << 24)
+#define MTHCA_ARBEL_CQ_DB_REQ_NOT      (2 << 24)
+#define MTHCA_ARBEL_CQ_DB_REQ_NOT_MULT (3 << 24)
 
 static inline struct mthca_cqe *get_cqe(struct mthca_cq *cq, int entry)
 {

@@ -159,7 +163,7 @@
 
 static inline struct mthca_cqe *next_cqe_sw(struct mthca_cq *cq)
 {
-	return cqe_sw(cq, cq->cons_index);
+	return cqe_sw(cq, cq->cons_index & cq->ibcq.cqe);
 }
 
 static inline void set_cqe_hw(struct mthca_cqe *cqe)

@@ -167,17 +171,26 @@
 	cqe->owner = MTHCA_CQ_ENTRY_OWNER_HW;
 }
 
-static inline void inc_cons_index(struct mthca_dev *dev, struct mthca_cq *cq,
-				  int nent)
+/*
+ * incr is ignored in native Arbel (mem-free) mode, so cq->cons_index
+ * should be correct before calling update_cons_index().
+ */
+static inline void update_cons_index(struct mthca_dev *dev, struct mthca_cq *cq,
+				     int incr)
 {
 	u32 doorbell[2];
 
-	doorbell[0] = cpu_to_be32(MTHCA_CQ_DB_INC_CI | cq->cqn);
-	doorbell[1] = cpu_to_be32(nent - 1);
+	if (dev->hca_type == ARBEL_NATIVE) {
+		*cq->set_ci_db = cpu_to_be32(cq->cons_index);
+		wmb();
+	} else {
+		doorbell[0] = cpu_to_be32(MTHCA_TAVOR_CQ_DB_INC_CI | cq->cqn);
+		doorbell[1] = cpu_to_be32(incr - 1);
 
-	mthca_write64(doorbell,
-		      dev->kar + MTHCA_CQ_DOORBELL,
-		      MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
+		mthca_write64(doorbell,
+			      dev->kar + MTHCA_CQ_DOORBELL,
+			      MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
+	}
 }
 
 void mthca_cq_event(struct mthca_dev *dev, u32 cqn)

@@ -191,6 +204,8 @@
 		return;
 	}
 
+	++cq->arm_sn;
+
 	cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
 

@@ -247,8 +262,8 @@
 
 	if (nfreed) {
 		wmb();
-		inc_cons_index(dev, cq, nfreed);
-		cq->cons_index = (cq->cons_index + nfreed) & cq->ibcq.cqe;
+		cq->cons_index += nfreed;
+		update_cons_index(dev, cq, nfreed);
 	}
 
 	spin_unlock_irq(&cq->lock);

@@ -341,7 +356,7 @@
 		break;
 	}
 
-	err = mthca_free_err_wqe(qp, is_send, wqe_index, &dbd, &new_wqe);
+	err = mthca_free_err_wqe(dev, qp, is_send, wqe_index, &dbd, &new_wqe);
 	if (err)
 		return err;
 

@@ -411,7 +426,7 @@
 		if (*cur_qp) {
 			if (*freed) {
 				wmb();
-				inc_cons_index(dev, cq, *freed);
+				update_cons_index(dev, cq, *freed);
 				*freed = 0;
 			}
 			spin_unlock(&(*cur_qp)->lock);

@@ -505,7 +520,7 @@
 	if (likely(free_cqe)) {
 		set_cqe_hw(cqe);
 		++(*freed);
-		cq->cons_index = (cq->cons_index + 1) & cq->ibcq.cqe;
+		++cq->cons_index;
 	}
 
 	return err;

@@ -533,7 +548,7 @@
 
 	if (freed) {
 		wmb();
-		inc_cons_index(dev, cq, freed);
+		update_cons_index(dev, cq, freed);
 	}
 
 	if (qp)

@@ -544,20 +559,57 @@
 	return err == 0 || err == -EAGAIN ? npolled : err;
 }
 
-void mthca_arm_cq(struct mthca_dev *dev, struct mthca_cq *cq,
-		  int solicited)
+int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify)
 {
 	u32 doorbell[2];
 
-	doorbell[0] =  cpu_to_be32((solicited ?
-				    MTHCA_CQ_DB_REQ_NOT_SOL :
-				    MTHCA_CQ_DB_REQ_NOT)      |
-				   cq->cqn);
+	doorbell[0] = cpu_to_be32((notify == IB_CQ_SOLICITED ?
+				   MTHCA_TAVOR_CQ_DB_REQ_NOT_SOL :
+				   MTHCA_TAVOR_CQ_DB_REQ_NOT)      |
+				  to_mcq(cq)->cqn);
 	doorbell[1] = 0xffffffff;
 
 	mthca_write64(doorbell,
-		      dev->kar + MTHCA_CQ_DOORBELL,
-		      MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));
+		      to_mdev(cq->device)->kar + MTHCA_CQ_DOORBELL,
+		      MTHCA_GET_DOORBELL_LOCK(&to_mdev(cq->device)->doorbell_lock));
+
+	return 0;
+}
+
+int mthca_arbel_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify notify)
+{
+	struct mthca_cq *cq = to_mcq(ibcq);
+	u32 doorbell[2];
+	u32 sn;
+	u32 ci;
+
+	sn = cq->arm_sn & 3;
+	ci = cpu_to_be32(cq->cons_index);
+
+	doorbell[0] = ci;
+	doorbell[1] = cpu_to_be32((cq->cqn << 8) | (2 << 5) | (sn << 3) |
+				  (notify == IB_CQ_SOLICITED ? 1 : 2));
+
+	mthca_write_db_rec(doorbell, cq->arm_db);
+
+	/*
+	 * Make sure that the doorbell record in host memory is
+	 * written before ringing the doorbell via PCI MMIO.
+	 */
+	wmb();
+
+	doorbell[0] = cpu_to_be32((sn << 28)                       |
+				  (notify == IB_CQ_SOLICITED ?
+				   MTHCA_ARBEL_CQ_DB_REQ_NOT_SOL :
+				   MTHCA_ARBEL_CQ_DB_REQ_NOT)      |
+				  cq->cqn);
+	doorbell[1] = ci;
+
+	mthca_write64(doorbell,
+		      to_mdev(ibcq->device)->kar + MTHCA_CQ_DOORBELL,
+		      MTHCA_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->doorbell_lock));
+
+	return 0;
 }
 
 static void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq *cq)

@@ -368,8 +368,8 @@
 
 int mthca_poll_cq(struct ib_cq *ibcq, int num_entries,
 		  struct ib_wc *entry);
-void mthca_arm_cq(struct mthca_dev *dev, struct mthca_cq *cq,
-		  int solicited);
+int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify);
+int mthca_arbel_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify);
 int mthca_init_cq(struct mthca_dev *dev, int nent,
 		  struct mthca_cq *cq);
 void mthca_free_cq(struct mthca_dev *dev,

@@ -384,7 +384,7 @@
 		    struct ib_send_wr **bad_wr);
 int mthca_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 		       struct ib_recv_wr **bad_wr);
-int mthca_free_err_wqe(struct mthca_qp *qp, int is_send,
+int mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send,
 		       int index, int *dbd, u32 *new_wqe);
 int mthca_alloc_qp(struct mthca_dev *dev,
 		   struct mthca_pd *pd,

@@ -421,13 +421,6 @@
 	return 0;
 }
 
-static int mthca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify notify)
-{
-	mthca_arm_cq(to_mdev(cq->device), to_mcq(cq),
-		     notify == IB_CQ_SOLICITED);
-	return 0;
-}
-
 static inline u32 convert_access(int acc)
 {
 	return (acc & IB_ACCESS_REMOTE_ATOMIC ? MTHCA_MPT_FLAG_ATOMIC       : 0) |

@@ -625,7 +618,6 @@
 	dev->ib_dev.create_cq            = mthca_create_cq;
 	dev->ib_dev.destroy_cq           = mthca_destroy_cq;
 	dev->ib_dev.poll_cq              = mthca_poll_cq;
-	dev->ib_dev.req_notify_cq        = mthca_req_notify_cq;
 	dev->ib_dev.get_dma_mr           = mthca_get_dma_mr;
 	dev->ib_dev.reg_phys_mr          = mthca_reg_phys_mr;
 	dev->ib_dev.dereg_mr             = mthca_dereg_mr;

@@ -633,6 +625,11 @@
 	dev->ib_dev.detach_mcast         = mthca_multicast_detach;
 	dev->ib_dev.process_mad          = mthca_process_mad;
 
+	if (dev->hca_type == ARBEL_NATIVE)
+		dev->ib_dev.req_notify_cq = mthca_arbel_arm_cq;
+	else
+		dev->ib_dev.req_notify_cq = mthca_tavor_arm_cq;
+
 	init_MUTEX(&dev->cap_mask_mutex);
 
 	ret = ib_register_device(&dev->ib_dev);

@@ -141,7 +141,7 @@
 	spinlock_t             lock;
 	atomic_t               refcount;
 	int                    cqn;
-	int                    cons_index;
+	u32                    cons_index;
 	int                    is_direct;
 
 	/* Next fields are Arbel only */

@@ -1551,7 +1551,7 @@
 	return err;
 }
 
-int mthca_free_err_wqe(struct mthca_qp *qp, int is_send,
+int mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send,
 		       int index, int *dbd, u32 *new_wqe)
 {
 	struct mthca_next_seg *next;

@@ -1561,7 +1561,10 @@
 	else
 		next = get_recv_wqe(qp, index);
 
-	*dbd = !!(next->ee_nds & cpu_to_be32(MTHCA_NEXT_DBD));
+	if (dev->hca_type == ARBEL_NATIVE)
+		*dbd = 1;
+	else
+		*dbd = !!(next->ee_nds & cpu_to_be32(MTHCA_NEXT_DBD));
 	if (next->ee_nds & cpu_to_be32(0x3f))
 		*new_wqe = (next->nda_op & cpu_to_be32(~0x3f)) |
 			(next->ee_nds & cpu_to_be32(0x3f));

@@ -167,6 +167,9 @@
 	void *last;
 	int   max_gs;
 	int   wqe_shift;
+
+	int   db_index;		/* Arbel only */
+	u32  *db;
 };
 
 struct mthca_qp {

335 lines not on this page · raw message

@@ -60,27 +60,34 @@
 	u32 index = -1;
 	struct mthca_av *av = NULL;
 
-	ah->on_hca = 0;
+	ah->type = MTHCA_AH_PCI_POOL;
 
-	if (!atomic_read(&pd->sqp_count) &&
-	    !(dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) {
+	if (dev->hca_type == ARBEL_NATIVE) {
+		ah->av   = kmalloc(sizeof *ah->av, GFP_KERNEL);
+		if (!ah->av)
+			return -ENOMEM;
+
+		ah->type = MTHCA_AH_KMALLOC;
+		av       = ah->av;
+	} else if (!atomic_read(&pd->sqp_count) &&
+		 !(dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) {
 		index = mthca_alloc(&dev->av_table.alloc);
 
 		/* fall back to allocate in host memory */
 		if (index == -1)
-			goto host_alloc;
+			goto on_hca_fail;
 
 		av = kmalloc(sizeof *av, GFP_KERNEL);
 		if (!av)
-			goto host_alloc;
+			goto on_hca_fail;
 
-		ah->on_hca = 1;
+		ah->type = MTHCA_AH_ON_HCA;
 		ah->avdma  = dev->av_table.ddr_av_base +
 			index * MTHCA_AV_SIZE;
 	}
 
- host_alloc:
-	if (!ah->on_hca) {
+on_hca_fail:
+	if (ah->type == MTHCA_AH_PCI_POOL) {
 		ah->av = pci_pool_alloc(dev->av_table.pool,
 					SLAB_KERNEL, &ah->avdma);
 		if (!ah->av)

@@ -123,7 +130,7 @@
 			       j * 4, be32_to_cpu(((u32 *) av)[j]));
 	}
 
-	if (ah->on_hca) {
+	if (ah->type == MTHCA_AH_ON_HCA) {
 		memcpy_toio(dev->av_table.av_map + index * MTHCA_AV_SIZE,
 			    av, MTHCA_AV_SIZE);
 		kfree(av);

@@ -134,12 +141,21 @@
 
 int mthca_destroy_ah(struct mthca_dev *dev, struct mthca_ah *ah)
 {
-	if (ah->on_hca)
+	switch (ah->type) {
+	case MTHCA_AH_ON_HCA:
 		mthca_free(&dev->av_table.alloc,
  			   (ah->avdma - dev->av_table.ddr_av_base) /
 			   MTHCA_AV_SIZE);
-	else
+		break;
+
+	case MTHCA_AH_PCI_POOL:
 		pci_pool_free(dev->av_table.pool, ah->av, ah->avdma);
+		break;
+
+	case MTHCA_AH_KMALLOC:
+		kfree(ah->av);
+		break;
+	}
 
 	return 0;
 }

@@ -147,7 +163,7 @@
 int mthca_read_ah(struct mthca_dev *dev, struct mthca_ah *ah,
 		  struct ib_ud_header *header)
 {
-	if (ah->on_hca)
+	if (ah->type == MTHCA_AH_ON_HCA)
 		return -EINVAL;
 
 	header->lrh.service_level   = be32_to_cpu(ah->av->sl_tclass_flowlabel) >> 28;

@@ -176,6 +192,9 @@
 {
 	int err;
 
+	if (dev->hca_type == ARBEL_NATIVE)
+		return 0;
+
 	err = mthca_alloc_init(&dev->av_table.alloc,
 			       dev->av_table.num_ddr_avs,
 			       dev->av_table.num_ddr_avs - 1,

@@ -212,6 +231,9 @@
 
 void __devexit mthca_cleanup_av_table(struct mthca_dev *dev)
 {
+	if (dev->hca_type == ARBEL_NATIVE)
+		return;
+
 	if (dev->av_table.av_map)
 		iounmap(dev->av_table.av_map);
 	pci_pool_destroy(dev->av_table.pool);

@@ -82,12 +82,18 @@
 
 struct mthca_av;
 
+enum mthca_ah_type {
+	MTHCA_AH_ON_HCA,
+	MTHCA_AH_PCI_POOL,
+	MTHCA_AH_KMALLOC
+};
+
 struct mthca_ah {
-	struct ib_ah     ibah;
-	int              on_hca;
-	u32              key;
-	struct mthca_av *av;
-	dma_addr_t       avdma;
+	struct ib_ah       ibah;
+	enum mthca_ah_type type;
+	u32                key;
+	struct mthca_av   *av;
+	dma_addr_t         avdma;
 };
 
 /*

@@ -380,10 +380,14 @@
 void mthca_qp_event(struct mthca_dev *dev, u32 qpn,
 		    enum ib_event_type event_type);
 int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask);
-int mthca_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
-		    struct ib_send_wr **bad_wr);
-int mthca_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
-		       struct ib_recv_wr **bad_wr);
+int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
+			  struct ib_send_wr **bad_wr);
+int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
+			     struct ib_recv_wr **bad_wr);
+int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
+			  struct ib_send_wr **bad_wr);
+int mthca_arbel_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
+			     struct ib_recv_wr **bad_wr);
 int mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send,
 		       int index, int *dbd, u32 *new_wqe);
 int mthca_alloc_qp(struct mthca_dev *dev,

@@ -613,8 +613,6 @@
 	dev->ib_dev.create_qp            = mthca_create_qp;
 	dev->ib_dev.modify_qp            = mthca_modify_qp;
 	dev->ib_dev.destroy_qp           = mthca_destroy_qp;
-	dev->ib_dev.post_send            = mthca_post_send;
-	dev->ib_dev.post_recv            = mthca_post_receive;
 	dev->ib_dev.create_cq            = mthca_create_cq;
 	dev->ib_dev.destroy_cq           = mthca_destroy_cq;
 	dev->ib_dev.poll_cq              = mthca_poll_cq;

@@ -625,10 +623,15 @@
 	dev->ib_dev.detach_mcast         = mthca_multicast_detach;
 	dev->ib_dev.process_mad          = mthca_process_mad;
 
-	if (dev->hca_type == ARBEL_NATIVE)
+	if (dev->hca_type == ARBEL_NATIVE) {
 		dev->ib_dev.req_notify_cq = mthca_arbel_arm_cq;
-	else
+		dev->ib_dev.post_send     = mthca_arbel_post_send;
+		dev->ib_dev.post_recv     = mthca_arbel_post_receive;
+	} else {
 		dev->ib_dev.req_notify_cq = mthca_tavor_arm_cq;
+		dev->ib_dev.post_send     = mthca_tavor_post_send;
+		dev->ib_dev.post_recv     = mthca_tavor_post_receive;
+	}
 
 	init_MUTEX(&dev->cap_mask_mutex);
 

379 lines not on this page · raw message

@@ -207,8 +207,9 @@
 };
 
 struct mthca_mcg_table {
-	struct semaphore   sem;
-	struct mthca_alloc alloc;
+	struct semaphore   	sem;
+	struct mthca_alloc 	alloc;
+	struct mthca_icm_table *table;
 };
 
 struct mthca_dev {

@@ -412,8 +412,29 @@
 		goto err_unmap_eqp;
 	}
 
+	/*
+	 * It's not strictly required, but for simplicity just map the
+	 * whole multicast group table now.  The table isn't very big
+	 * and it's a lot easier than trying to track ref counts.
+	 */
+	mdev->mcg_table.table = mthca_alloc_icm_table(mdev, init_hca->mc_base,
+						      MTHCA_MGM_ENTRY_SIZE,
+						      mdev->limits.num_mgms +
+						      mdev->limits.num_amgms,
+						      mdev->limits.num_mgms +
+						      mdev->limits.num_amgms,
+						      0);
+	if (!mdev->mcg_table.table) {
+		mthca_err(mdev, "Failed to map MCG context memory, aborting.\n");
+		err = -ENOMEM;
+		goto err_unmap_cq;
+	}
+
 	return 0;
 
+err_unmap_cq:
+	mthca_free_icm_table(mdev, mdev->cq_table.table);
+
 err_unmap_eqp:
 	mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);
 

@@ -587,7 +608,7 @@
 		goto err_uar_free;
 	}
 
-       err = mthca_init_pd_table(dev);
+	err = mthca_init_pd_table(dev);
 	if (err) {
 		mthca_err(dev, "Failed to initialize "
 			  "protection domain table, aborting.\n");

@@ -635,13 +656,6 @@
 
 	mthca_dbg(dev, "NOP command IRQ test passed\n");
 
-	if (dev->hca_type == ARBEL_NATIVE) {
-		mthca_warn(dev, "Sorry, native MT25208 mode support is not complete, "
-			   "aborting.\n");
-		err = -ENODEV;
-		goto err_cmd_poll;
-	}
-
 	err = mthca_init_cq_table(dev);
 	if (err) {
 		mthca_err(dev, "Failed to initialize "

@@ -704,7 +718,7 @@
 
 err_uar_table_free:
 	mthca_cleanup_uar_table(dev);
-       return err;
+	return err;
 }
 
 static int __devinit mthca_request_regions(struct pci_dev *pdev,

@@ -814,6 +828,7 @@
 				    const struct pci_device_id *id)
 {
 	static int mthca_version_printed = 0;
+	static int mthca_memfree_warned = 0;
 	int ddr_hidden = 0;
 	int err;
 	struct mthca_dev *mdev;

@@ -893,6 +908,10 @@
 	mdev->pdev     = pdev;
 	mdev->hca_type = id->driver_data;
 
+	if (mdev->hca_type == ARBEL_NATIVE && !mthca_memfree_warned++)
+		mthca_warn(mdev, "Warning: native MT25208 mode support is incomplete.  "
+			   "Your HCA may not work properly.\n");
+
 	if (ddr_hidden)
 		mdev->mthca_flags |= MTHCA_FLAG_DDR_HIDDEN;
 

@@ -423,15 +423,6 @@
 	is_send  = is_error ? cqe->opcode & 0x01 : cqe->is_send & 0x80;
 
 	if (!*cur_qp || be32_to_cpu(cqe->my_qpn) != (*cur_qp)->qpn) {
-		if (*cur_qp) {
-			if (*freed) {
-				wmb();
-				update_cons_index(dev, cq, *freed);
-				*freed = 0;
-			}
-			spin_unlock(&(*cur_qp)->lock);
-		}
-
 		/*
 		 * We do not have to take the QP table lock here,
 		 * because CQs will be locked while QPs are removed

@@ -446,8 +437,6 @@
 			err = -EINVAL;
 			goto out;
 		}
-
-		spin_lock(&(*cur_qp)->lock);
 	}
 
 	entry->qp_num = (*cur_qp)->qpn;

@@ -465,9 +454,9 @@
 	}
 
 	if (wq->last_comp < wqe_index)
-		wq->cur -= wqe_index - wq->last_comp;
+		wq->tail += wqe_index - wq->last_comp;
 	else
-		wq->cur -= wq->max - wq->last_comp + wqe_index;
+		wq->tail += wqe_index + wq->max - wq->last_comp;
 
 	wq->last_comp = wqe_index;
 

@@ -551,9 +540,6 @@
 		update_cons_index(dev, cq, freed);
 	}
 
-	if (qp)
-		spin_unlock(&qp->lock);
-
 	spin_unlock_irqrestore(&cq->lock, flags);
 
 	return err == 0 || err == -EAGAIN ? npolled : err;

@@ -166,21 +166,22 @@
 };
 
 struct mthca_wq {
-	int   max;
-	int   cur;
-	int   next;
-	int   last_comp;
-	void *last;
-	int   max_gs;
-	int   wqe_shift;
+	spinlock_t lock;
+	int        max;
+	unsigned   next_ind;
+	unsigned   last_comp;
+	unsigned   head;
+	unsigned   tail;
+	void      *last;
+	int        max_gs;
+	int        wqe_shift;
 
-	int   db_index;		/* Arbel only */
-	u32  *db;
+	int        db_index;	/* Arbel only */
+	u32       *db;
 };
 
 struct mthca_qp {
 	struct ib_qp           ibqp;
-	spinlock_t             lock;
 	atomic_t               refcount;
 	u32                    qpn;
 	int                    is_direct;

258 lines not on this page · raw message

@@ -95,7 +95,21 @@
 };
 
 enum {
-	DEV_LIM_FLAG_SRQ = 1 << 6
+	DEV_LIM_FLAG_RC                 = 1 << 0,
+	DEV_LIM_FLAG_UC                 = 1 << 1,
+	DEV_LIM_FLAG_UD                 = 1 << 2,
+	DEV_LIM_FLAG_RD                 = 1 << 3,
+	DEV_LIM_FLAG_RAW_IPV6           = 1 << 4,
+	DEV_LIM_FLAG_RAW_ETHER          = 1 << 5,
+	DEV_LIM_FLAG_SRQ                = 1 << 6,
+	DEV_LIM_FLAG_BAD_PKEY_CNTR      = 1 << 8,
+	DEV_LIM_FLAG_BAD_QKEY_CNTR      = 1 << 9,
+	DEV_LIM_FLAG_MW                 = 1 << 16,
+	DEV_LIM_FLAG_AUTO_PATH_MIG      = 1 << 17,
+	DEV_LIM_FLAG_ATOMIC             = 1 << 18,
+	DEV_LIM_FLAG_RAW_MULTI          = 1 << 19,
+	DEV_LIM_FLAG_UD_AV_PORT_ENFORCE = 1 << 20,
+	DEV_LIM_FLAG_UD_MULTI           = 1 << 21,
 };
 
 struct mthca_dev_lim {

@@ -218,6 +218,7 @@
 
 	int          	 hca_type;
 	unsigned long	 mthca_flags;
+	unsigned long    device_cap_flags;
 
 	u32              rev_id;
 

@@ -171,6 +171,33 @@
 	mdev->limits.reserved_uars      = dev_lim->reserved_uars;
 	mdev->limits.reserved_pds       = dev_lim->reserved_pds;
 
+	/* IB_DEVICE_RESIZE_MAX_WR not supported by driver.
+	   May be doable since hardware supports it for SRQ.
+
+	   IB_DEVICE_N_NOTIFY_CQ is supported by hardware but not by driver.
+
+	   IB_DEVICE_SRQ_RESIZE is supported by hardware but SRQ is not
+	   supported by driver. */
+	mdev->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT |
+		IB_DEVICE_PORT_ACTIVE_EVENT |
+		IB_DEVICE_SYS_IMAGE_GUID |
+		IB_DEVICE_RC_RNR_NAK_GEN;
+
+	if (dev_lim->flags & DEV_LIM_FLAG_BAD_PKEY_CNTR)
+		mdev->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
+
+	if (dev_lim->flags & DEV_LIM_FLAG_BAD_QKEY_CNTR)
+		mdev->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
+				
+	if (dev_lim->flags & DEV_LIM_FLAG_RAW_MULTI)
+		mdev->device_cap_flags |= IB_DEVICE_RAW_MULTI;
+
+	if (dev_lim->flags & DEV_LIM_FLAG_AUTO_PATH_MIG)
+		mdev->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
+
+	if (dev_lim->flags & DEV_LIM_FLAG_UD_AV_PORT_ENFORCE)
+		mdev->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE;
+
 	if (dev_lim->flags & DEV_LIM_FLAG_SRQ)
 		mdev->mthca_flags |= MTHCA_FLAG_SRQ;
 

@@ -43,6 +43,8 @@
 	struct ib_smp *in_mad  = NULL;
 	struct ib_smp *out_mad = NULL;
 	int err = -ENOMEM;
+	struct mthca_dev* mdev = to_mdev(ibdev);
+
 	u8 status;
 
 	in_mad  = kmalloc(sizeof *in_mad, GFP_KERNEL);

@@ -50,7 +52,7 @@
 	if (!in_mad || !out_mad)
 		goto out;
 
-	props->fw_ver        = to_mdev(ibdev)->fw_ver;
+	props->fw_ver              = mdev->fw_ver;
 
 	memset(in_mad, 0, sizeof *in_mad);
 	in_mad->base_version       = 1;

@@ -59,7 +61,7 @@
 	in_mad->method         	   = IB_MGMT_METHOD_GET;
 	in_mad->attr_id   	   = IB_SMP_ATTR_NODE_INFO;
 
-	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
+	err = mthca_MAD_IFC(mdev, 1, 1,
 			    1, NULL, NULL, in_mad, out_mad,
 			    &status);
 	if (err)

@@ -69,10 +71,11 @@
 		goto out;
 	}
 
-	props->vendor_id      = be32_to_cpup((u32 *) (out_mad->data + 36)) &
+	props->device_cap_flags = mdev->device_cap_flags;
+	props->vendor_id        = be32_to_cpup((u32 *) (out_mad->data + 36)) &
 		0xffffff;
-	props->vendor_part_id = be16_to_cpup((u16 *) (out_mad->data + 30));
-	props->hw_ver         = be16_to_cpup((u16 *) (out_mad->data + 32));
+	props->vendor_part_id   = be16_to_cpup((u16 *) (out_mad->data + 30));
+	props->hw_ver           = be16_to_cpup((u16 *) (out_mad->data + 32));
 	memcpy(&props->sys_image_guid, out_mad->data +  4, 8);
 	memcpy(&props->node_guid,      out_mad->data + 12, 8);
 

@@ -68,6 +68,7 @@
 static void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
 				    struct ib_mad_send_wc *mad_send_wc);
 static void timeout_sends(void *data);
+static void cancel_sends(void *data);
 static void local_completions(void *data);
 static int solicited_mad(struct ib_mad *mad);
 static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req,

@@ -341,6 +342,8 @@
 	INIT_LIST_HEAD(&mad_agent_priv->local_list);
 	INIT_WORK(&mad_agent_priv->local_work, local_completions,
 		   mad_agent_priv);
+	INIT_LIST_HEAD(&mad_agent_priv->canceled_list);
+	INIT_WORK(&mad_agent_priv->canceled_work, cancel_sends, mad_agent_priv);
 	atomic_set(&mad_agent_priv->refcount, 1);
 	init_waitqueue_head(&mad_agent_priv->wait);
 

@@ -2004,12 +2007,44 @@
 	return NULL;
 }
 
+void cancel_sends(void *data)
+{
+	struct ib_mad_agent_private *mad_agent_priv;
+	struct ib_mad_send_wr_private *mad_send_wr;
+	struct ib_mad_send_wc mad_send_wc;
+	unsigned long flags;
+
+	mad_agent_priv = (struct ib_mad_agent_private *)data;
+
+	mad_send_wc.status = IB_WC_WR_FLUSH_ERR;
+	mad_send_wc.vendor_err = 0;
+
+	spin_lock_irqsave(&mad_agent_priv->lock, flags);
+	while (!list_empty(&mad_agent_priv->canceled_list)) {
+		mad_send_wr = list_entry(mad_agent_priv->canceled_list.next,
+					 struct ib_mad_send_wr_private,
+					 agent_list);
+
+		list_del(&mad_send_wr->agent_list);
+		spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
+
+		mad_send_wc.wr_id = mad_send_wr->wr_id;
+		mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
+						   &mad_send_wc);
+
+		kfree(mad_send_wr);
+		if (atomic_dec_and_test(&mad_agent_priv->refcount))
+			wake_up(&mad_agent_priv->wait);
+		spin_lock_irqsave(&mad_agent_priv->lock, flags);
+	}
+	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
+}
+
 void ib_cancel_mad(struct ib_mad_agent *mad_agent,
 		  u64 wr_id)
 {
 	struct ib_mad_agent_private *mad_agent_priv;
 	struct ib_mad_send_wr_private *mad_send_wr;
-	struct ib_mad_send_wc mad_send_wc;
 	unsigned long flags;
 
 	mad_agent_priv = container_of(mad_agent, struct ib_mad_agent_private,

@@ -2031,19 +2066,12 @@
 	}
 
 	list_del(&mad_send_wr->agent_list);
+	list_add_tail(&mad_send_wr->agent_list, &mad_agent_priv->canceled_list);
 	adjust_timeout(mad_agent_priv);
 	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 
-	mad_send_wc.status = IB_WC_WR_FLUSH_ERR;
-	mad_send_wc.vendor_err = 0;
-	mad_send_wc.wr_id = mad_send_wr->wr_id;
-	mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
-					   &mad_send_wc);
-
-	kfree(mad_send_wr);
-	if (atomic_dec_and_test(&mad_agent_priv->refcount))
-		wake_up(&mad_agent_priv->wait);
-
+	queue_work(mad_agent_priv->qp_info->port_priv->wq,
+		   &mad_agent_priv->canceled_work);
 out:
 	return;
 }

@@ -95,6 +95,8 @@
 	unsigned long timeout;
 	struct list_head local_list;
 	struct work_struct local_work;
+	struct list_head canceled_list;
+	struct work_struct canceled_work;
 
 	atomic_t refcount;
 	wait_queue_head_t wait;