Skip to contentSkip to conversation

mlx4_core: add slave resource allocation

PATCH In review Yevgeny Petrilin 3 participants 3 messages Nov 4, 2009 – Nov 5, 2009 1 message not in the archive

Subsystems (5)
lore
Share & tools
lore
#0 lore raw

PATCH 5/25 mlx4_core: add slave resource allocation

All QPs/CQs/SRQs/MPTs/MTTs are allocated from shared pools, which are owned by the master. In addition, all backing ICM memory for these objects is managed by the master.
To allow slaves to allocate resources, ICM allocation is separated from the rest of the object state, which is held at the slave.
Slaves can then reserve resource ranges and allocate ICM over the comm channel.

---
 drivers/net/mlx4/cmd.c   |   96 +++++++++++++++++++++++++++++
 drivers/net/mlx4/cq.c    |   91 +++++++++++++++++++++-------
 drivers/net/mlx4/mlx4.h  |   27 ++++++++
 drivers/net/mlx4/mr.c    |  125 ++++++++++++++++++++++++++++++++++----
 drivers/net/mlx4/qp.c    |  151 +++++++++++++++++++++++++++++++++-------------
 drivers/net/mlx4/srq.c   |   88 ++++++++++++++++++++-------
 include/linux/mlx4/cmd.h |    2 +
 7 files changed, 482 insertions(+), 98 deletions(-)
drivers/net/mlx4/cmd.c +96 -0
index 96462f7..b1701d2 100644
@@ -418,6 +418,100 @@ int mlx4_ACCESS_MEM(struct mlx4_dev *dev, u64 master_addr, int slave, u64 slave_
 					   MLX4_CMD_TIME_CLASS_A);
 }
 
+static int mlx4_RESOURCE_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+						       struct mlx4_cmd_mailbox *inbox,
+						       struct mlx4_cmd_mailbox *outbox)
+{
+	u32 param1 = *((u32 *) &vhcr->in_param);
+	u32 param2 = *(((u32 *) &vhcr->in_param) + 1);
+	int ret;
+
+#if 0
+	char *res[] = {"QP", "CQ", "SRQ", "MPT", "MTT"};
+	mlx4_warn(dev, "resource wrapper - %s (mode: %s) type:%s param1:%d param2:%d\n",
+			vhcr->op == MLX4_CMD_ALLOC_RES ? "allocate" : "free",
+			vhcr->op_modifier == ICM_RESERVE ? "reserve" :
+				(vhcr->op_modifier == ICM_ALLOC ? "alloc" : "reserve+alloc"),
+			res[vhcr->in_modifier], param1, param2);
+#endif
+
+	vhcr->errno = 0;
+	switch (vhcr->in_modifier) {
+	case RES_QP:
+		switch (vhcr->op_modifier) {
+		case ICM_RESERVE:
+			if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+				vhcr->errno = mlx4_qp_reserve_range(dev, param1, param2, &ret);
+				if (!vhcr->errno)
+					vhcr->out_param = ret;
+			} else {
+				mlx4_qp_release_range(dev, param1, param2);
+			}
+			break;
+		case ICM_ALLOC:
+			if (vhcr->op == MLX4_CMD_ALLOC_RES)
+				vhcr->errno = mlx4_qp_alloc_icm(dev, param1);
+			else
+				mlx4_qp_free_icm(dev, param1);
+			break;
+		default:
+			vhcr->errno = -EINVAL;
+		}
+		break;
+	case RES_CQ:
+		if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+			vhcr->errno = mlx4_cq_alloc_icm(dev, &ret);
+			if (!vhcr->errno)
+				vhcr->out_param = ret;
+		} else
+			mlx4_cq_free_icm(dev, param1);
+		break;
+	case RES_SRQ:
+		if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+			vhcr->errno = mlx4_srq_alloc_icm(dev, &ret);
+			if (!vhcr->errno)
+				vhcr->out_param = ret;
+		} else
+			mlx4_srq_free_icm(dev, param1);
+		break;
+	case RES_MPT:
+		switch (vhcr->op_modifier) {
+		case ICM_RESERVE:
+			if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+				ret = mlx4_mr_reserve(dev);
+				if (ret == -1)
+					vhcr->errno = -ENOMEM;
+				else
+					vhcr->out_param = ret;
+			} else
+				mlx4_mr_release(dev, param1);
+			break;
+		case ICM_ALLOC:
+			if (vhcr->op == MLX4_CMD_ALLOC_RES)
+				vhcr->errno = mlx4_mr_alloc_icm(dev, param1);
+			else
+				mlx4_mr_free_icm(dev, param1);
+			break;
+		default:
+			vhcr->errno = -EINVAL;
+		}
+		break;
+	case RES_MTT:
+		if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+			ret = mlx4_alloc_mtt_range(dev, param1 /* order */);
+			if (ret == -1)
+				vhcr->errno = -ENOMEM;
+			else
+				vhcr->out_param = ret;
+		} else
+			mlx4_free_mtt_range(dev, param1 /* first */, param2 /* order */);
+		break;
+	default:
+		vhcr->errno = -EINVAL;
+	}
+	return 0;
+}
+
 static struct mlx4_cmd_info {
 	u8 opcode;
 	u8 has_inbox;
@@ -434,6 +528,8 @@ static struct mlx4_cmd_info {
 
 	{MLX4_CMD_SW2HW_EQ,        1, 0, 0, NULL, NULL}, /* need verifier */
 	{MLX4_CMD_NOP,             0, 0, 0, NULL, NULL},
+	{MLX4_CMD_ALLOC_RES,	   0, 0, 1, NULL, mlx4_RESOURCE_wrapper},
+	{MLX4_CMD_FREE_RES,	   0, 0, 0, NULL, mlx4_RESOURCE_wrapper},
 
 	{MLX4_CMD_SW2HW_MPT,       1, 0, 0, NULL, NULL}, /* need verifier */
 	{MLX4_CMD_QUERY_MPT,       0, 1, 0, NULL, NULL}, /* need verifier */
drivers/net/mlx4/cq.c +70 -21

121 lines not on this page · raw message

drivers/net/mlx4/mlx4.h +27 -0
index 4796982..91f1215 100644
@@ -94,6 +94,21 @@ enum {
 	MLX4_COMM_CMD_VHCR_POST
 };
 
+enum mlx4_resource {
+	RES_QP,
+	RES_CQ,
+	RES_SRQ,
+	RES_MPT,
+	RES_MTT
+};
+
+enum mlx4_alloc_mode {
+	ICM_RESERVE_AND_ALLOC,
+	ICM_RESERVE,
+	ICM_ALLOC,
+	ICM_MAC_VLAN,
+};
+
 enum {
 	MLX4_MFUNC_MAX_EQES     = 8,
 	MLX4_MFUNC_EQE_MASK     = (MLX4_MFUNC_MAX_EQES - 1)
@@ -422,6 +437,18 @@ void mlx4_cleanup_qp_table(struct mlx4_dev *dev);
 void mlx4_cleanup_srq_table(struct mlx4_dev *dev);
 void mlx4_cleanup_mcg_table(struct mlx4_dev *dev);
 
+int mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn);
+void mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn);
+int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn);
+void mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn);
+int mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn);
+void mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn);
+int mlx4_mr_reserve(struct mlx4_dev *dev);
+void mlx4_mr_release(struct mlx4_dev *dev, u32 index);
+int mlx4_mr_alloc_icm(struct mlx4_dev *dev, u32 index);
+void mlx4_mr_free_icm(struct mlx4_dev *dev, u32 index);
+u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order);
+void mlx4_free_mtt_range(struct mlx4_dev *dev, u32 first_seg, int order);
 int mlx4_WRITE_MTT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
 						 struct mlx4_cmd_mailbox *inbox,
 						 struct mlx4_cmd_mailbox *outbox);
drivers/net/mlx4/mr.c +112 -13

191 lines not on this page · raw message

drivers/net/mlx4/qp.c +108 -43

211 lines not on this page · raw message

drivers/net/mlx4/srq.c +67 -21

122 lines not on this page · raw message

include/linux/mlx4/cmd.h +2 -0
index dda1762..dceb93f 100644
@@ -120,6 +120,8 @@ enum {
 	MLX4_CMD_ACCESS_MEM	 = 0x2e,
 
 	/* virtual commands */
+	MLX4_CMD_ALLOC_RES	 = 0x50,
+	MLX4_CMD_FREE_RES	 = 0x51,
 	MLX4_CMD_GET_EVENT	 = 0x52,
 
 	/* debug commands */
Signature · 1 line
1.6.1.3
#1 lore raw

drivers/net/mlx4/cmd.c int mlx4_ACCESS_MEM(struct mlx4_dev *dev, u64 master_addr, int slave, u64 slave_ · line 426

Quoted 3 lines
drivers/net/mlx4/cmd.c +2 -0
+	u32 param1 = *((u32 *) &vhcr->in_param);
+	u32 param2 = *(((u32 *) &vhcr->in_param) + 1);

Is this endian clean?

- R.

A message that is not in the archive (sent off-list or not yet received): [email protected]

#3 lore raw

drivers/net/mlx4/cmd.c int mlx4_ACCESS_MEM(struct mlx4_dev *dev, u64 master_addr, int slave, u64 slave_ · line 426

Quoted 2 lines
drivers/net/mlx4/cmd.c +2 -0
+	u32 param1 = *((u32 *) &vhcr->in_param);
+	u32 param2 = *(((u32 *) &vhcr->in_param) + 1);

Is this endian clean?

LL: Yes - the master and slave always run on the same arch...

- R.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html