Skip to contentSkip to conversation

mlx4_core: add slave resource allocation

PATCH In review Yevgeny Petrilin 1 participant 1 message Nov 6, 2009

Subsystems (5)
lore
Share & tools
lore
#0 lore raw

PATCH v2 5/25 mlx4_core: add slave resource allocation

From: Liran Liss <liranl-VPRAkNaXOzVS1MOuV/[email protected]>

All QPs/CQs/SRQs/MPTs/MTTs are allocated from shared pools, which are owned by the master. In addition, all backing ICM memory for these objects is managed by the master.
To allow slaves to allocate resources, ICM allocation is separated from the rest of the object state, which is held at the slave.
Slaves can then reserve resource ranges and allocate ICM over the comm channel.

---
 drivers/net/mlx4/cmd.c   |  110 +++++++++++++++++++++++++++++++++
 drivers/net/mlx4/cq.c    |   91 +++++++++++++++++++++-------
 drivers/net/mlx4/mlx4.h  |   27 ++++++++
 drivers/net/mlx4/mr.c    |  125 ++++++++++++++++++++++++++++++++++----
 drivers/net/mlx4/qp.c    |  151 +++++++++++++++++++++++++++++++++-------------
 drivers/net/mlx4/srq.c   |   88 ++++++++++++++++++++-------
 include/linux/mlx4/cmd.h |    2 +
 7 files changed, 496 insertions(+), 98 deletions(-)
drivers/net/mlx4/cmd.c +110 -0
index 717dd50..fe23415 100644
@@ -418,6 +418,100 @@ static int mlx4_ACCESS_MEM(struct mlx4_dev *dev, u64 master_addr,
 					   MLX4_CMD_TIME_CLASS_A);
 }
 
+static int mlx4_RESOURCE_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
+						       struct mlx4_cmd_mailbox *inbox,
+						       struct mlx4_cmd_mailbox *outbox)
+{
+	u32 param1 = *((u32 *) &vhcr->in_param);
+	u32 param2 = *(((u32 *) &vhcr->in_param) + 1);
+	int ret;
+
+#if 0
+	char *res[] = {"QP", "CQ", "SRQ", "MPT", "MTT"};
+	mlx4_warn(dev, "resource wrapper - %s (mode: %s) type:%s param1:%d param2:%d\n",
+			vhcr->op == MLX4_CMD_ALLOC_RES ? "allocate" : "free",
+			vhcr->op_modifier == ICM_RESERVE ? "reserve" :
+				(vhcr->op_modifier == ICM_ALLOC ? "alloc" : "reserve+alloc"),
+			res[vhcr->in_modifier], param1, param2);
+#endif
+
+	vhcr->errno = 0;
+	switch (vhcr->in_modifier) {
+	case RES_QP:
+		switch (vhcr->op_modifier) {
+		case ICM_RESERVE:
+			if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+				vhcr->errno = mlx4_qp_reserve_range(dev, param1, param2, &ret);
+				if (!vhcr->errno)
+					vhcr->out_param = ret;
+			} else {
+				mlx4_qp_release_range(dev, param1, param2);
+			}
+			break;
+		case ICM_ALLOC:
+			if (vhcr->op == MLX4_CMD_ALLOC_RES)
+				vhcr->errno = mlx4_qp_alloc_icm(dev, param1);
+			else
+				mlx4_qp_free_icm(dev, param1);
+			break;
+		default:
+			vhcr->errno = -EINVAL;
+		}
+		break;
+	case RES_CQ:
+		if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+			vhcr->errno = mlx4_cq_alloc_icm(dev, &ret);
+			if (!vhcr->errno)
+				vhcr->out_param = ret;
+		} else
+			mlx4_cq_free_icm(dev, param1);
+		break;
+	case RES_SRQ:
+		if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+			vhcr->errno = mlx4_srq_alloc_icm(dev, &ret);
+			if (!vhcr->errno)
+				vhcr->out_param = ret;
+		} else
+			mlx4_srq_free_icm(dev, param1);
+		break;
+	case RES_MPT:
+		switch (vhcr->op_modifier) {
+		case ICM_RESERVE:
+			if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+				ret = mlx4_mr_reserve(dev);
+				if (ret == -1)
+					vhcr->errno = -ENOMEM;
+				else
+					vhcr->out_param = ret;
+			} else
+				mlx4_mr_release(dev, param1);
+			break;
+		case ICM_ALLOC:
+			if (vhcr->op == MLX4_CMD_ALLOC_RES)
+				vhcr->errno = mlx4_mr_alloc_icm(dev, param1);
+			else
+				mlx4_mr_free_icm(dev, param1);
+			break;
+		default:
+			vhcr->errno = -EINVAL;
+		}
+		break;
+	case RES_MTT:
+		if (vhcr->op == MLX4_CMD_ALLOC_RES) {
+			ret = mlx4_alloc_mtt_range(dev, param1 /* order */);
+			if (ret == -1)
+				vhcr->errno = -ENOMEM;
+			else
+				vhcr->out_param = ret;
+		} else
+			mlx4_free_mtt_range(dev, param1 /* first */, param2 /* order */);
+		break;
+	default:
+		vhcr->errno = -EINVAL;
+	}
+	return 0;
+}
+
 static struct mlx4_cmd_info {
 	u8 opcode;
 	bool has_inbox;
@@ -462,6 +556,22 @@ static struct mlx4_cmd_info {
 		.verify = NULL,
 		.wrapper = NULL
 	},
+	{
+		.opcode = MLX4_CMD_ALLOC_RES,
+		.has_inbox = false,
+		.has_outbox = false,
+		.out_is_imm = true,
+		.verify = NULL,
+		.wrapper = mlx4_RESOURCE_wrapper
+	},
+	{
+		.opcode = MLX4_CMD_FREE_RES,
+		.has_inbox = false,
+		.has_outbox = false,
+		.out_is_imm = false,
+		.verify = NULL,
+		.wrapper = mlx4_RESOURCE_wrapper
+	},
 
 	{
 		.opcode = MLX4_CMD_SW2HW_MPT,
drivers/net/mlx4/cq.c +70 -21

121 lines not on this page · raw message

drivers/net/mlx4/mlx4.h +27 -0
index 7516730..ace0bce 100644
@@ -94,6 +94,21 @@ enum {
 	MLX4_COMM_CMD_VHCR_POST
 };
 
+enum mlx4_resource {
+	RES_QP,
+	RES_CQ,
+	RES_SRQ,
+	RES_MPT,
+	RES_MTT
+};
+
+enum mlx4_alloc_mode {
+	ICM_RESERVE_AND_ALLOC,
+	ICM_RESERVE,
+	ICM_ALLOC,
+	ICM_MAC_VLAN,
+};
+
 enum {
 	MLX4_MFUNC_MAX_EQES     = 8,
 	MLX4_MFUNC_EQE_MASK     = (MLX4_MFUNC_MAX_EQES - 1)
@@ -422,6 +437,18 @@ void mlx4_cleanup_qp_table(struct mlx4_dev *dev);
 void mlx4_cleanup_srq_table(struct mlx4_dev *dev);
 void mlx4_cleanup_mcg_table(struct mlx4_dev *dev);
 
+int mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn);
+void mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn);
+int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn);
+void mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn);
+int mlx4_srq_alloc_icm(struct mlx4_dev *dev, int *srqn);
+void mlx4_srq_free_icm(struct mlx4_dev *dev, int srqn);
+int mlx4_mr_reserve(struct mlx4_dev *dev);
+void mlx4_mr_release(struct mlx4_dev *dev, u32 index);
+int mlx4_mr_alloc_icm(struct mlx4_dev *dev, u32 index);
+void mlx4_mr_free_icm(struct mlx4_dev *dev, u32 index);
+u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order);
+void mlx4_free_mtt_range(struct mlx4_dev *dev, u32 first_seg, int order);
 int mlx4_WRITE_MTT_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr,
 						 struct mlx4_cmd_mailbox *inbox,
 						 struct mlx4_cmd_mailbox *outbox);
drivers/net/mlx4/mr.c +112 -13

191 lines not on this page · raw message

drivers/net/mlx4/qp.c +108 -43

211 lines not on this page · raw message

drivers/net/mlx4/srq.c +67 -21

122 lines not on this page · raw message

include/linux/mlx4/cmd.h +2 -0
index b84ff08..be2a184 100644
@@ -125,6 +125,8 @@ enum {
 	MLX4_CMD_DUMP_ETH_STATS	 = 0x49,
 
 	/* virtual commands */
+	MLX4_CMD_ALLOC_RES	 = 0x50,
+	MLX4_CMD_FREE_RES	 = 0x51,
 	MLX4_CMD_GET_EVENT	 = 0x52,
 
 	/* debug commands */
Signature · 10 lines
1.5.3.7





--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html