Skip to contentSkip to conversation

RDMA/hns: Support RoCE on virtual functions of HIP09

PATCH Applied Weihang Li 2 participants 8 messages Apr 6, 2021 – Apr 13, 2021

Subsystems (2)
lore
Share & tools
lore

Weihang Li

author
#0 lore raw

v2 cover RDMA/hns: Support RoCE on virtual functions of HIP09

This series first do necessary refactor on function's resource code to reduce the complexity of subsequent patches. Then the RoCE support for VFs is introduced.

Changes since v1:
- Simplify hr_reg_read() by FIELD_GET in #1.
- Link: https://patchwork.kernel.org/project/linux-rdma/cover/[email protected]/

Wei Xu (4):
RDMA/hns: Query the number of functions supported by the PF RDMA/hns: Reserve the resource for the VFs RDMA/hns: Set parameters of all the functions belong to a PF RDMA/hns: Enable RoCE on virtual functions

Xi Wang (2):
RDMA/hns: Simplify function's resource related command RDMA/hns: Remove duplicated hem page size config code

 drivers/infiniband/hw/hns/hns_roce_common.h |  10 +
 drivers/infiniband/hw/hns/hns_roce_device.h |   6 +-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 553 ++++++++++++++++------------
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h  | 236 +++---------
 4 files changed, 385 insertions(+), 420 deletions(-)
Signature · 1 line
2.8.1
author
#1 lore raw

PATCH v2 1/6 RDMA/hns: Simplify function's resource related command

From: Xi Wang <[email protected]>

Use hr_reg_write/read() to simplify codes about configuring function's resource. And because the design of PF/VF fields is same, they can be defined only once.

---
 drivers/infiniband/hw/hns/hns_roce_common.h |  10 ++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 161 ++++++--------------
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h  | 223 +++++-----------------------
 3 files changed, 89 insertions(+), 305 deletions(-)
drivers/infiniband/hw/hns/hns_roce_common.h +10 -0
index 23c438c..27df969 100644
@@ -87,6 +87,16 @@
 
 #define hr_reg_write(ptr, field, val) _hr_reg_write(ptr, field, val)
 
+#define _hr_reg_read(ptr, field_type, field_h, field_l)                        \
+	({                                                                     \
+		const field_type *_ptr = ptr;                                  \
+		BUILD_BUG_ON(((field_h) / 32) != ((field_l) / 32));            \
+		FIELD_GET(GENMASK((field_h) % 32, (field_l) % 32),             \
+			  le32_to_cpu(*((__le32 *)_ptr + (field_h) / 32)));    \
+	})
+
+#define hr_reg_read(ptr, field) _hr_reg_read(ptr, field)
+
 #define ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S 3
 #define ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S 4
 
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +43 -118

208 lines not on this page · raw message

drivers/infiniband/hw/hns/hns_roce_hw_v2.h +36 -187

230 lines not on this page · raw message

Signature · 1 line
2.8.1
author
#2 lore raw

PATCH v2 2/6 RDMA/hns: Query the number of functions supported by the PF

In reply to Weihang Li

From: Wei Xu <[email protected]>

Query how many functions are supported by the PF from the FW and store it in the hns_roce_dev structure which will be used to support the configuration of virtual functions.

---
 drivers/infiniband/hw/hns/hns_roce_device.h | 1 +
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 9 +++++++--
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h  | 4 ++--
 3 files changed, 10 insertions(+), 4 deletions(-)
drivers/infiniband/hw/hns/hns_roce_device.h +1 -0
index 55cbbd5..3a5378e 100644
@@ -1006,6 +1006,7 @@ struct hns_roce_dev {
 	void			*priv;
 	struct workqueue_struct *irq_workq;
 	const struct hns_roce_dfx_hw *dfx;
+	u32 func_num;
 	u32 cong_algo_tmpl_id;
 };
 
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +7 -2
index 0695c2c..5c0d341 100644
@@ -1607,15 +1607,20 @@ static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev)
 	struct hns_roce_cmq_desc desc;
 	int ret;
 
-	if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09)
+	if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09) {
+		hr_dev->func_num = 1;
 		return 0;
+	}
 
 	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_FUNC_INFO,
 				      true);
 	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
-	if (ret)
+	if (ret) {
+		hr_dev->func_num = 1;
 		return ret;
+	}
 
+	hr_dev->func_num = le32_to_cpu(desc.func_info.own_func_num);
 	hr_dev->cong_algo_tmpl_id = le32_to_cpu(desc.func_info.own_mac_id);
 
 	return 0;
drivers/infiniband/hw/hns/hns_roce_hw_v2.h +2 -2
index 44be39a..535587f 100644
@@ -1697,9 +1697,9 @@ struct hns_roce_cmq_desc {
 	union {
 		__le32 data[6];
 		struct {
-			__le32 rsv1;
+			__le32 own_func_num;
 			__le32 own_mac_id;
-			__le32 rsv2[4];
+			__le32 rsv[4];
 		} func_info;
 	};
 
Signature · 1 line
2.8.1
author
#3 lore raw

PATCH v2 3/6 RDMA/hns: Reserve the resource for the VFs

In reply to Weihang Li

From: Wei Xu <[email protected]>

Query the resource including EQC/SMAC/SGID from the firmware in the PF and distribute fairly among all the functions belong to the PF.

---
 drivers/infiniband/hw/hns/hns_roce_device.h |  3 ++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 83 ++++++++++++++++++++---------
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h  |  2 -
 3 files changed, 60 insertions(+), 28 deletions(-)
drivers/infiniband/hw/hns/hns_roce_device.h +3 -0
index 3a5378e..37d6f01 100644
@@ -809,6 +809,9 @@ struct hns_roce_caps {
 	u32		cqc_bt_num;
 	u32		cqc_timer_bt_num;
 	u32		mpt_bt_num;
+	u32		eqc_bt_num;
+	u32		smac_bt_num;
+	u32		sgid_bt_num;
 	u32		sccc_bt_num;
 	u32		gmv_bt_num;
 	u32		qpc_ba_pg_sz;
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +57 -26

117 lines not on this page · raw message

drivers/infiniband/hw/hns/hns_roce_hw_v2.h +0 -2
index 535587f..efd33ae 100644
@@ -40,9 +40,7 @@
 #define HNS_ROCE_VF_SRQC_BT_NUM			64
 #define HNS_ROCE_VF_CQC_BT_NUM			64
 #define HNS_ROCE_VF_MPT_BT_NUM			64
-#define HNS_ROCE_VF_EQC_NUM			64
 #define HNS_ROCE_VF_SMAC_NUM			32
-#define HNS_ROCE_VF_SGID_NUM			32
 #define HNS_ROCE_VF_SL_NUM			8
 #define HNS_ROCE_VF_GMV_BT_NUM			256
 
Signature · 1 line
2.8.1
author
#4 lore raw

PATCH v2 4/6 RDMA/hns: Set parameters of all the functions belong to a PF

In reply to Weihang Li

From: Wei Xu <[email protected]>

Switch parameters of all functions belong to a PF should be set including VFs.

---
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +16 -2
index 4cc08aa..860e74b 100644
@@ -1693,7 +1693,8 @@ static int hns_roce_query_pf_timer_resource(struct hns_roce_dev *hr_dev)
 	return 0;
 }
 
-static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, int vf_id)
+static int __hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev,
+					  u32 vf_id)
 {
 	struct hns_roce_cmq_desc desc;
 	struct hns_roce_vf_switch *swt;
@@ -1718,6 +1719,19 @@ static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev, int vf_id)
 	return hns_roce_cmq_send(hr_dev, &desc, 1);
 }
 
+static int hns_roce_set_vf_switch_param(struct hns_roce_dev *hr_dev)
+{
+	u32 vf_id;
+	int ret;
+
+	for (vf_id = 0; vf_id < hr_dev->func_num; vf_id++) {
+		ret = __hns_roce_set_vf_switch_param(hr_dev, vf_id);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
 static int __hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev, int vf_id)
 {
 	struct hns_roce_cmq_desc desc[2];
@@ -2319,7 +2333,7 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
 		return ret;
 	}
 
-	ret = hns_roce_set_vf_switch_param(hr_dev, 0);
+	ret = hns_roce_set_vf_switch_param(hr_dev);
 	if (ret) {
 		dev_err(hr_dev->dev,
 			"failed to set function switch param, ret = %d.\n",
Signature · 1 line
2.8.1
author
#5 lore raw

PATCH v2 5/6 RDMA/hns: Enable RoCE on virtual functions

In reply to Weihang Li

From: Wei Xu <[email protected]>

Introduce the VF support by adding code changes to allow VF PCI device initialization, assgining the reserved resource of the PF to the active VFs, setting the default abilities, applying the interruptions, resetting and reducing the default QP/GID number to aovid exceeding the hardware limitation.

---
 drivers/infiniband/hw/hns/hns_roce_device.h |   1 +
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 232 +++++++++++++++++++++++-----
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h  |   8 +-
 3 files changed, 202 insertions(+), 39 deletions(-)
drivers/infiniband/hw/hns/hns_roce_device.h +1 -0
index 37d6f01..e631914 100644
@@ -1010,6 +1010,7 @@ struct hns_roce_dev {
 	struct workqueue_struct *irq_workq;
 	const struct hns_roce_dfx_hw *dfx;
 	u32 func_num;
+	u32 is_vf;
 	u32 cong_algo_tmpl_id;
 };
 
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +196 -36

427 lines not on this page · raw message

drivers/infiniband/hw/hns/hns_roce_hw_v2.h +5 -3
index efd33ae..7a25ffd 100644
@@ -44,7 +44,7 @@
 #define HNS_ROCE_VF_SL_NUM			8
 #define HNS_ROCE_VF_GMV_BT_NUM			256
 
-#define HNS_ROCE_V2_MAX_QP_NUM			0x100000
+#define HNS_ROCE_V2_MAX_QP_NUM			0x1000
 #define HNS_ROCE_V2_MAX_QPC_TIMER_NUM		0x200
 #define HNS_ROCE_V2_MAX_WQE_NUM			0x8000
 #define	HNS_ROCE_V2_MAX_SRQ			0x100000
@@ -121,7 +121,7 @@
 
 #define HNS_ROCE_BA_PG_SZ_SUPPORTED_256K	6
 #define HNS_ROCE_BA_PG_SZ_SUPPORTED_16K		2
-#define HNS_ROCE_V2_GID_INDEX_NUM		256
+#define HNS_ROCE_V2_GID_INDEX_NUM		16
 
 #define HNS_ROCE_V2_TABLE_CHUNK_SIZE		(1 << 18)
 
@@ -247,6 +247,7 @@ enum hns_roce_opcode_type {
 	HNS_ROCE_OPC_CLR_SCCC				= 0x8509,
 	HNS_ROCE_OPC_QUERY_SCCC				= 0x850a,
 	HNS_ROCE_OPC_RESET_SCCC				= 0x850b,
+	HNS_ROCE_OPC_QUERY_VF_RES			= 0x850e,
 	HNS_ROCE_OPC_CFG_GMV_TBL			= 0x850f,
 	HNS_ROCE_OPC_CFG_GMV_BT				= 0x8510,
 	HNS_SWITCH_PARAMETER_CFG			= 0x1033,
@@ -1366,7 +1367,8 @@ struct hns_roce_cfg_llm_b {
 #define CFG_GLOBAL_PARAM_UDP_PORT CMQ_REQ_FIELD_LOC(31, 16)
 
 /*
- * Fields of HNS_ROCE_OPC_QUERY_PF_RES and HNS_ROCE_OPC_ALLOC_VF_RES
+ * Fields of HNS_ROCE_OPC_QUERY_PF_RES, HNS_ROCE_OPC_QUERY_VF_RES
+ * and HNS_ROCE_OPC_ALLOC_VF_RES
  */
 #define FUNC_RES_A_VF_ID CMQ_REQ_FIELD_LOC(7, 0)
 #define FUNC_RES_A_QPC_BT_IDX CMQ_REQ_FIELD_LOC(42, 32)
Signature · 1 line
2.8.1
author
#6 lore raw

PATCH v2 6/6 RDMA/hns: Remove duplicated hem page size config code

In reply to Weihang Li

From: Xi Wang <[email protected]>

Remove duplicated code for setting hem page size in PF and VF.

---
 drivers/infiniband/hw/hns/hns_roce_device.h |   1 -
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 184 ++++++++++++----------------
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h  |   1 +
 3 files changed, 76 insertions(+), 110 deletions(-)
drivers/infiniband/hw/hns/hns_roce_device.h +0 -1
index e631914..7309369 100644
@@ -772,7 +772,6 @@ struct hns_roce_caps {
 	int		num_other_vectors;
 	u32		num_mtpts;
 	u32		num_mtt_segs;
-	u32		num_cqe_segs;
 	u32		num_srqwqe_segs;
 	u32		num_idx_segs;
 	int		reserved_mrws;
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +75 -109

305 lines not on this page · raw message

drivers/infiniband/hw/hns/hns_roce_hw_v2.h +1 -0
index 7a25ffd..b8d0969 100644
@@ -59,6 +59,7 @@
 #define HNS_ROCE_V2_MAX_SQ_SGE_NUM		64
 #define HNS_ROCE_V2_MAX_EXTEND_SGE_NUM		0x200000
 #define HNS_ROCE_V2_MAX_SQ_INLINE		0x20
+#define HNS_ROCE_V2_MAX_SQ_INL_EXT		0x400
 #define HNS_ROCE_V2_MAX_RC_INL_INN_SZ		32
 #define HNS_ROCE_V2_UAR_NUM			256
 #define HNS_ROCE_V2_PHY_UAR_NUM			1
Signature · 1 line
2.8.1
maintainer Applied
#7 lore raw
In reply to Weihang Li
Quoted 14 lines · Weihang Li

This series first do necessary refactor on function's resource code to reduce the complexity of subsequent patches. Then the RoCE support for VFs is introduced.

Changes since v1:
- Simplify hr_reg_read() by FIELD_GET in #1.
- Link: https://patchwork.kernel.org/project/linux-rdma/cover/[email protected]/

Wei Xu (4):
RDMA/hns: Query the number of functions supported by the PF RDMA/hns: Reserve the resource for the VFs RDMA/hns: Set parameters of all the functions belong to a PF RDMA/hns: Enable RoCE on virtual functions

Quoted 3 lines

Xi Wang (2):
RDMA/hns: Simplify function's resource related command RDMA/hns: Remove duplicated hem page size config code

Applied to for-next, thanks

Jason