Skip to contentSkip to conversation

RDMA/hns: Support to select congestion control algorithm

PATCH Superseded Weihang Li 2 participants 7 messages Mar 12, 2021 – Mar 24, 2021

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Weihang Li

author
#0 lore raw

cover RDMA/hns: Support to select congestion control algorithm

The configuration of congestion control algorithm is recorded in the firmware, the driver queries it and then sets it to the hardware.

Wei Xu (1):
RDMA/hns: Support query information of functions from FW

Yangyang Li (1):
RDMA/hns: Support congestion control type selection according to the
FW

 drivers/infiniband/hw/hns/hns_roce_device.h |  11 ++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 192 ++++++++++++++++++++++++++++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h  |  34 ++++-
 drivers/infiniband/hw/hns/hns_roce_main.c   |   2 +
 4 files changed, 236 insertions(+), 3 deletions(-)
Signature · 1 line
2.8.1
author
#1 lore raw

PATCH 1/2 RDMA/hns: Support query information of functions from FW

From: Wei Xu <[email protected]>

Add a new type of command to query mac id of functions from the firmware, it is used to select the template of congestion algorithm. More info will be supported in the future.

---
 drivers/infiniband/hw/hns/hns_roce_device.h |  1 +
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 28 ++++++++++++++++++++++++++++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h  |  7 +++++++
 3 files changed, 36 insertions(+)
drivers/infiniband/hw/hns/hns_roce_device.h +1 -0
index 1a95fb7..869548e 100644
@@ -1001,6 +1001,7 @@ struct hns_roce_dev {
 	void			*priv;
 	struct workqueue_struct *irq_workq;
 	const struct hns_roce_dfx_hw *dfx;
+	u32 cong_algo_tmpl_id;
 };
 
 static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev)
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +28 -0
index 8f73d006..816006d 100644
@@ -1537,6 +1537,27 @@ static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
 	return 0;
 }
 
+static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_pf_func_info *resp;
+	struct hns_roce_cmq_desc desc;
+	int ret;
+
+	if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09)
+		return 0;
+
+	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_FUNC_INFO,
+				      true);
+	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
+	if (ret)
+		return ret;
+
+	resp = (struct hns_roce_pf_func_info *)desc.data;
+	hr_dev->cong_algo_tmpl_id = le32_to_cpu(resp->own_mac_id);
+
+	return 0;
+}
+
 static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
 {
 	struct hns_roce_cfg_global_param *req;
@@ -2279,6 +2300,13 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
 		return ret;
 	}
 
+	ret = hns_roce_query_func_info(hr_dev);
+	if (ret) {
+		dev_err(hr_dev->dev, "Query function info fail, ret = %d.\n",
+			ret);
+		return ret;
+	}
+
 	ret = hns_roce_config_global_param(hr_dev);
 	if (ret) {
 		dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n",
drivers/infiniband/hw/hns/hns_roce_hw_v2.h +7 -0
index ffdae15..74a1c15 100644
@@ -235,6 +235,7 @@ enum hns_roce_opcode_type {
 	HNS_ROCE_OPC_CFG_EXT_LLM			= 0x8403,
 	HNS_ROCE_OPC_CFG_TMOUT_LLM			= 0x8404,
 	HNS_ROCE_OPC_QUERY_PF_TIMER_RES			= 0x8406,
+	HNS_ROCE_OPC_QUERY_FUNC_INFO			= 0x8407,
 	HNS_ROCE_OPC_QUERY_PF_CAPS_NUM                  = 0x8408,
 	HNS_ROCE_OPC_CFG_ENTRY_SIZE			= 0x8409,
 	HNS_ROCE_OPC_CFG_SGID_TB			= 0x8500,
@@ -1469,6 +1470,12 @@ struct hns_roce_pf_timer_res_a {
 #define PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_S 16
 #define PF_RES_DATA_2_PF_CQC_TIMER_BT_NUM_M GENMASK(27, 16)
 
+struct hns_roce_pf_func_info {
+	__le32 rsv1;
+	__le32 own_mac_id;
+	__le32 rsv2[4];
+};
+
 struct hns_roce_vf_res_a {
 	__le32 vf_id;
 	__le32 vf_qpc_bt_idx_num;
Signature · 1 line
2.8.1
maintainer
#2 lore raw

Re: [PATCH for-next 1/2] RDMA/hns: Support query information of functions from FW

Quoted 6 lines · Weihang Li
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +6 -0
+static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_pf_func_info *resp;
+	struct hns_roce_cmq_desc desc;
+	int ret;
+

drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev) · line 1555

Quoted 10 lines
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +10 -0
+	if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09)
+		return 0;
+
+	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_FUNC_INFO,
+				      true);
+	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
+	if (ret)
+		return ret;
+
+	resp = (struct hns_roce_pf_func_info *)desc.data;

WTF is this cast?

struct hns_roce_cmq_desc {
        __le16 opcode;
        __le16 flag;
        __le16 retval;
        __le16 rsv;
        __le32 data[6];
};

Casting __le32 to a pointer is wrong

Jason

author
#3 lore raw

Re: [PATCH for-next 1/2] RDMA/hns: Support query information of functions from FW

Quoted 29 lines · Jason Gunthorpe
Quoted 6 lines · Weihang Li
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +6 -0
+static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_pf_func_info *resp;
+	struct hns_roce_cmq_desc desc;
+	int ret;
+

drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev) · line 1555

Quoted 10 lines
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +10 -0
+	if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09)
+		return 0;
+
+	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_FUNC_INFO,
+				      true);
+	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
+	if (ret)
+		return ret;
+
+	resp = (struct hns_roce_pf_func_info *)desc.data;

WTF is this cast?

struct hns_roce_cmq_desc {
        __le16 opcode;
        __le16 flag;
        __le16 retval;
        __le16 rsv;
        __le32 data[6];
};
Quoted 4 lines

Casting __le32 to a pointer is wrong

Jason

Hi Jason

desc.data is the address of array 'data[6]', it is got from the firmware, we cast it to 'struct hns_roce_pf_func_info *' to parse its contents. I think this is a cast from '__le32 *' to 'struct hns_roce_pf_func_info *'.

Thanks
Weihang

author
#4 lore raw

Re: [PATCH for-next 1/2] RDMA/hns: Support query information of functions from FW

Quoted 35 lines · liweihang
Quoted 29 lines · Jason Gunthorpe
Quoted 6 lines · Weihang Li
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +6 -0
+static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_pf_func_info *resp;
+	struct hns_roce_cmq_desc desc;
+	int ret;
+

drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev) · line 1555

Quoted 10 lines
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +10 -0
+	if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09)
+		return 0;
+
+	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_FUNC_INFO,
+				      true);
+	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
+	if (ret)
+		return ret;
+
+	resp = (struct hns_roce_pf_func_info *)desc.data;

WTF is this cast?

struct hns_roce_cmq_desc {
        __le16 opcode;
        __le16 flag;
        __le16 retval;
        __le16 rsv;
        __le32 data[6];
};
Quoted 4 lines

Casting __le32 to a pointer is wrong

Jason

Quoted 10 lines

Hi Jason

desc.data is the address of array 'data[6]', it is got from the firmware, we cast it to 'struct hns_roce_pf_func_info *' to parse its contents. I think this is a cast from '__le32 *' to 'struct hns_roce_pf_func_info *'.

Thanks
Weihang

Jason, do you mean the current code is not written correctly? Do you have any suggestions for achieving our purpose?

Weihang

maintainer
#5 lore raw

Re: [PATCH for-next 1/2] RDMA/hns: Support query information of functions from FW

Quoted 42 lines · liweihang
Quoted 41 lines · liweihang
Quoted 29 lines · Jason Gunthorpe
Quoted 6 lines · Weihang Li
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +6 -0
+static int hns_roce_query_func_info(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_pf_func_info *resp;
+	struct hns_roce_cmq_desc desc;
+	int ret;
+

drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev) · line 1555

Quoted 10 lines
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +10 -0
+	if (hr_dev->pci_dev->revision < PCI_REVISION_ID_HIP09)
+		return 0;
+
+	hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_FUNC_INFO,
+				      true);
+	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
+	if (ret)
+		return ret;
+
+	resp = (struct hns_roce_pf_func_info *)desc.data;

WTF is this cast?

struct hns_roce_cmq_desc {
        __le16 opcode;
        __le16 flag;
        __le16 retval;
        __le16 rsv;
        __le32 data[6];
};
Quoted 4 lines

Casting __le32 to a pointer is wrong

Jason

Hi Jason

desc.data is the address of array 'data[6]', it is got from the firmware, we cast it to 'struct hns_roce_pf_func_info *' to parse its contents. I think this is a cast from '__le32 *' to 'struct hns_roce_pf_func_info *'.

Quoted 7 lines
Quoted 4 lines

Thanks
Weihang

Jason, do you mean the current code is not written correctly? Do you have any suggestions for achieving our purpose?

Use a union

Jason

author
#6 lore raw

PATCH 2/2 RDMA/hns: Support congestion control type selection according to the FW

In reply to Weihang Li

From: Yangyang Li <[email protected]>

The type of congestion control algorithm includes DCQCN, LDCP, HC3 and DIP. The driver will select one of them according to the firmware when querying PF capabilities, and then set the related configuration fields into QPC.

---
 drivers/infiniband/hw/hns/hns_roce_device.h |  10 ++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 164 ++++++++++++++++++++++++++++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h  |  27 ++++-
 drivers/infiniband/hw/hns/hns_roce_main.c   |   2 +
 4 files changed, 200 insertions(+), 3 deletions(-)
drivers/infiniband/hw/hns/hns_roce_device.h +10 -0
index 869548e..30c2c86 100644
@@ -735,6 +735,13 @@ struct hns_roce_eq_table {
 	void __iomem		**eqc_base; /* only for hw v1 */
 };
 
+enum cong_type {
+	CONG_TYPE_DCQCN,
+	CONG_TYPE_LDCP,
+	CONG_TYPE_HC3,
+	CONG_TYPE_DIP,
+};
+
 struct hns_roce_caps {
 	u64		fw_ver;
 	u8		num_ports;
@@ -865,6 +872,7 @@ struct hns_roce_caps {
 	u16		default_aeq_period;
 	u16		default_aeq_arm_st;
 	u16		default_ceq_arm_st;
+	enum cong_type	cong_type;
 };
 
 struct hns_roce_dfx_hw {
@@ -959,6 +967,8 @@ struct hns_roce_dev {
 	enum hns_roce_device_state state;
 	struct list_head	qp_list; /* list of all qps on this dev */
 	spinlock_t		qp_list_lock; /* protect qp_list */
+	struct list_head	dip_list; /* list of all dest ips on this dev */
+	spinlock_t		dip_list_lock; /* protect dip_list */
 
 	struct list_head        pgdir_list;
 	struct mutex            pgdir_mutex;
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +164 -0

195 lines not on this page · raw message

drivers/infiniband/hw/hns/hns_roce_hw_v2.h +24 -3
index 74a1c15..7e9b777 100644
@@ -145,6 +145,8 @@
 
 #define HNS_ROCE_CMQ_SCC_CLR_DONE_CNT		5
 
+#define HNS_ROCE_CONG_SIZE 64
+
 #define check_whether_last_step(hop_num, step_idx) \
 	((step_idx == 0 && hop_num == HNS_ROCE_HOP_NUM_0) || \
 	(step_idx == 1 && hop_num == 1) || \
@@ -575,6 +577,10 @@ struct hns_roce_v2_qp_context {
 	struct hns_roce_v2_qp_context_ex ext;
 };
 
+#define QPC_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_qp_context, h, l)
+
+#define QPC_CONG_ALGO_TMPL_ID QPC_FIELD_LOC(455, 448)
+
 #define	V2_QPC_BYTE_4_TST_S 0
 #define V2_QPC_BYTE_4_TST_M GENMASK(2, 0)
 
@@ -666,9 +672,6 @@ struct hns_roce_v2_qp_context {
 #define	V2_QPC_BYTE_56_LP_PKTN_INI_S 28
 #define V2_QPC_BYTE_56_LP_PKTN_INI_M GENMASK(31, 28)
 
-#define	V2_QPC_BYTE_60_TEMPID_S 0
-#define V2_QPC_BYTE_60_TEMPID_M GENMASK(7, 0)
-
 #define V2_QPC_BYTE_60_SCC_TOKEN_S 8
 #define V2_QPC_BYTE_60_SCC_TOKEN_M GENMASK(26, 8)
 
@@ -943,6 +946,10 @@ struct hns_roce_v2_qp_context {
 
 #define QPCEX_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_qp_context_ex, h, l)
 
+#define QPCEX_CONG_ALG_SEL QPCEX_FIELD_LOC(0, 0)
+#define QPCEX_CONG_ALG_SUB_SEL QPCEX_FIELD_LOC(1, 1)
+#define QPCEX_DIP_CTX_IDX_VLD QPCEX_FIELD_LOC(2, 2)
+#define QPCEX_DIP_CTX_IDX QPCEX_FIELD_LOC(22, 3)
 #define QPCEX_STASH QPCEX_FIELD_LOC(82, 82)
 
 #define	V2_QP_RWE_S 1 /* rdma write enable */
@@ -1814,6 +1821,14 @@ struct hns_roce_query_pf_caps_d {
 #define V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_S 24
 #define V2_QUERY_PF_CAPS_D_SQWQE_HOP_NUM_M GENMASK(25, 24)
 
+#define V2_QUERY_PF_CAPS_D_CONG_TYPE_S 26
+#define V2_QUERY_PF_CAPS_D_CONG_TYPE_M GENMASK(29, 26)
+
+struct hns_roce_congestion_algorithm {
+	u8 alg_sel;
+	u8 alg_sub_sel;
+	u8 dip_vld;
+};
 
 #define V2_QUERY_PF_CAPS_D_CEQ_DEPTH_S 0
 #define V2_QUERY_PF_CAPS_D_CEQ_DEPTH_M GENMASK(21, 0)
@@ -1941,6 +1956,12 @@ struct hns_roce_eq_context {
 	__le32	rsv[5];
 };
 
+struct hns_roce_dip {
+	u8 dgid[GID_LEN_V2];
+	u8 dip_idx;
+	struct list_head node;	/* all dips are on a list */
+};
+
 #define HNS_ROCE_AEQ_DEFAULT_BURST_NUM	0x0
 #define HNS_ROCE_AEQ_DEFAULT_INTERVAL	0x0
 #define HNS_ROCE_CEQ_DEFAULT_BURST_NUM	0x0
drivers/infiniband/hw/hns/hns_roce_main.c +2 -0
index 1a747f7..563fc88 100644
@@ -910,6 +910,8 @@ int hns_roce_init(struct hns_roce_dev *hr_dev)
 
 	INIT_LIST_HEAD(&hr_dev->qp_list);
 	spin_lock_init(&hr_dev->qp_list_lock);
+	INIT_LIST_HEAD(&hr_dev->dip_list);
+	spin_lock_init(&hr_dev->dip_list_lock);
 
 	ret = hns_roce_register_device(hr_dev);
 	if (ret)
Signature · 1 line
2.8.1