Skip to contentSkip to conversation

RDMA/hns: Add more restrack attributes to hns driver

PATCH Superseded Wenpeng Liang 2 participants 7 messages Jan 24, 2022 – Jan 27, 2022

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover RDMA/hns: Add more restrack attributes to hns driver

First refactored the restrack code of the driver, and then added more restrack attributes.

Wenpeng Liang (4):
RDMA/hns: Refactor the restrack code of CQ RDMA/hns: Add more restrack attributes to CQ RDMA/hns: Add support for QP's restrack attributes RDMA/hns: Add support for MR's restrack attributes

 drivers/infiniband/hw/hns/Makefile            |   2 +-
 drivers/infiniband/hw/hns/hns_roce_device.h   |  13 +-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c    |  79 ++++-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h    |  30 --
 .../infiniband/hw/hns/hns_roce_hw_v2_dfx.c    |  35 ---
 drivers/infiniband/hw/hns/hns_roce_main.c     |   2 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 272 ++++++++++++++----
 7 files changed, 288 insertions(+), 145 deletions(-)
 delete mode 100644 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c

--
2.33.0

#1 lore raw

PATCH 1/4 RDMA/hns: Refactor the restrack code of CQ

Use macros to construct resource entries to make the code clearer.

---
 drivers/infiniband/hw/hns/Makefile            |   2 +-
 drivers/infiniband/hw/hns/hns_roce_device.h   |   9 +-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c    |  34 ++++-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h    |  30 -----
 .../infiniband/hw/hns/hns_roce_hw_v2_dfx.c    |  35 -----
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 121 ++++++------------
 6 files changed, 73 insertions(+), 158 deletions(-)
 delete mode 100644 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
drivers/infiniband/hw/hns/Makefile +1 -1
index 9f04f25d9631..a7d259238305 100644
@@ -10,6 +10,6 @@ hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \
 	hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o
 
 ifdef CONFIG_INFINIBAND_HNS_HIP08
-hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o $(hns-roce-objs)
+hns-roce-hw-v2-objs := hns_roce_hw_v2.o $(hns-roce-objs)
 obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v2.o
 endif
drivers/infiniband/hw/hns/hns_roce_device.h +2 -7
index 1e0bae136997..43e4b2945bf7 100644
@@ -834,11 +834,6 @@ struct hns_roce_caps {
 	enum cong_type	cong_type;
 };
 
-struct hns_roce_dfx_hw {
-	int (*query_cqc_info)(struct hns_roce_dev *hr_dev, u32 cqn,
-			      int *buffer);
-};
-
 enum hns_roce_device_state {
 	HNS_ROCE_DEVICE_STATE_INITED,
 	HNS_ROCE_DEVICE_STATE_RST_DOWN,
@@ -885,6 +880,7 @@ struct hns_roce_hw {
 	int (*init_eq)(struct hns_roce_dev *hr_dev);
 	void (*cleanup_eq)(struct hns_roce_dev *hr_dev);
 	int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);
+	int (*query_cqc)(struct hns_roce_dev *hr_dev, u32 cqn, void *buffer);
 	const struct ib_device_ops *hns_roce_dev_ops;
 	const struct ib_device_ops *hns_roce_dev_srq_ops;
 };
@@ -1217,8 +1213,7 @@ u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u32 port, int gid_index);
 void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev);
 int hns_roce_init(struct hns_roce_dev *hr_dev);
 void hns_roce_exit(struct hns_roce_dev *hr_dev);
-int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
-			       struct ib_cq *ib_cq);
+int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +29 -5
index 1e539e228315..3c079fadd1e7 100644
@@ -5511,6 +5511,34 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
 	return ret;
 }
 
+static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn,
+				 void *buffer)
+{
+	struct hns_roce_v2_cq_context *context;
+	struct hns_roce_cmd_mailbox *mailbox;
+	int ret;
+
+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+	if (IS_ERR(mailbox))
+		return PTR_ERR(mailbox);
+
+	context = mailbox->buf;
+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, cqn, 0,
+				HNS_ROCE_CMD_QUERY_CQC,
+				HNS_ROCE_CMD_TIMEOUT_MSECS);
+	if (ret) {
+		dev_err(hr_dev->dev, "QUERY cqc cmd process error.\n");
+		goto err_mailbox;
+	}
+
+	memcpy(buffer, context, sizeof(*context));
+
+err_mailbox:
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+
+	return ret;
+}
+
 static void hns_roce_irq_work_handle(struct work_struct *work)
 {
 	struct hns_roce_work *irq_work =
@@ -6169,10 +6197,6 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
 	kfree(eq_table->eq);
 }
 
-static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
-	.query_cqc_info = hns_roce_v2_query_cqc_info,
-};
-
 static const struct ib_device_ops hns_roce_v2_dev_ops = {
 	.destroy_qp = hns_roce_v2_destroy_qp,
 	.modify_cq = hns_roce_v2_modify_cq,
@@ -6212,6 +6236,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {
 	.init_eq = hns_roce_v2_init_eq_table,
 	.cleanup_eq = hns_roce_v2_cleanup_eq_table,
 	.write_srqc = hns_roce_v2_write_srqc,
+	.query_cqc = hns_roce_v2_query_cqc,
 	.hns_roce_dev_ops = &hns_roce_v2_dev_ops,
 	.hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
 };
@@ -6243,7 +6268,6 @@ static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
 	hr_dev->is_vf = id->driver_data;
 	hr_dev->dev = &handle->pdev->dev;
 	hr_dev->hw = &hns_roce_hw_v2;
-	hr_dev->dfx = &hns_roce_dfx_hw_v2;
 	hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
 	hr_dev->odb_offset = hr_dev->sdb_offset;
 
drivers/infiniband/hw/hns/hns_roce_hw_v2.h +0 -30
index e9a73c34389b..fb47a6730e1c 100644
@@ -291,33 +291,6 @@ struct hns_roce_v2_cq_context {
 #define HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM 0x0
 #define HNS_ROCE_V2_CQ_DEFAULT_INTERVAL	0x0
 
-#define	V2_CQC_BYTE_4_ARM_ST_S 6
-#define V2_CQC_BYTE_4_ARM_ST_M GENMASK(7, 6)
-
-#define	V2_CQC_BYTE_4_CEQN_S 15
-#define V2_CQC_BYTE_4_CEQN_M GENMASK(23, 15)
-
-#define	V2_CQC_BYTE_8_CQN_S 0
-#define V2_CQC_BYTE_8_CQN_M GENMASK(23, 0)
-
-#define	V2_CQC_BYTE_16_CQE_HOP_NUM_S 30
-#define V2_CQC_BYTE_16_CQE_HOP_NUM_M GENMASK(31, 30)
-
-#define	V2_CQC_BYTE_28_CQ_PRODUCER_IDX_S 0
-#define V2_CQC_BYTE_28_CQ_PRODUCER_IDX_M GENMASK(23, 0)
-
-#define	V2_CQC_BYTE_32_CQ_CONSUMER_IDX_S 0
-#define V2_CQC_BYTE_32_CQ_CONSUMER_IDX_M GENMASK(23, 0)
-
-#define	V2_CQC_BYTE_52_CQE_CNT_S 0
-#define	V2_CQC_BYTE_52_CQE_CNT_M GENMASK(23, 0)
-
-#define	V2_CQC_BYTE_56_CQ_MAX_CNT_S 0
-#define V2_CQC_BYTE_56_CQ_MAX_CNT_M GENMASK(15, 0)
-
-#define	V2_CQC_BYTE_56_CQ_PERIOD_S 16
-#define V2_CQC_BYTE_56_CQ_PERIOD_M GENMASK(31, 16)
-
 #define CQC_FIELD_LOC(h, l) FIELD_LOC(struct hns_roce_v2_cq_context, h, l)
 
 #define CQC_CQ_ST CQC_FIELD_LOC(1, 0)
@@ -1545,9 +1518,6 @@ struct hns_roce_sccc_clr_done {
 	__le32 rsv[5];
 };
 
-int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
-			       int *buffer);
-
 static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2],
 				    void __iomem *dest)
 {
drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c deleted +0 -35
deleted file mode 100644
index 5a97b5a0b7be..000000000000
@@ -1,35 +0,0 @@
-// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
-// Copyright (c) 2019 Hisilicon Limited.
-
-#include "hnae3.h"
-#include "hns_roce_device.h"
-#include "hns_roce_cmd.h"
-#include "hns_roce_hw_v2.h"
-
-int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
-			       int *buffer)
-{
-	struct hns_roce_v2_cq_context *cq_context;
-	struct hns_roce_cmd_mailbox *mailbox;
-	int ret;
-
-	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
-	if (IS_ERR(mailbox))
-		return PTR_ERR(mailbox);
-
-	cq_context = mailbox->buf;
-	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, cqn, 0,
-				HNS_ROCE_CMD_QUERY_CQC,
-				HNS_ROCE_CMD_TIMEOUT_MSECS);
-	if (ret) {
-		dev_err(hr_dev->dev, "QUERY cqc cmd process error\n");
-		goto err_mailbox;
-	}
-
-	memcpy(buffer, cq_context, sizeof(*cq_context));
-
-err_mailbox:
-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
-
-	return ret;
-}
drivers/infiniband/hw/hns/hns_roce_restrack.c +41 -80

152 lines not on this page · raw message

Signature · 1 line
2.33.0
#2 lore raw

PATCH 2/4 RDMA/hns: Add more restrack attributes to CQ

In reply to Wenpeng Liang

Add arm_st, shift and cqe_size of CQ.

---
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)
drivers/infiniband/hw/hns/hns_roce_restrack.c +9 -2
index 4420639445a1..a0d601cd2cb6 100644
@@ -15,7 +15,8 @@
 #define _HR_REG_CFG(type, h, l) _HR_REG_MASK(h, l), l
 #define HR_REG_CFG(field) _HR_REG_CFG(field)
 
-static int hns_roce_fill_cq(struct sk_buff *msg,
+static int hns_roce_fill_cq(struct hns_roce_cq *hr_cq,
+			    struct sk_buff *msg,
 			    struct hns_roce_v2_cq_context *context)
 {
 	static struct {
@@ -24,8 +25,11 @@ static int hns_roce_fill_cq(struct sk_buff *msg,
 		u32 l;
 	} reg[] = {
 		{ "cq_st", HR_REG_CFG(CQC_CQ_ST) },
+		{ "arm_st", HR_REG_CFG(CQC_ARM_ST) },
+		{ "shift", HR_REG_CFG(CQC_SHIFT) },
 		{ "ceqn", HR_REG_CFG(CQC_CEQN) },
 		{ "cqn", HR_REG_CFG(CQC_CQN) },
+		{ "cqe_size", HR_REG_CFG(CQC_CQE_SIZE) },
 		{ "hopnum", HR_REG_CFG(CQC_CQE_HOP_NUM) },
 		{ "pi", HR_REG_CFG(CQC_CQ_PRODUCER_IDX) },
 		{ "ci", HR_REG_CFG(CQC_CQ_CONSUMER_IDX) },
@@ -43,6 +47,9 @@ static int hns_roce_fill_cq(struct sk_buff *msg,
 				(reg[i].l % 32)))
 			goto err;
 
+	if (rdma_nl_put_driver_u32_hex(msg, "soft_cq_ci", hr_cq->cons_index))
+		goto err;
+
 	return 0;
 
 err:
@@ -68,7 +75,7 @@ int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq)
 	if (!table_attr)
 		return -EMSGSIZE;
 
-	if (hns_roce_fill_cq(msg, &context))
+	if (hns_roce_fill_cq(hr_cq, msg, &context))
 		goto err_cancel_table;
 
 	nla_nest_end(msg, table_attr);
Signature · 1 line
2.33.0
#3 lore raw

PATCH 3/4 RDMA/hns: Add support for QP's restrack attributes

In reply to Wenpeng Liang

The restrack attributes of QP come from the QPC and the queue information maintained by the software code.

---
 drivers/infiniband/hw/hns/hns_roce_device.h   |   2 +
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c    |  16 +--
 drivers/infiniband/hw/hns/hns_roce_main.c     |   1 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 116 ++++++++++++++++++
 4 files changed, 128 insertions(+), 7 deletions(-)
drivers/infiniband/hw/hns/hns_roce_device.h +2 -0
index 43e4b2945bf7..a34c12970200 100644
@@ -881,6 +881,7 @@ struct hns_roce_hw {
 	void (*cleanup_eq)(struct hns_roce_dev *hr_dev);
 	int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);
 	int (*query_cqc)(struct hns_roce_dev *hr_dev, u32 cqn, void *buffer);
+	int (*query_qpc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer);
 	const struct ib_device_ops *hns_roce_dev_ops;
 	const struct ib_device_ops *hns_roce_dev_srq_ops;
 };
@@ -1214,6 +1215,7 @@ void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev);
 int hns_roce_init(struct hns_roce_dev *hr_dev);
 void hns_roce_exit(struct hns_roce_dev *hr_dev);
 int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq);
+int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +9 -7
index 3c079fadd1e7..4912c531ef91 100644
@@ -5050,9 +5050,8 @@ static int to_ib_qp_st(enum hns_roce_v2_qp_state state)
 	return (state < ARRAY_SIZE(map)) ? map[state] : -1;
 }
 
-static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
-				 struct hns_roce_qp *hr_qp,
-				 struct hns_roce_v2_qp_context *hr_context)
+static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, u32 qpn,
+				 void *buffer)
 {
 	struct hns_roce_cmd_mailbox *mailbox;
 	int ret;
@@ -5061,13 +5060,15 @@ static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
 	if (IS_ERR(mailbox))
 		return PTR_ERR(mailbox);
 
-	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, qpn, 0,
 				HNS_ROCE_CMD_QUERY_QPC,
 				HNS_ROCE_CMD_TIMEOUT_MSECS);
-	if (ret)
+	if (ret) {
+		dev_err(hr_dev->dev, "QUERY qpc cmd process error.\n");
 		goto out;
+	}
 
-	memcpy(hr_context, mailbox->buf, hr_dev->caps.qpc_sz);
+	memcpy(buffer, mailbox->buf, hr_dev->caps.qpc_sz);
 
 out:
 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
@@ -5097,7 +5098,7 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 		goto done;
 	}
 
-	ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context);
+	ret = hns_roce_v2_query_qpc(hr_dev, hr_qp->qpn, &context);
 	if (ret) {
 		ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret);
 		ret = -EINVAL;
@@ -6237,6 +6238,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {
 	.cleanup_eq = hns_roce_v2_cleanup_eq_table,
 	.write_srqc = hns_roce_v2_write_srqc,
 	.query_cqc = hns_roce_v2_query_cqc,
+	.query_qpc = hns_roce_v2_query_qpc,
 	.hns_roce_dev_ops = &hns_roce_v2_dev_ops,
 	.hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
 };
drivers/infiniband/hw/hns/hns_roce_main.c +1 -0
index f73ba619f375..cf33182355c1 100644
@@ -516,6 +516,7 @@ static const struct ib_device_ops hns_roce_dev_ops = {
 	.destroy_cq = hns_roce_destroy_cq,
 	.disassociate_ucontext = hns_roce_disassociate_ucontext,
 	.fill_res_cq_entry = hns_roce_fill_res_cq_entry,
+	.fill_res_qp_entry = hns_roce_fill_res_qp_entry,
 	.get_dma_mr = hns_roce_get_dma_mr,
 	.get_link_layer = hns_roce_get_link_layer,
 	.get_port_immutable = hns_roce_port_immutable,
drivers/infiniband/hw/hns/hns_roce_restrack.c +116 -0

119 lines not on this page · raw message

Signature · 1 line
2.33.0
#4 lore raw

Re: [PATCH for-next 3/4] RDMA/hns: Add support for QP's restrack attributes

Quoted 10 lines · Wenpeng Liang

The restrack attributes of QP come from the QPC and the queue information maintained by the software code.

---
 drivers/infiniband/hw/hns/hns_roce_device.h   |   2 +
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c    |  16 +--
 drivers/infiniband/hw/hns/hns_roce_main.c     |   1 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 116 ++++++++++++++++++
 4 files changed, 128 insertions(+), 7 deletions(-)

<...>

drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, · line 5067

Quoted 2 lines
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +2 -0
+	if (ret) {
+		dev_err(hr_dev->dev, "QUERY qpc cmd process error.\n");

ibdev_err(...) in all places.

Quoted 2 lines
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +1 -0
 		goto out;
+	}

<...>

Quoted 44 lines
drivers/infiniband/hw/hns/hns_roce_restrack.c +44 -0
+static int hns_roce_fill_qp(struct hns_roce_qp *hr_qp,
+			    struct sk_buff *msg,
+			    struct hns_roce_v2_qp_context *context)
+{
+	static struct {
+		char *name;
+		u32 mask;
+		u32 l;
+	} reg[] = {
+		{ "tst", HR_REG_CFG(QPC_TST) },
+		{ "qp_st", HR_REG_CFG(QPC_QP_ST) },
+		{ "chk_flg", HR_REG_CFG(QPC_CHECK_FLG) },
+		{ "err_type", HR_REG_CFG(QPC_ERR_TYPE) },
+		{ "srq_en", HR_REG_CFG(QPC_SRQ_EN) },
+		{ "srqn", HR_REG_CFG(QPC_SRQN) },
+		{ "qkey_xrcd", HR_REG_CFG(QPC_QKEY_XRCD) },
+		{ "tx_cqn", HR_REG_CFG(QPC_TX_CQN) },
+		{ "rx_cqn", HR_REG_CFG(QPC_RX_CQN) },
+		{ "sq_pi", HR_REG_CFG(QPC_SQ_PRODUCER_IDX) },
+		{ "sq_ci", HR_REG_CFG(QPC_SQ_CONSUMER_IDX) },
+		{ "rq_pi", HR_REG_CFG(QPC_RQ_PRODUCER_IDX) },
+		{ "rq_ci", HR_REG_CFG(QPC_RQ_CONSUMER_IDX) },
+		{ "sq_shift", HR_REG_CFG(QPC_SQ_SHIFT) },
+		{ "rqws", HR_REG_CFG(QPC_RQWS) },
+		{ "rq_shift", HR_REG_CFG(QPC_RQ_SHIFT) },
+		{ "sge_shift", HR_REG_CFG(QPC_SGE_SHIFT) },
+		{ "max_ird", HR_REG_CFG(QPC_SR_MAX) },
+		{ "max_ord", HR_REG_CFG(QPC_RR_MAX) },
+		{ "gmv_idx", HR_REG_CFG(QPC_GMV_IDX) },
+		{ "sq_vlan_en", HR_REG_CFG(QPC_SQ_VLAN_EN) },
+		{ "rq_vlan_en", HR_REG_CFG(QPC_RQ_VLAN_EN) },
+		{ "vlan_id", HR_REG_CFG(QPC_VLAN_ID) },
+		{ "mtu", HR_REG_CFG(QPC_MTU) },
+		{ "hop_limit", HR_REG_CFG(QPC_HOPLIMIT) },
+		{ "tc", HR_REG_CFG(QPC_TC) },
+		{ "fl", HR_REG_CFG(QPC_FL) },
+		{ "sl", HR_REG_CFG(QPC_SL) },
+		{ "rre", HR_REG_CFG(QPC_RRE) },
+		{ "rwe", HR_REG_CFG(QPC_RWE) },
+		{ "ate", HR_REG_CFG(QPC_ATE) },
+		{ "ext_ate", HR_REG_CFG(QPC_EXT_ATE) },
+		{ "fre", HR_REG_CFG(QPC_FRE) },
+		{ "rmt_e2e", HR_REG_CFG(QPC_RMT_E2E) },
+		{ "retry_num_init", HR_REG_CFG(QPC_RETRY_NUM_INIT) },

drivers/infiniband/hw/hns/hns_roce_restrack.c int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq) · line 143

Quoted 10 lines
drivers/infiniband/hw/hns/hns_roce_restrack.c +10 -0
+		{ "retry_cnt", HR_REG_CFG(QPC_RETRY_CNT) },
+		{ "flush_idx", HR_REG_CFG(QPC_SQ_FLUSH_IDX) },
+		{ "sq_max_idx", HR_REG_CFG(QPC_SQ_MAX_IDX) },
+		{ "sq_tx_err", HR_REG_CFG(QPC_SQ_TX_ERR) },
+		{ "sq_rx_err", HR_REG_CFG(QPC_SQ_RX_ERR) },
+		{ "rq_rx_err", HR_REG_CFG(QPC_RQ_RX_ERR) },
+		{ "rq_tx_err", HR_REG_CFG(QPC_RQ_TX_ERR) },
+		{ "rq_cqeidx", HR_REG_CFG(QPC_RQ_CQE_IDX) },
+		{ "rq_rty_tx_err", HR_REG_CFG(QPC_RQ_RTY_TX_ERR) },
+	};

I have a feeling that this is abuse of our vendor specific attributes. Why don't you use RDMA_NLDEV_ATTR_RES_RAW for such huge dumps?

Thanks

#5 lore raw

Re: [PATCH for-next 3/4] RDMA/hns: Add support for QP's restrack attributes

Quoted 4 lines · Leon Romanovsky
Quoted 3 lines · Wenpeng Liang

The restrack attributes of QP come from the QPC and the queue information maintained by the software code.

I forgot to add the output of rdmatool in the commit message, it will be added in the next version.

Quoted 8 lines
Quoted 7 lines
---
 drivers/infiniband/hw/hns/hns_roce_device.h   |   2 +
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c    |  16 +--
 drivers/infiniband/hw/hns/hns_roce_main.c     |   1 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 116 ++++++++++++++++++
 4 files changed, 128 insertions(+), 7 deletions(-)
Quoted 7 lines

<...>

drivers/infiniband/hw/hns/hns_roce_hw_v2.c static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, · line 5067

Quoted 2 lines
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +2 -0
+	if (ret) {
+		dev_err(hr_dev->dev, "QUERY qpc cmd process error.\n");

ibdev_err(...) in all places.

Thanks, I will fix this in v2.

Quoted 54 lines
Quoted 2 lines
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +1 -0
 		goto out;
+	}

<...>

Quoted 49 lines
drivers/infiniband/hw/hns/hns_roce_restrack.c +49 -0
+static int hns_roce_fill_qp(struct hns_roce_qp *hr_qp,
+			    struct sk_buff *msg,
+			    struct hns_roce_v2_qp_context *context)
+{
+	static struct {
+		char *name;
+		u32 mask;
+		u32 l;
+	} reg[] = {
+		{ "tst", HR_REG_CFG(QPC_TST) },
+		{ "qp_st", HR_REG_CFG(QPC_QP_ST) },
+		{ "chk_flg", HR_REG_CFG(QPC_CHECK_FLG) },
+		{ "err_type", HR_REG_CFG(QPC_ERR_TYPE) },
+		{ "srq_en", HR_REG_CFG(QPC_SRQ_EN) },
+		{ "srqn", HR_REG_CFG(QPC_SRQN) },
+		{ "qkey_xrcd", HR_REG_CFG(QPC_QKEY_XRCD) },
+		{ "tx_cqn", HR_REG_CFG(QPC_TX_CQN) },
+		{ "rx_cqn", HR_REG_CFG(QPC_RX_CQN) },
+		{ "sq_pi", HR_REG_CFG(QPC_SQ_PRODUCER_IDX) },
+		{ "sq_ci", HR_REG_CFG(QPC_SQ_CONSUMER_IDX) },
+		{ "rq_pi", HR_REG_CFG(QPC_RQ_PRODUCER_IDX) },
+		{ "rq_ci", HR_REG_CFG(QPC_RQ_CONSUMER_IDX) },
+		{ "sq_shift", HR_REG_CFG(QPC_SQ_SHIFT) },
+		{ "rqws", HR_REG_CFG(QPC_RQWS) },
+		{ "rq_shift", HR_REG_CFG(QPC_RQ_SHIFT) },
+		{ "sge_shift", HR_REG_CFG(QPC_SGE_SHIFT) },
+		{ "max_ird", HR_REG_CFG(QPC_SR_MAX) },
+		{ "max_ord", HR_REG_CFG(QPC_RR_MAX) },
+		{ "gmv_idx", HR_REG_CFG(QPC_GMV_IDX) },
+		{ "sq_vlan_en", HR_REG_CFG(QPC_SQ_VLAN_EN) },
+		{ "rq_vlan_en", HR_REG_CFG(QPC_RQ_VLAN_EN) },
+		{ "vlan_id", HR_REG_CFG(QPC_VLAN_ID) },
+		{ "mtu", HR_REG_CFG(QPC_MTU) },
+		{ "hop_limit", HR_REG_CFG(QPC_HOPLIMIT) },
+		{ "tc", HR_REG_CFG(QPC_TC) },
+		{ "fl", HR_REG_CFG(QPC_FL) },
+		{ "sl", HR_REG_CFG(QPC_SL) },
+		{ "rre", HR_REG_CFG(QPC_RRE) },
+		{ "rwe", HR_REG_CFG(QPC_RWE) },
+		{ "ate", HR_REG_CFG(QPC_ATE) },
+		{ "ext_ate", HR_REG_CFG(QPC_EXT_ATE) },
+		{ "fre", HR_REG_CFG(QPC_FRE) },
+		{ "rmt_e2e", HR_REG_CFG(QPC_RMT_E2E) },
+		{ "retry_num_init", HR_REG_CFG(QPC_RETRY_NUM_INIT) },
+		{ "retry_cnt", HR_REG_CFG(QPC_RETRY_CNT) },
+		{ "flush_idx", HR_REG_CFG(QPC_SQ_FLUSH_IDX) },
+		{ "sq_max_idx", HR_REG_CFG(QPC_SQ_MAX_IDX) },
+		{ "sq_tx_err", HR_REG_CFG(QPC_SQ_TX_ERR) },
+		{ "sq_rx_err", HR_REG_CFG(QPC_SQ_RX_ERR) },
Quoted 9 lines

drivers/infiniband/hw/hns/hns_roce_restrack.c int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq) · line 143

Quoted 5 lines
drivers/infiniband/hw/hns/hns_roce_restrack.c +5 -0
+		{ "rq_rx_err", HR_REG_CFG(QPC_RQ_RX_ERR) },
+		{ "rq_tx_err", HR_REG_CFG(QPC_RQ_TX_ERR) },
+		{ "rq_cqeidx", HR_REG_CFG(QPC_RQ_CQE_IDX) },
+		{ "rq_rty_tx_err", HR_REG_CFG(QPC_RQ_RTY_TX_ERR) },
+	};

I have a feeling that this is abuse of our vendor specific attributes. Why don't you use RDMA_NLDEV_ATTR_RES_RAW for such huge dumps?

I will reconsider the attributes that should be dumped. I will use RDMA_NLDEV_ATTR_RES for key attributes and RDMA_NLDEV_ATTR_RES_RAW for huge dumps.

Thanks
Wenpeng

Quoted 3 lines

Thanks
.

#6 lore raw

PATCH 4/4 RDMA/hns: Add support for MR's restrack attributes

In reply to Wenpeng Liang

The restrack attributes of MR come from the MPT.

---
 drivers/infiniband/hw/hns/hns_roce_device.h   |  2 +
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c    | 29 ++++++++
 drivers/infiniband/hw/hns/hns_roce_main.c     |  1 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 68 +++++++++++++++++++
 4 files changed, 100 insertions(+)
drivers/infiniband/hw/hns/hns_roce_device.h +2 -0
index a34c12970200..84f9eedeab49 100644
@@ -882,6 +882,7 @@ struct hns_roce_hw {
 	int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);
 	int (*query_cqc)(struct hns_roce_dev *hr_dev, u32 cqn, void *buffer);
 	int (*query_qpc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer);
+	int (*query_mpt)(struct hns_roce_dev *hr_dev, u32 key, void *buffer);
 	const struct ib_device_ops *hns_roce_dev_ops;
 	const struct ib_device_ops *hns_roce_dev_srq_ops;
 };
@@ -1216,6 +1217,7 @@ int hns_roce_init(struct hns_roce_dev *hr_dev);
 void hns_roce_exit(struct hns_roce_dev *hr_dev);
 int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ibcq);
 int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ibqp);
+int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ibmr);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +29 -0
index 4912c531ef91..d746747631a0 100644
@@ -5540,6 +5540,34 @@ static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn,
 	return ret;
 }
 
+static int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key,
+				 void *buffer)
+{
+	struct hns_roce_v2_mpt_entry *context;
+	struct hns_roce_cmd_mailbox *mailbox;
+	int ret;
+
+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+	if (IS_ERR(mailbox))
+		return PTR_ERR(mailbox);
+
+	context = mailbox->buf;
+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, key_to_hw_index(key),
+				0, HNS_ROCE_CMD_QUERY_MPT,
+				HNS_ROCE_CMD_TIMEOUT_MSECS);
+	if (ret) {
+		dev_err(hr_dev->dev, "QUERY mpt cmd process error.\n");
+		goto err_mailbox;
+	}
+
+	memcpy(buffer, context, sizeof(*context));
+
+err_mailbox:
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+
+	return ret;
+}
+
 static void hns_roce_irq_work_handle(struct work_struct *work)
 {
 	struct hns_roce_work *irq_work =
@@ -6239,6 +6267,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {
 	.write_srqc = hns_roce_v2_write_srqc,
 	.query_cqc = hns_roce_v2_query_cqc,
 	.query_qpc = hns_roce_v2_query_qpc,
+	.query_mpt = hns_roce_v2_query_mpt,
 	.hns_roce_dev_ops = &hns_roce_v2_dev_ops,
 	.hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
 };
drivers/infiniband/hw/hns/hns_roce_main.c +1 -0
index cf33182355c1..ff3e7f58f6c3 100644
@@ -517,6 +517,7 @@ static const struct ib_device_ops hns_roce_dev_ops = {
 	.disassociate_ucontext = hns_roce_disassociate_ucontext,
 	.fill_res_cq_entry = hns_roce_fill_res_cq_entry,
 	.fill_res_qp_entry = hns_roce_fill_res_qp_entry,
+	.fill_res_mr_entry = hns_roce_fill_res_mr_entry,
 	.get_dma_mr = hns_roce_get_dma_mr,
 	.get_link_layer = hns_roce_get_link_layer,
 	.get_port_immutable = hns_roce_port_immutable,
drivers/infiniband/hw/hns/hns_roce_restrack.c +68 -0

71 lines not on this page · raw message

Signature · 1 line
2.33.0