Skip to contentSkip to conversation

RDMA/hns: Add more restrack attributes to hns driver

PATCH Applied Wenpeng Liang 2 participants 9 messages Aug 22, 2022 – Aug 23, 2022

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

v2 cover RDMA/hns: Add more restrack attributes to hns driver

The hns driver already supports CQ's restrack ops. Therefore, this patchset supports QP/MR's restrack ops and CQ/QP/MR's restrack raw ops. Among them, restrack ops dump the queue information maintained by the driver, and restrack raw ops dump the queue context maintained by ROCEE.

Changes since v1:
* Add support for restrack raw ops.
* Add the result dumped by rdmatool in the commit message.
* v1 Link: https://patchwork.kernel.org/project/linux-rdma/cover/[email protected]/

Wenpeng Liang (7):
RDMA/hns: Remove redundant DFX file and DFX ops structure RDMA/hns: Add or remove CQ's restrack attributes RDMA/hns: Support CQ's restrack raw ops for hns driver RDMA/hns: Support QP's restrack ops for hns driver RDMA/hns: Support QP's restrack raw ops for hns driver RDMA/hns: Support MR's restrack ops for hns driver RDMA/hns: Support MR's restrack raw ops for hns driver

 drivers/infiniband/hw/hns/Makefile            |   2 +-
 drivers/infiniband/hw/hns/hns_roce_device.h   |  17 +-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c    |  76 +++++-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h    |   6 +-
 .../infiniband/hw/hns/hns_roce_hw_v2_dfx.c    |  34 ---
 drivers/infiniband/hw/hns/hns_roce_main.c     |  11 +-
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 220 ++++++++++++++----
 7 files changed, 263 insertions(+), 103 deletions(-)
 delete mode 100644 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c

--
2.30.0

#1 lore raw

PATCH v2 1/7 RDMA/hns: Remove redundant DFX file and DFX ops structure

There is no need to use a dedicated DXF file and DFX structure to manage the interface of the query queue context.

---
 drivers/infiniband/hw/hns/Makefile            |  2 +-
 drivers/infiniband/hw/hns/hns_roce_device.h   | 10 ++----
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c    | 35 ++++++++++++++++---
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h    |  3 --
 .../infiniband/hw/hns/hns_roce_hw_v2_dfx.c    | 34 ------------------
 drivers/infiniband/hw/hns/hns_roce_main.c     |  6 +++-
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 35 +++++++------------
 7 files changed, 50 insertions(+), 75 deletions(-)
 delete mode 100644 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
drivers/infiniband/hw/hns/Makefile +1 -1
index 9f04f25d9631..a7d259238305 100644
@@ -10,6 +10,6 @@ hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \
 	hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o
 
 ifdef CONFIG_INFINIBAND_HNS_HIP08
-hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o $(hns-roce-objs)
+hns-roce-hw-v2-objs := hns_roce_hw_v2.o $(hns-roce-objs)
 obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v2.o
 endif
drivers/infiniband/hw/hns/hns_roce_device.h +2 -8
index f848eedc6a23..103d50564b89 100644
@@ -849,11 +849,6 @@ struct hns_roce_caps {
 	enum cong_type	cong_type;
 };
 
-struct hns_roce_dfx_hw {
-	int (*query_cqc_info)(struct hns_roce_dev *hr_dev, u32 cqn,
-			      int *buffer);
-};
-
 enum hns_roce_device_state {
 	HNS_ROCE_DEVICE_STATE_INITED,
 	HNS_ROCE_DEVICE_STATE_RST_DOWN,
@@ -899,6 +894,7 @@ struct hns_roce_hw {
 	int (*init_eq)(struct hns_roce_dev *hr_dev);
 	void (*cleanup_eq)(struct hns_roce_dev *hr_dev);
 	int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);
+	int (*query_cqc)(struct hns_roce_dev *hr_dev, u32 cqn, void *buffer);
 	const struct ib_device_ops *hns_roce_dev_ops;
 	const struct ib_device_ops *hns_roce_dev_srq_ops;
 };
@@ -960,7 +956,6 @@ struct hns_roce_dev {
 	void			*priv;
 	struct workqueue_struct *irq_workq;
 	struct work_struct ecc_work;
-	const struct hns_roce_dfx_hw *dfx;
 	u32 func_num;
 	u32 is_vf;
 	u32 cong_algo_tmpl_id;
@@ -1228,8 +1223,7 @@ u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u32 port, int gid_index);
 void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev);
 int hns_roce_init(struct hns_roce_dev *hr_dev);
 void hns_roce_exit(struct hns_roce_dev *hr_dev);
-int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
-			       struct ib_cq *ib_cq);
+int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +30 -5
index cbdafaac678a..979cd57a72fb 100644
@@ -5774,6 +5774,35 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
 	return ret;
 }
 
+static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn,
+				 void *buffer)
+{
+	struct hns_roce_v2_cq_context *context;
+	struct hns_roce_cmd_mailbox *mailbox;
+	int ret;
+
+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+	if (IS_ERR(mailbox))
+		return PTR_ERR(mailbox);
+
+	context = mailbox->buf;
+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma,
+				HNS_ROCE_CMD_QUERY_CQC, cqn);
+	if (ret) {
+		ibdev_err(&hr_dev->ib_dev,
+			  "failed to process cmd when querying CQ, ret = %d.\n",
+			  ret);
+		goto err_mailbox;
+	}
+
+	memcpy(buffer, context, sizeof(*context));
+
+err_mailbox:
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+
+	return ret;
+}
+
 static void hns_roce_irq_work_handle(struct work_struct *work)
 {
 	struct hns_roce_work *irq_work =
@@ -6575,10 +6604,6 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
 	kfree(eq_table->eq);
 }
 
-static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
-	.query_cqc_info = hns_roce_v2_query_cqc_info,
-};
-
 static const struct ib_device_ops hns_roce_v2_dev_ops = {
 	.destroy_qp = hns_roce_v2_destroy_qp,
 	.modify_cq = hns_roce_v2_modify_cq,
@@ -6619,6 +6644,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {
 	.init_eq = hns_roce_v2_init_eq_table,
 	.cleanup_eq = hns_roce_v2_cleanup_eq_table,
 	.write_srqc = hns_roce_v2_write_srqc,
+	.query_cqc = hns_roce_v2_query_cqc,
 	.hns_roce_dev_ops = &hns_roce_v2_dev_ops,
 	.hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
 };
@@ -6650,7 +6676,6 @@ static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
 	hr_dev->is_vf = id->driver_data;
 	hr_dev->dev = &handle->pdev->dev;
 	hr_dev->hw = &hns_roce_hw_v2;
-	hr_dev->dfx = &hns_roce_dfx_hw_v2;
 	hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
 	hr_dev->odb_offset = hr_dev->sdb_offset;
 
drivers/infiniband/hw/hns/hns_roce_hw_v2.h +0 -3
index f96debac30fe..49ec29973ed7 100644
@@ -1462,9 +1462,6 @@ struct hns_roce_sccc_clr_done {
 	__le32 rsv[5];
 };
 
-int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
-			       int *buffer);
-
 static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2],
 				    void __iomem *dest)
 {
drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c deleted +0 -34
deleted file mode 100644
index f7a75a7cda74..000000000000
@@ -1,34 +0,0 @@
-// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
-// Copyright (c) 2019 Hisilicon Limited.
-
-#include "hnae3.h"
-#include "hns_roce_device.h"
-#include "hns_roce_cmd.h"
-#include "hns_roce_hw_v2.h"
-
-int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
-			       int *buffer)
-{
-	struct hns_roce_v2_cq_context *cq_context;
-	struct hns_roce_cmd_mailbox *mailbox;
-	int ret;
-
-	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
-	if (IS_ERR(mailbox))
-		return PTR_ERR(mailbox);
-
-	cq_context = mailbox->buf;
-	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_CQC,
-				cqn);
-	if (ret) {
-		dev_err(hr_dev->dev, "QUERY cqc cmd process error\n");
-		goto err_mailbox;
-	}
-
-	memcpy(buffer, cq_context, sizeof(*cq_context));
-
-err_mailbox:
-	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
-
-	return ret;
-}
drivers/infiniband/hw/hns/hns_roce_main.c +5 -1
index c8af4ebd7cbd..caf73e8f4bbe 100644
@@ -515,7 +515,6 @@ static const struct ib_device_ops hns_roce_dev_ops = {
 	.destroy_ah = hns_roce_destroy_ah,
 	.destroy_cq = hns_roce_destroy_cq,
 	.disassociate_ucontext = hns_roce_disassociate_ucontext,
-	.fill_res_cq_entry = hns_roce_fill_res_cq_entry,
 	.get_dma_mr = hns_roce_get_dma_mr,
 	.get_link_layer = hns_roce_get_link_layer,
 	.get_port_immutable = hns_roce_port_immutable,
@@ -566,6 +565,10 @@ static const struct ib_device_ops hns_roce_dev_xrcd_ops = {
 	INIT_RDMA_OBJ_SIZE(ib_xrcd, hns_roce_xrcd, ibxrcd),
 };
 
+static const struct ib_device_ops hns_roce_dev_restrack_ops = {
+	.fill_res_cq_entry = hns_roce_fill_res_cq_entry,
+};
+
 static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
 {
 	int ret;
@@ -605,6 +608,7 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
 
 	ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_ops);
 	ib_set_device_ops(ib_dev, &hns_roce_dev_ops);
+	ib_set_device_ops(ib_dev, &hns_roce_dev_restrack_ops);
 	for (i = 0; i < hr_dev->caps.num_ports; i++) {
 		if (!hr_dev->iboe.netdevs[i])
 			continue;
drivers/infiniband/hw/hns/hns_roce_restrack.c +12 -23
index 24a154d64630..83417be15d3f 100644
@@ -55,45 +55,34 @@ static int hns_roce_fill_cq(struct sk_buff *msg,
 	return -EMSGSIZE;
 }
 
-int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
-			       struct ib_cq *ib_cq)
+int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
 	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
-	struct hns_roce_v2_cq_context *context;
+	struct hns_roce_v2_cq_context context;
 	struct nlattr *table_attr;
 	int ret;
 
-	if (!hr_dev->dfx->query_cqc_info)
+	if (!hr_dev->hw->query_cqc)
 		return -EINVAL;
 
-	context = kzalloc(sizeof(struct hns_roce_v2_cq_context), GFP_KERNEL);
-	if (!context)
-		return -ENOMEM;
-
-	ret = hr_dev->dfx->query_cqc_info(hr_dev, hr_cq->cqn, (int *)context);
+	ret = hr_dev->hw->query_cqc(hr_dev, hr_cq->cqn, &context);
 	if (ret)
-		goto err;
+		return -EINVAL;
 
 	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
-	if (!table_attr) {
-		ret = -EMSGSIZE;
-		goto err;
-	}
+	if (!table_attr)
+		return -EMSGSIZE;
 
-	if (hns_roce_fill_cq(msg, context)) {
-		ret = -EMSGSIZE;
-		goto err_cancel_table;
-	}
+	if (hns_roce_fill_cq(msg, &context))
+		goto err;
 
 	nla_nest_end(msg, table_attr);
-	kfree(context);
 
 	return 0;
 
-err_cancel_table:
-	nla_nest_cancel(msg, table_attr);
 err:
-	kfree(context);
-	return ret;
+	nla_nest_cancel(msg, table_attr);
+
+	return -EMSGSIZE;
 }
Signature · 1 line
2.30.0
#2 lore raw

PATCH v2 2/7 RDMA/hns: Add or remove CQ's restrack attributes

In reply to Wenpeng Liang

Remove the resttrack attributes from the queue context held by ROCEE, and add the resttrack attributes from the queue information maintained by the driver.

For example:

$ rdma res show cq dev hns_0 cqn 14 -dd -jp
[ {
        "ifindex": 4,
        "ifname": "hns_0",
        "cqn": 14,
        "cqe": 127,
        "users": 1,
        "adaptive-moderation": false,
        "ctxn": 8,
        "pid": 1524,
        "comm": "ib_send_bw"
    },
    "drv_cq_depth": 128,
    "drv_cons_index": 0,
    "drv_cqe_size": 32,
    "drv_arm_sn": 1
}
---
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 67 +++----------------
 1 file changed, 10 insertions(+), 57 deletions(-)
drivers/infiniband/hw/hns/hns_roce_restrack.c +10 -57
index 83417be15d3f..2e8299784bc2 100644
@@ -9,72 +9,25 @@
 #include "hns_roce_device.h"
 #include "hns_roce_hw_v2.h"
 
-static int hns_roce_fill_cq(struct sk_buff *msg,
-			    struct hns_roce_v2_cq_context *context)
-{
-	if (rdma_nl_put_driver_u32(msg, "state",
-				   hr_reg_read(context, CQC_ARM_ST)))
-
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "ceqn",
-				   hr_reg_read(context, CQC_CEQN)))
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "cqn",
-				   hr_reg_read(context, CQC_CQN)))
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "hopnum",
-				   hr_reg_read(context, CQC_CQE_HOP_NUM)))
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "pi",
-				   hr_reg_read(context, CQC_CQ_PRODUCER_IDX)))
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "ci",
-				   hr_reg_read(context, CQC_CQ_CONSUMER_IDX)))
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "coalesce",
-				   hr_reg_read(context, CQC_CQ_MAX_CNT)))
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "period",
-				   hr_reg_read(context, CQC_CQ_PERIOD)))
-		goto err;
-
-	if (rdma_nl_put_driver_u32(msg, "cnt",
-				   hr_reg_read(context, CQC_CQE_CNT)))
-		goto err;
-
-	return 0;
-
-err:
-	return -EMSGSIZE;
-}
-
 int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq)
 {
-	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
 	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
-	struct hns_roce_v2_cq_context context;
 	struct nlattr *table_attr;
-	int ret;
-
-	if (!hr_dev->hw->query_cqc)
-		return -EINVAL;
-
-	ret = hr_dev->hw->query_cqc(hr_dev, hr_cq->cqn, &context);
-	if (ret)
-		return -EINVAL;
 
 	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
 	if (!table_attr)
 		return -EMSGSIZE;
 
-	if (hns_roce_fill_cq(msg, &context))
+	if (rdma_nl_put_driver_u32(msg, "cq_depth", hr_cq->cq_depth))
+		goto err;
+
+	if (rdma_nl_put_driver_u32(msg, "cons_index", hr_cq->cons_index))
+		goto err;
+
+	if (rdma_nl_put_driver_u32(msg, "cqe_size", hr_cq->cqe_size))
+		goto err;
+
+	if (rdma_nl_put_driver_u32(msg, "arm_sn", hr_cq->arm_sn))
 		goto err;
 
 	nla_nest_end(msg, table_attr);
Signature · 1 line
2.30.0
#3 lore raw

PATCH v2 3/7 RDMA/hns: Support CQ's restrack raw ops for hns driver

In reply to Wenpeng Liang

The CQ raw restrack attributes come from the queue context maintained by the ROCEE.

For example:

$ rdma res show cq dev hns_0 cqn 14 -dd -jp -r
[ {
        "ifindex": 4,
        "ifname": "hns_0",
        "data": [ 1,0,0,0,7,0,0,0,0,0,0,0,0,82,6,0,0,82,6,0,0,82,6,0,
		  1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,
		  6,0,0,0,0,0,0,0 ]
    } ]
---
 drivers/infiniband/hw/hns/hns_roce_device.h   |  1 +
 drivers/infiniband/hw/hns/hns_roce_main.c     |  1 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 39 +++++++++++++++++++
 3 files changed, 41 insertions(+)
drivers/infiniband/hw/hns/hns_roce_device.h +1 -0
index 103d50564b89..c73adc0d3555 100644
@@ -1224,6 +1224,7 @@ void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev);
 int hns_roce_init(struct hns_roce_dev *hr_dev);
 void hns_roce_exit(struct hns_roce_dev *hr_dev);
 int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);
+int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
drivers/infiniband/hw/hns/hns_roce_main.c +1 -0
index caf73e8f4bbe..1b66ed45350e 100644
@@ -567,6 +567,7 @@ static const struct ib_device_ops hns_roce_dev_xrcd_ops = {
 
 static const struct ib_device_ops hns_roce_dev_restrack_ops = {
 	.fill_res_cq_entry = hns_roce_fill_res_cq_entry,
+	.fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw,
 };
 
 static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
drivers/infiniband/hw/hns/hns_roce_restrack.c +39 -0
index 2e8299784bc2..3f9c2f9dfdf6 100644
@@ -9,6 +9,8 @@
 #include "hns_roce_device.h"
 #include "hns_roce_hw_v2.h"
 
+#define MAX_ENTRY_NUM 256
+
 int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq)
 {
 	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
@@ -39,3 +41,40 @@ int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq)
 
 	return -EMSGSIZE;
 }
+
+int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
+	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
+	struct hns_roce_v2_cq_context context;
+	u32 data[MAX_ENTRY_NUM] = {};
+	int offset = 0;
+	int ret;
+
+	if (!hr_dev->hw->query_cqc)
+		return -EINVAL;
+
+	ret = hr_dev->hw->query_cqc(hr_dev, hr_cq->cqn, &context);
+	if (ret)
+		return -EINVAL;
+
+	data[offset++] = hr_reg_read(&context, CQC_CQ_ST);
+	data[offset++] = hr_reg_read(&context, CQC_SHIFT);
+	data[offset++] = hr_reg_read(&context, CQC_CQE_SIZE);
+	data[offset++] = hr_reg_read(&context, CQC_CQE_CNT);
+	data[offset++] = hr_reg_read(&context, CQC_CQ_PRODUCER_IDX);
+	data[offset++] = hr_reg_read(&context, CQC_CQ_CONSUMER_IDX);
+	data[offset++] = hr_reg_read(&context, CQC_DB_RECORD_EN);
+	data[offset++] = hr_reg_read(&context, CQC_ARM_ST);
+	data[offset++] = hr_reg_read(&context, CQC_CMD_SN);
+	data[offset++] = hr_reg_read(&context, CQC_CEQN);
+	data[offset++] = hr_reg_read(&context, CQC_CQ_MAX_CNT);
+	data[offset++] = hr_reg_read(&context, CQC_CQ_PERIOD);
+	data[offset++] = hr_reg_read(&context, CQC_CQE_HOP_NUM);
+	data[offset++] = hr_reg_read(&context, CQC_CQE_BAR_PG_SZ);
+	data[offset++] = hr_reg_read(&context, CQC_CQE_BUF_PG_SZ);
+
+	ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, offset * sizeof(u32), data);
+
+	return ret;
+}
Signature · 1 line
2.30.0
#4 lore raw

PATCH v2 4/7 RDMA/hns: Support QP's restrack ops for hns driver

In reply to Wenpeng Liang

The QP restrack attributes come from the queue information maintained by the driver.

For example:

$ rdma res show qp link hns_0 lqpn 41 -jp -dd
[ {
        "ifindex": 4,
        "ifname": "hns_0",
        "port": 1,
        "lqpn": 41,
        "rqpn": 40,
        "type": "RC",
        "state": "RTR",
        "rq-psn": 12474738,
        "sq-psn": 0,
        "path-mig-state": "ARMED",
        "pdn": 9,
        "pid": 1523,
        "comm": "ib_send_bw"
    },
    "drv_sq_wqe_cnt": 128,
    "drv_sq_max_gs": 1,
    "drv_rq_wqe_cnt": 512,
    "drv_rq_max_gs": 2,
    "drv_ext_sge_sge_cnt": 0
}
---
 drivers/infiniband/hw/hns/hns_roce_device.h   |  1 +
 drivers/infiniband/hw/hns/hns_roce_main.c     |  1 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 34 +++++++++++++++++++
 3 files changed, 36 insertions(+)
drivers/infiniband/hw/hns/hns_roce_device.h +1 -0
index c73adc0d3555..7578c0c6313b 100644
@@ -1225,6 +1225,7 @@ int hns_roce_init(struct hns_roce_dev *hr_dev);
 void hns_roce_exit(struct hns_roce_dev *hr_dev);
 int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);
 int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);
+int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
drivers/infiniband/hw/hns/hns_roce_main.c +1 -0
index 1b66ed45350e..87442027b808 100644
@@ -568,6 +568,7 @@ static const struct ib_device_ops hns_roce_dev_xrcd_ops = {
 static const struct ib_device_ops hns_roce_dev_restrack_ops = {
 	.fill_res_cq_entry = hns_roce_fill_res_cq_entry,
 	.fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw,
+	.fill_res_qp_entry = hns_roce_fill_res_qp_entry,
 };
 
 static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
drivers/infiniband/hw/hns/hns_roce_restrack.c +34 -0
index 3f9c2f9dfdf6..e8fef37f810d 100644
@@ -78,3 +78,37 @@ int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq)
 
 	return ret;
 }
+
+int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp)
+{
+	struct hns_roce_qp *hr_qp = to_hr_qp(ib_qp);
+	struct nlattr *table_attr;
+
+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
+	if (!table_attr)
+		return -EMSGSIZE;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "sq_wqe_cnt", hr_qp->sq.wqe_cnt))
+		goto err;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "sq_max_gs", hr_qp->sq.max_gs))
+		goto err;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "rq_wqe_cnt", hr_qp->rq.wqe_cnt))
+		goto err;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "rq_max_gs", hr_qp->rq.max_gs))
+		goto err;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "ext_sge_sge_cnt", hr_qp->sge.sge_cnt))
+		goto err;
+
+	nla_nest_end(msg, table_attr);
+
+	return 0;
+
+err:
+	nla_nest_cancel(msg, table_attr);
+
+	return -EMSGSIZE;
+}
Signature · 1 line
2.30.0
#5 lore raw

PATCH v2 5/7 RDMA/hns: Support QP's restrack raw ops for hns driver

In reply to Wenpeng Liang

The QP raw restrack attributes come from the queue context maintained by the ROCEE.

For example:

$ rdma res show qp link hns_0 -jp -dd -r
[ {
        "ifindex": 4,
        "ifname": "hns_0",
        "data": [ 2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,
		  5,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,255,156,0,0,63,156,0,0,
		  7,0,0,0,1,0,0,0,9,0,0,0,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,
		  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
		  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,63,156,0,
		  0,0,0,0,0 ]
    } ]
---
 drivers/infiniband/hw/hns/hns_roce_device.h   |  2 +
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c    | 12 ++--
 drivers/infiniband/hw/hns/hns_roce_main.c     |  1 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 56 +++++++++++++++++++
 4 files changed, 65 insertions(+), 6 deletions(-)
drivers/infiniband/hw/hns/hns_roce_device.h +2 -0
index 7578c0c6313b..e0395870b819 100644
@@ -895,6 +895,7 @@ struct hns_roce_hw {
 	void (*cleanup_eq)(struct hns_roce_dev *hr_dev);
 	int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);
 	int (*query_cqc)(struct hns_roce_dev *hr_dev, u32 cqn, void *buffer);
+	int (*query_qpc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer);
 	const struct ib_device_ops *hns_roce_dev_ops;
 	const struct ib_device_ops *hns_roce_dev_srq_ops;
 };
@@ -1226,6 +1227,7 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev);
 int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);
 int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);
 int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp);
+int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +6 -6
index 979cd57a72fb..319de9a4d2ef 100644
@@ -5307,9 +5307,8 @@ static int to_ib_qp_st(enum hns_roce_v2_qp_state state)
 	return (state < ARRAY_SIZE(map)) ? map[state] : -1;
 }
 
-static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
-				 struct hns_roce_qp *hr_qp,
-				 struct hns_roce_v2_qp_context *hr_context)
+static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, u32 qpn,
+				 void *buffer)
 {
 	struct hns_roce_cmd_mailbox *mailbox;
 	int ret;
@@ -5319,11 +5318,11 @@ static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
 		return PTR_ERR(mailbox);
 
 	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_QPC,
-				hr_qp->qpn);
+				qpn);
 	if (ret)
 		goto out;
 
-	memcpy(hr_context, mailbox->buf, hr_dev->caps.qpc_sz);
+	memcpy(buffer, mailbox->buf, hr_dev->caps.qpc_sz);
 
 out:
 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
@@ -5353,7 +5352,7 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 		goto done;
 	}
 
-	ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context);
+	ret = hns_roce_v2_query_qpc(hr_dev, hr_qp->qpn, &context);
 	if (ret) {
 		ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret);
 		ret = -EINVAL;
@@ -6645,6 +6644,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {
 	.cleanup_eq = hns_roce_v2_cleanup_eq_table,
 	.write_srqc = hns_roce_v2_write_srqc,
 	.query_cqc = hns_roce_v2_query_cqc,
+	.query_qpc = hns_roce_v2_query_qpc,
 	.hns_roce_dev_ops = &hns_roce_v2_dev_ops,
 	.hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
 };
drivers/infiniband/hw/hns/hns_roce_main.c +1 -0
index 87442027b808..17bc73c108f2 100644
@@ -569,6 +569,7 @@ static const struct ib_device_ops hns_roce_dev_restrack_ops = {
 	.fill_res_cq_entry = hns_roce_fill_res_cq_entry,
 	.fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw,
 	.fill_res_qp_entry = hns_roce_fill_res_qp_entry,
+	.fill_res_qp_entry_raw = hns_roce_fill_res_qp_entry_raw,
 };
 
 static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
drivers/infiniband/hw/hns/hns_roce_restrack.c +56 -0

59 lines not on this page · raw message

Signature · 1 line
2.30.0
#6 lore raw

PATCH v2 6/7 RDMA/hns: Support MR's restrack ops for hns driver

In reply to Wenpeng Liang

The MR restrack attributes come from the queue information maintained by the driver.

For example:

$ rdma res show mr dev hns_0 mrn 6 -dd -jp
[ {
        "ifindex": 4,
        "ifname": "hns_0",
        "mrn": 6,
        "rkey": "300",
        "lkey": "300",
        "mrlen": 131072,
        "pdn": 8,
        "pid": 1524,
        "comm": "ib_send_bw"
    },
    "drv_pbl_hop_num": 2,
    "drv_ba_pg_shift": 14,
    "drv_buf_pg_shift": 12
}
---
 drivers/infiniband/hw/hns/hns_roce_device.h   |  1 +
 drivers/infiniband/hw/hns/hns_roce_main.c     |  1 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 30 +++++++++++++++++++
 3 files changed, 32 insertions(+)
drivers/infiniband/hw/hns/hns_roce_device.h +1 -0
index e0395870b819..30a67bc70f1a 100644
@@ -1228,6 +1228,7 @@ int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);
 int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);
 int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp);
 int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp);
+int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
drivers/infiniband/hw/hns/hns_roce_main.c +1 -0
index 17bc73c108f2..ff4386b5c064 100644
@@ -570,6 +570,7 @@ static const struct ib_device_ops hns_roce_dev_restrack_ops = {
 	.fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw,
 	.fill_res_qp_entry = hns_roce_fill_res_qp_entry,
 	.fill_res_qp_entry_raw = hns_roce_fill_res_qp_entry_raw,
+	.fill_res_mr_entry = hns_roce_fill_res_mr_entry,
 };
 
 static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
drivers/infiniband/hw/hns/hns_roce_restrack.c +30 -0
index 9bafc627864b..84f942e19743 100644
@@ -168,3 +168,33 @@ int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp)
 
 	return ret;
 }
+
+int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr)
+{
+	struct hns_roce_mr *hr_mr = to_hr_mr(ib_mr);
+	struct nlattr *table_attr;
+
+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
+	if (!table_attr)
+		return -EMSGSIZE;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "pbl_hop_num", hr_mr->pbl_hop_num))
+		goto err;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "ba_pg_shift",
+				       hr_mr->pbl_mtr.hem_cfg.ba_pg_shift))
+		goto err;
+
+	if (rdma_nl_put_driver_u32_hex(msg, "buf_pg_shift",
+				       hr_mr->pbl_mtr.hem_cfg.buf_pg_shift))
+		goto err;
+
+	nla_nest_end(msg, table_attr);
+
+	return 0;
+
+err:
+	nla_nest_cancel(msg, table_attr);
+
+	return -EMSGSIZE;
+}
Signature · 1 line
2.30.0
#7 lore raw

PATCH v2 7/7 RDMA/hns: Support MR's restrack raw ops for hns driver

In reply to Wenpeng Liang

The MR raw restrack attributes come from the queue context maintained by the ROCEE.

For example:

$ rdma res show mr dev hns_0 mrn 6 -dd -jp -r
[ {
        "ifindex": 4,
        "ifname": "hns_0",
        "data": [ 1,0,0,0,2,0,0,0,0,3,0,0,0,0,2,0,0,0,0,0,32,0,0,0,2,0,0,0,
		  2,0,0,0,0,0,0,0 ]
    } ]
---
 drivers/infiniband/hw/hns/hns_roce_device.h   |  2 ++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c    | 29 +++++++++++++++++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h    |  3 +-
 drivers/infiniband/hw/hns/hns_roce_main.c     |  1 +
 drivers/infiniband/hw/hns/hns_roce_restrack.c | 31 +++++++++++++++++++
 5 files changed, 65 insertions(+), 1 deletion(-)
drivers/infiniband/hw/hns/hns_roce_device.h +2 -0
index 30a67bc70f1a..1bcecc5589fa 100644
@@ -896,6 +896,7 @@ struct hns_roce_hw {
 	int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf);
 	int (*query_cqc)(struct hns_roce_dev *hr_dev, u32 cqn, void *buffer);
 	int (*query_qpc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer);
+	int (*query_mpt)(struct hns_roce_dev *hr_dev, u32 key, void *buffer);
 	const struct ib_device_ops *hns_roce_dev_ops;
 	const struct ib_device_ops *hns_roce_dev_srq_ops;
 };
@@ -1229,6 +1230,7 @@ int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);
 int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp);
 int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp);
 int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr);
+int hns_roce_fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ib_mr);
 struct hns_user_mmap_entry *
 hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 				size_t length,
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +29 -0
index 319de9a4d2ef..f59176561420 100644
@@ -5802,6 +5802,34 @@ static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn,
 	return ret;
 }
 
+int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key, void *buffer)
+{
+	struct hns_roce_v2_mpt_entry *context;
+	struct hns_roce_cmd_mailbox *mailbox;
+	int ret;
+
+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+	if (IS_ERR(mailbox))
+		return PTR_ERR(mailbox);
+
+	context = mailbox->buf;
+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT,
+				key_to_hw_index(key));
+	if (ret) {
+		ibdev_err(&hr_dev->ib_dev,
+			  "failed to process cmd when querying MPT, ret = %d.\n",
+			  ret);
+		goto err_mailbox;
+	}
+
+	memcpy(buffer, context, sizeof(*context));
+
+err_mailbox:
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+
+	return ret;
+}
+
 static void hns_roce_irq_work_handle(struct work_struct *work)
 {
 	struct hns_roce_work *irq_work =
@@ -6645,6 +6673,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {
 	.write_srqc = hns_roce_v2_write_srqc,
 	.query_cqc = hns_roce_v2_query_cqc,
 	.query_qpc = hns_roce_v2_query_qpc,
+	.query_mpt = hns_roce_v2_query_mpt,
 	.hns_roce_dev_ops = &hns_roce_v2_dev_ops,
 	.hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
 };
drivers/infiniband/hw/hns/hns_roce_hw_v2.h +2 -1
index 49ec29973ed7..ae29780dd63a 100644
@@ -758,7 +758,8 @@ struct hns_roce_v2_mpt_entry {
 #define MPT_INNER_PA_VLD MPT_FIELD_LOC(71, 71)
 #define MPT_MW_BIND_QPN MPT_FIELD_LOC(95, 72)
 #define MPT_BOUND_LKEY MPT_FIELD_LOC(127, 96)
-#define MPT_LEN MPT_FIELD_LOC(191, 128)
+#define MPT_LEN_L MPT_FIELD_LOC(159, 128)
+#define MPT_LEN_H MPT_FIELD_LOC(191, 160)
 #define MPT_LKEY MPT_FIELD_LOC(223, 192)
 #define MPT_VA MPT_FIELD_LOC(287, 224)
 #define MPT_PBL_SIZE MPT_FIELD_LOC(319, 288)
drivers/infiniband/hw/hns/hns_roce_main.c +1 -0
index ff4386b5c064..9de3a522980a 100644
@@ -571,6 +571,7 @@ static const struct ib_device_ops hns_roce_dev_restrack_ops = {
 	.fill_res_qp_entry = hns_roce_fill_res_qp_entry,
 	.fill_res_qp_entry_raw = hns_roce_fill_res_qp_entry_raw,
 	.fill_res_mr_entry = hns_roce_fill_res_mr_entry,
+	.fill_res_mr_entry_raw = hns_roce_fill_res_mr_entry_raw,
 };
 
 static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
drivers/infiniband/hw/hns/hns_roce_restrack.c +31 -0
index 84f942e19743..989a2af2e938 100644
@@ -198,3 +198,34 @@ int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr)
 
 	return -EMSGSIZE;
 }
+
+int hns_roce_fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ib_mr)
+{
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_mr->device);
+	struct hns_roce_mr *hr_mr = to_hr_mr(ib_mr);
+	struct hns_roce_v2_mpt_entry context;
+	u32 data[MAX_ENTRY_NUM] = {};
+	int offset = 0;
+	int ret;
+
+	if (!hr_dev->hw->query_mpt)
+		return -EINVAL;
+
+	ret = hr_dev->hw->query_mpt(hr_dev, hr_mr->key, &context);
+	if (ret)
+		return -EINVAL;
+
+	data[offset++] = hr_reg_read(&context, MPT_ST);
+	data[offset++] = hr_reg_read(&context, MPT_PD);
+	data[offset++] = hr_reg_read(&context, MPT_LKEY);
+	data[offset++] = hr_reg_read(&context, MPT_LEN_L);
+	data[offset++] = hr_reg_read(&context, MPT_LEN_H);
+	data[offset++] = hr_reg_read(&context, MPT_PBL_SIZE);
+	data[offset++] = hr_reg_read(&context, MPT_PBL_HOP_NUM);
+	data[offset++] = hr_reg_read(&context, MPT_PBL_BA_PG_SZ);
+	data[offset++] = hr_reg_read(&context, MPT_PBL_BUF_PG_SZ);
+
+	ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, offset * sizeof(u32), data);
+
+	return ret;
+}
Signature · 1 line
2.30.0
#8 lore raw
In reply to Wenpeng Liang
Quoted 9 lines · Wenpeng Liang

The hns driver already supports CQ's restrack ops. Therefore, this patchset supports QP/MR's restrack ops and CQ/QP/MR's restrack raw ops. Among them, restrack ops dump the queue information maintained by the driver, and restrack raw ops dump the queue context maintained by ROCEE.

Changes since v1:
* Add support for restrack raw ops.
* Add the result dumped by rdmatool in the commit message.
* v1 Link: https://patchwork.kernel.org/project/linux-rdma/cover/[email protected]/

https://lore.kernel.org/all/[email protected]/

Quoted 10 lines

Wenpeng Liang (7):
RDMA/hns: Remove redundant DFX file and DFX ops structure RDMA/hns: Add or remove CQ's restrack attributes RDMA/hns: Support CQ's restrack raw ops for hns driver RDMA/hns: Support QP's restrack ops for hns driver RDMA/hns: Support QP's restrack raw ops for hns driver RDMA/hns: Support MR's restrack ops for hns driver RDMA/hns: Support MR's restrack raw ops for hns driver

Thanks, applied.