RDMA/hns: Add more restrack attributes to hns driver
Subsystems (2)
v2 cover RDMA/hns: Add more restrack attributes to hns driver
The hns driver already supports CQ's restrack ops. Therefore, this patchset supports QP/MR's restrack ops and CQ/QP/MR's restrack raw ops. Among them, restrack ops dump the queue information maintained by the driver, and restrack raw ops dump the queue context maintained by ROCEE.
Changes since v1:
* Add support for restrack raw ops.
* Add the result dumped by rdmatool in the commit message.
* v1 Link: https://patchwork.kernel.org/project/linux-rdma/cover/[email protected]/
Wenpeng Liang (7):
RDMA/hns: Remove redundant DFX file and DFX ops structure RDMA/hns: Add or remove CQ's restrack attributes RDMA/hns: Support CQ's restrack raw ops for hns driver RDMA/hns: Support QP's restrack ops for hns driver RDMA/hns: Support QP's restrack raw ops for hns driver RDMA/hns: Support MR's restrack ops for hns driver RDMA/hns: Support MR's restrack raw ops for hns driver
drivers/infiniband/hw/hns/Makefile | 2 +- drivers/infiniband/hw/hns/hns_roce_device.h | 17 +- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 76 +++++- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 6 +- .../infiniband/hw/hns/hns_roce_hw_v2_dfx.c | 34 --- drivers/infiniband/hw/hns/hns_roce_main.c | 11 +- drivers/infiniband/hw/hns/hns_roce_restrack.c | 220 ++++++++++++++---- 7 files changed, 263 insertions(+), 103 deletions(-) delete mode 100644 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
--
2.30.0
PATCH v2 1/7 RDMA/hns: Remove redundant DFX file and DFX ops structure
There is no need to use a dedicated DXF file and DFX structure to manage the interface of the query queue context.
--- drivers/infiniband/hw/hns/Makefile | 2 +- drivers/infiniband/hw/hns/hns_roce_device.h | 10 ++---- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 35 ++++++++++++++++--- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 3 -- .../infiniband/hw/hns/hns_roce_hw_v2_dfx.c | 34 ------------------ drivers/infiniband/hw/hns/hns_roce_main.c | 6 +++- drivers/infiniband/hw/hns/hns_roce_restrack.c | 35 +++++++------------ 7 files changed, 50 insertions(+), 75 deletions(-) delete mode 100644 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
drivers/infiniband/hw/hns/Makefile +1 -1
@@ -10,6 +10,6 @@ hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \ hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o ifdef CONFIG_INFINIBAND_HNS_HIP08 -hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o $(hns-roce-objs) +hns-roce-hw-v2-objs := hns_roce_hw_v2.o $(hns-roce-objs) obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v2.o endif
drivers/infiniband/hw/hns/hns_roce_device.h +2 -8
@@ -849,11 +849,6 @@ struct hns_roce_caps {enum cong_type cong_type; }; -struct hns_roce_dfx_hw { -int (*query_cqc_info)(struct hns_roce_dev *hr_dev,u32 cqn, -int *buffer); -}; -enum hns_roce_device_state {HNS_ROCE_DEVICE_STATE_INITED ,HNS_ROCE_DEVICE_STATE_RST_DOWN ,
@@ -899,6 +894,7 @@ struct hns_roce_hw {int (*init_eq)(struct hns_roce_dev *hr_dev);void (*cleanup_eq)(struct hns_roce_dev *hr_dev);int (*write_srqc)(struct hns_roce_srq *srq,void *mb_buf); +int (*query_cqc)(struct hns_roce_dev *hr_dev,u32 cqn,void *buffer);const struct ib_device_ops *hns_roce_dev_ops;const struct ib_device_ops *hns_roce_dev_srq_ops; };
@@ -960,7 +956,6 @@ struct hns_roce_dev {void *priv;struct workqueue_struct *irq_workq;struct work_struct ecc_work; -const struct hns_roce_dfx_hw *dfx;u32 func_num;u32 is_vf;u32 cong_algo_tmpl_id;
@@ -1228,8 +1223,7 @@ u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u32 port, int gid_index);void hns_roce_handle_device_err (struct hns_roce_dev *hr_dev);int hns_roce_init (struct hns_roce_dev *hr_dev);void hns_roce_exit (struct hns_roce_dev *hr_dev); -int hns_roce_fill_res_cq_entry (struct sk_buff *msg, -struct ib_cq *ib_cq); +int hns_roce_fill_res_cq_entry (struct sk_buff *msg,struct ib_cq *ib_cq);struct hns_user_mmap_entry * hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, size_t length,
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +30 -5
@@ -5774,6 +5774,35 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)return ret; } +static int hns_roce_v2_query_cqc (struct hns_roce_dev *hr_dev,u32 cqn, +void *buffer) +{ +struct hns_roce_v2_cq_context *context; +struct hns_roce_cmd_mailbox *mailbox; +int ret; + + mailbox =hns_roce_alloc_cmd_mailbox (hr_dev); +if (IS_ERR (mailbox)) +return PTR_ERR (mailbox); + + context = mailbox->buf; + ret =hns_roce_cmd_mbox (hr_dev,0 , mailbox->dma, +HNS_ROCE_CMD_QUERY_CQC , cqn); +if (ret) { +ibdev_err (&hr_dev->ib_dev, +"failed to process cmd when querying CQ, ret = %d.\n" , + ret); + goto err_mailbox; + } + +memcpy (buffer, context,sizeof (*context)); + +err_mailbox: +hns_roce_free_cmd_mailbox (hr_dev, mailbox); + +return ret; +} +static void hns_roce_irq_work_handle (struct work_struct *work) { struct hns_roce_work *irq_work =
@@ -6575,10 +6604,6 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)kfree (eq_table->eq); } -static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = { - .query_cqc_info = hns_roce_v2_query_cqc_info, -}; -static const struct ib_device_ops hns_roce_v2_dev_ops = { .destroy_qp = hns_roce_v2_destroy_qp, .modify_cq = hns_roce_v2_modify_cq,
@@ -6619,6 +6644,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { .init_eq = hns_roce_v2_init_eq_table, .cleanup_eq = hns_roce_v2_cleanup_eq_table, .write_srqc = hns_roce_v2_write_srqc, + .query_cqc = hns_roce_v2_query_cqc, .hns_roce_dev_ops = &hns_roce_v2_dev_ops, .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops, };
@@ -6650,7 +6676,6 @@ static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev, hr_dev->is_vf = id->driver_data; hr_dev->dev = &handle->pdev->dev; hr_dev->hw = &hns_roce_hw_v2; - hr_dev->dfx = &hns_roce_dfx_hw_v2; hr_dev->sdb_offset =ROCEE_DB_SQ_L_0_REG ; hr_dev->odb_offset = hr_dev->sdb_offset;
drivers/infiniband/hw/hns/hns_roce_hw_v2.h +0 -3
@@ -1462,9 +1462,6 @@ struct hns_roce_sccc_clr_done {__le32 rsv[5 ]; }; -int hns_roce_v2_query_cqc_info (struct hns_roce_dev *hr_dev,u32 cqn, -int *buffer); -static inline void hns_roce_write64 (struct hns_roce_dev *hr_dev,__le32 val[2 ],void __iomem *dest) {
drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c deleted +0 -34
@@ -1,34 +0,0 @@ -// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) -// Copyright (c) 2019 Hisilicon Limited. - -#include "hnae3.h" -#include "hns_roce_device.h" -#include "hns_roce_cmd.h" -#include "hns_roce_hw_v2.h" - -int hns_roce_v2_query_cqc_info (struct hns_roce_dev *hr_dev,u32 cqn, -int *buffer) -{ -struct hns_roce_v2_cq_context *cq_context; -struct hns_roce_cmd_mailbox *mailbox; -int ret; - - mailbox =hns_roce_alloc_cmd_mailbox (hr_dev); -if (IS_ERR (mailbox)) -return PTR_ERR (mailbox); - - cq_context = mailbox->buf; - ret =hns_roce_cmd_mbox (hr_dev,0 , mailbox->dma,HNS_ROCE_CMD_QUERY_CQC , - cqn); -if (ret) { -dev_err (hr_dev->dev,"QUERY cqc cmd process error\n" ); - goto err_mailbox; - } - -memcpy (buffer, cq_context,sizeof (*cq_context)); - -err_mailbox: -hns_roce_free_cmd_mailbox (hr_dev, mailbox); - -return ret; -}
drivers/infiniband/hw/hns/hns_roce_main.c +5 -1
@@ -515,7 +515,6 @@ static const struct ib_device_ops hns_roce_dev_ops = { .destroy_ah = hns_roce_destroy_ah, .destroy_cq = hns_roce_destroy_cq, .disassociate_ucontext = hns_roce_disassociate_ucontext, - .fill_res_cq_entry = hns_roce_fill_res_cq_entry, .get_dma_mr = hns_roce_get_dma_mr, .get_link_layer = hns_roce_get_link_layer, .get_port_immutable = hns_roce_port_immutable,
@@ -566,6 +565,10 @@ static const struct ib_device_ops hns_roce_dev_xrcd_ops = { INIT_RDMA_OBJ_SIZE(ib_xrcd, hns_roce_xrcd, ibxrcd), }; +static const struct ib_device_ops hns_roce_dev_restrack_ops = { + .fill_res_cq_entry = hns_roce_fill_res_cq_entry, +}; +static int hns_roce_register_device (struct hns_roce_dev *hr_dev) {int ret;
@@ -605,6 +608,7 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev)ib_set_device_ops (ib_dev, hr_dev->hw->hns_roce_dev_ops);ib_set_device_ops (ib_dev, &hns_roce_dev_ops); +ib_set_device_ops (ib_dev, &hns_roce_dev_restrack_ops);for (i =0 ; i < hr_dev->caps.num_ports; i++) {if (!hr_dev->iboe.netdevs[i])continue ;
drivers/infiniband/hw/hns/hns_roce_restrack.c +12 -23
@@ -55,45 +55,34 @@ static int hns_roce_fill_cq(struct sk_buff *msg,return -EMSGSIZE ; } -int hns_roce_fill_res_cq_entry (struct sk_buff *msg, -struct ib_cq *ib_cq) +int hns_roce_fill_res_cq_entry (struct sk_buff *msg,struct ib_cq *ib_cq) {struct hns_roce_dev *hr_dev =to_hr_dev (ib_cq->device);struct hns_roce_cq *hr_cq =to_hr_cq (ib_cq); -struct hns_roce_v2_cq_context *context; +struct hns_roce_v2_cq_context context;struct nlattr *table_attr;int ret; -if (!hr_dev->dfx->query_cqc_info) +if (!hr_dev->hw->query_cqc)return -EINVAL ; - context =kzalloc (sizeof (struct hns_roce_v2_cq_context ),GFP_KERNEL ); -if (!context) -return -ENOMEM ; - - ret = hr_dev->dfx->query_cqc_info (hr_dev, hr_cq->cqn, (int *)context); + ret = hr_dev->hw->query_cqc (hr_dev, hr_cq->cqn, &context);if (ret) - goto err; +return -EINVAL ; table_attr =nla_nest_start (msg,RDMA_NLDEV_ATTR_DRIVER ); -if (!table_attr) { - ret = -EMSGSIZE ; - goto err; - } +if (!table_attr) +return -EMSGSIZE ; -if (hns_roce_fill_cq (msg, context)) { - ret = -EMSGSIZE ; - goto err_cancel_table; - } +if (hns_roce_fill_cq (msg, &context)) + goto err;nla_nest_end (msg, table_attr); -kfree (context);return 0 ; -err_cancel_table: -nla_nest_cancel (msg, table_attr); err: -kfree (context); -return ret; +nla_nest_cancel (msg, table_attr); + +return -EMSGSIZE ; }
Signature · 1 line
2.30.0
PATCH v2 2/7 RDMA/hns: Add or remove CQ's restrack attributes
In reply to Wenpeng LiangRemove the resttrack attributes from the queue context held by ROCEE, and add the resttrack attributes from the queue information maintained by the driver.
For example:
$ rdma res show cq dev hns_0 cqn 14 -dd -jp
[ {
"ifindex": 4,
"ifname": "hns_0",
"cqn": 14,
"cqe": 127,
"users": 1,
"adaptive-moderation": false,
"ctxn": 8,
"pid": 1524,
"comm": "ib_send_bw"
},
"drv_cq_depth": 128,
"drv_cons_index": 0,
"drv_cqe_size": 32,
"drv_arm_sn": 1
}--- drivers/infiniband/hw/hns/hns_roce_restrack.c | 67 +++---------------- 1 file changed, 10 insertions(+), 57 deletions(-)
drivers/infiniband/hw/hns/hns_roce_restrack.c +10 -57
@@ -9,72 +9,25 @@#include "hns_roce_device.h" #include "hns_roce_hw_v2.h" -static int hns_roce_fill_cq (struct sk_buff *msg, -struct hns_roce_v2_cq_context *context) -{ -if (rdma_nl_put_driver_u32 (msg,"state" , -hr_reg_read (context,CQC_ARM_ST ))) - - goto err; - -if (rdma_nl_put_driver_u32 (msg,"ceqn" , -hr_reg_read (context,CQC_CEQN ))) - goto err; - -if (rdma_nl_put_driver_u32 (msg,"cqn" , -hr_reg_read (context,CQC_CQN ))) - goto err; - -if (rdma_nl_put_driver_u32 (msg,"hopnum" , -hr_reg_read (context,CQC_CQE_HOP_NUM ))) - goto err; - -if (rdma_nl_put_driver_u32 (msg,"pi" , -hr_reg_read (context,CQC_CQ_PRODUCER_IDX ))) - goto err; - -if (rdma_nl_put_driver_u32 (msg,"ci" , -hr_reg_read (context,CQC_CQ_CONSUMER_IDX ))) - goto err; - -if (rdma_nl_put_driver_u32 (msg,"coalesce" , -hr_reg_read (context,CQC_CQ_MAX_CNT ))) - goto err; - -if (rdma_nl_put_driver_u32 (msg,"period" , -hr_reg_read (context,CQC_CQ_PERIOD ))) - goto err; - -if (rdma_nl_put_driver_u32 (msg,"cnt" , -hr_reg_read (context,CQC_CQE_CNT ))) - goto err; - -return 0 ; - -err: -return -EMSGSIZE ; -} -int hns_roce_fill_res_cq_entry (struct sk_buff *msg,struct ib_cq *ib_cq) { -struct hns_roce_dev *hr_dev =to_hr_dev (ib_cq->device);struct hns_roce_cq *hr_cq =to_hr_cq (ib_cq); -struct hns_roce_v2_cq_context context;struct nlattr *table_attr; -int ret; - -if (!hr_dev->hw->query_cqc) -return -EINVAL ; - - ret = hr_dev->hw->query_cqc (hr_dev, hr_cq->cqn, &context); -if (ret) -return -EINVAL ; table_attr =nla_nest_start (msg,RDMA_NLDEV_ATTR_DRIVER );if (!table_attr)return -EMSGSIZE ; -if (hns_roce_fill_cq (msg, &context)) +if (rdma_nl_put_driver_u32 (msg,"cq_depth" , hr_cq->cq_depth)) + goto err; + +if (rdma_nl_put_driver_u32 (msg,"cons_index" , hr_cq->cons_index)) + goto err; + +if (rdma_nl_put_driver_u32 (msg,"cqe_size" , hr_cq->cqe_size)) + goto err; + +if (rdma_nl_put_driver_u32 (msg,"arm_sn" , hr_cq->arm_sn)) goto err;nla_nest_end (msg, table_attr);
Signature · 1 line
2.30.0
PATCH v2 3/7 RDMA/hns: Support CQ's restrack raw ops for hns driver
In reply to Wenpeng LiangThe CQ raw restrack attributes come from the queue context maintained by the ROCEE.
For example:
$ rdma res show cq dev hns_0 cqn 14 -dd -jp -r
[ {
"ifindex": 4,
"ifname": "hns_0",
"data": [ 1,0,0,0,7,0,0,0,0,0,0,0,0,82,6,0,0,82,6,0,0,82,6,0,
1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,
6,0,0,0,0,0,0,0 ]
} ]--- drivers/infiniband/hw/hns/hns_roce_device.h | 1 + drivers/infiniband/hw/hns/hns_roce_main.c | 1 + drivers/infiniband/hw/hns/hns_roce_restrack.c | 39 +++++++++++++++++++ 3 files changed, 41 insertions(+)
drivers/infiniband/hw/hns/hns_roce_device.h +1 -0
@@ -1224,6 +1224,7 @@ void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev);int hns_roce_init (struct hns_roce_dev *hr_dev);void hns_roce_exit (struct hns_roce_dev *hr_dev);int hns_roce_fill_res_cq_entry (struct sk_buff *msg,struct ib_cq *ib_cq); +int hns_roce_fill_res_cq_entry_raw (struct sk_buff *msg,struct ib_cq *ib_cq);struct hns_user_mmap_entry * hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, size_t length,
drivers/infiniband/hw/hns/hns_roce_main.c +1 -0
@@ -567,6 +567,7 @@ static const struct ib_device_ops hns_roce_dev_xrcd_ops = {static const struct ib_device_ops hns_roce_dev_restrack_ops = { .fill_res_cq_entry = hns_roce_fill_res_cq_entry, + .fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw, };static int hns_roce_register_device (struct hns_roce_dev *hr_dev)
drivers/infiniband/hw/hns/hns_roce_restrack.c +39 -0
@@ -9,6 +9,8 @@#include "hns_roce_device.h" #include "hns_roce_hw_v2.h" +#define MAX_ENTRY_NUM 256 +int hns_roce_fill_res_cq_entry (struct sk_buff *msg,struct ib_cq *ib_cq) {struct hns_roce_cq *hr_cq =to_hr_cq (ib_cq);
@@ -39,3 +41,40 @@ int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq)return -EMSGSIZE ; } + +int hns_roce_fill_res_cq_entry_raw (struct sk_buff *msg,struct ib_cq *ib_cq) +{ +struct hns_roce_dev *hr_dev =to_hr_dev (ib_cq->device); +struct hns_roce_cq *hr_cq =to_hr_cq (ib_cq); +struct hns_roce_v2_cq_context context; +u32 data[MAX_ENTRY_NUM ] = {}; +int offset =0 ; +int ret; + +if (!hr_dev->hw->query_cqc) +return -EINVAL ; + + ret = hr_dev->hw->query_cqc (hr_dev, hr_cq->cqn, &context); +if (ret) +return -EINVAL ; + + data[offset++] =hr_reg_read (&context,CQC_CQ_ST ); + data[offset++] =hr_reg_read (&context,CQC_SHIFT ); + data[offset++] =hr_reg_read (&context,CQC_CQE_SIZE ); + data[offset++] =hr_reg_read (&context,CQC_CQE_CNT ); + data[offset++] =hr_reg_read (&context,CQC_CQ_PRODUCER_IDX ); + data[offset++] =hr_reg_read (&context,CQC_CQ_CONSUMER_IDX ); + data[offset++] =hr_reg_read (&context,CQC_DB_RECORD_EN ); + data[offset++] =hr_reg_read (&context,CQC_ARM_ST ); + data[offset++] =hr_reg_read (&context,CQC_CMD_SN ); + data[offset++] =hr_reg_read (&context,CQC_CEQN ); + data[offset++] =hr_reg_read (&context,CQC_CQ_MAX_CNT ); + data[offset++] =hr_reg_read (&context,CQC_CQ_PERIOD ); + data[offset++] =hr_reg_read (&context,CQC_CQE_HOP_NUM ); + data[offset++] =hr_reg_read (&context,CQC_CQE_BAR_PG_SZ ); + data[offset++] =hr_reg_read (&context,CQC_CQE_BUF_PG_SZ ); + + ret =nla_put (msg,RDMA_NLDEV_ATTR_RES_RAW , offset *sizeof (u32), data); + +return ret; +}
Signature · 1 line
2.30.0
PATCH v2 4/7 RDMA/hns: Support QP's restrack ops for hns driver
In reply to Wenpeng LiangThe QP restrack attributes come from the queue information maintained by the driver.
For example:
$ rdma res show qp link hns_0 lqpn 41 -jp -dd
[ {
"ifindex": 4,
"ifname": "hns_0",
"port": 1,
"lqpn": 41,
"rqpn": 40,
"type": "RC",
"state": "RTR",
"rq-psn": 12474738,
"sq-psn": 0,
"path-mig-state": "ARMED",
"pdn": 9,
"pid": 1523,
"comm": "ib_send_bw"
},
"drv_sq_wqe_cnt": 128,
"drv_sq_max_gs": 1,
"drv_rq_wqe_cnt": 512,
"drv_rq_max_gs": 2,
"drv_ext_sge_sge_cnt": 0
}--- drivers/infiniband/hw/hns/hns_roce_device.h | 1 + drivers/infiniband/hw/hns/hns_roce_main.c | 1 + drivers/infiniband/hw/hns/hns_roce_restrack.c | 34 +++++++++++++++++++ 3 files changed, 36 insertions(+)
drivers/infiniband/hw/hns/hns_roce_device.h +1 -0
@@ -1225,6 +1225,7 @@ int hns_roce_init(struct hns_roce_dev *hr_dev);void hns_roce_exit (struct hns_roce_dev *hr_dev);int hns_roce_fill_res_cq_entry (struct sk_buff *msg,struct ib_cq *ib_cq);int hns_roce_fill_res_cq_entry_raw (struct sk_buff *msg,struct ib_cq *ib_cq); +int hns_roce_fill_res_qp_entry (struct sk_buff *msg,struct ib_qp *ib_qp);struct hns_user_mmap_entry * hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, size_t length,
drivers/infiniband/hw/hns/hns_roce_main.c +1 -0
@@ -568,6 +568,7 @@ static const struct ib_device_ops hns_roce_dev_xrcd_ops = {static const struct ib_device_ops hns_roce_dev_restrack_ops = { .fill_res_cq_entry = hns_roce_fill_res_cq_entry, .fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw, + .fill_res_qp_entry = hns_roce_fill_res_qp_entry, };static int hns_roce_register_device (struct hns_roce_dev *hr_dev)
drivers/infiniband/hw/hns/hns_roce_restrack.c +34 -0
@@ -78,3 +78,37 @@ int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq)return ret; } + +int hns_roce_fill_res_qp_entry (struct sk_buff *msg,struct ib_qp *ib_qp) +{ +struct hns_roce_qp *hr_qp =to_hr_qp (ib_qp); +struct nlattr *table_attr; + + table_attr =nla_nest_start (msg,RDMA_NLDEV_ATTR_DRIVER ); +if (!table_attr) +return -EMSGSIZE ; + +if (rdma_nl_put_driver_u32_hex (msg,"sq_wqe_cnt" , hr_qp->sq.wqe_cnt)) + goto err; + +if (rdma_nl_put_driver_u32_hex (msg,"sq_max_gs" , hr_qp->sq.max_gs)) + goto err; + +if (rdma_nl_put_driver_u32_hex (msg,"rq_wqe_cnt" , hr_qp->rq.wqe_cnt)) + goto err; + +if (rdma_nl_put_driver_u32_hex (msg,"rq_max_gs" , hr_qp->rq.max_gs)) + goto err; + +if (rdma_nl_put_driver_u32_hex (msg,"ext_sge_sge_cnt" , hr_qp->sge.sge_cnt)) + goto err; + +nla_nest_end (msg, table_attr); + +return 0 ; + +err: +nla_nest_cancel (msg, table_attr); + +return -EMSGSIZE ; +}
Signature · 1 line
2.30.0
PATCH v2 5/7 RDMA/hns: Support QP's restrack raw ops for hns driver
In reply to Wenpeng LiangThe QP raw restrack attributes come from the queue context maintained by the ROCEE.
For example:
$ rdma res show qp link hns_0 -jp -dd -r
[ {
"ifindex": 4,
"ifname": "hns_0",
"data": [ 2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,
5,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,255,156,0,0,63,156,0,0,
7,0,0,0,1,0,0,0,9,0,0,0,0,0,0,0,2,0,0,0,2,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,63,156,0,
0,0,0,0,0 ]
} ]--- drivers/infiniband/hw/hns/hns_roce_device.h | 2 + drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 12 ++-- drivers/infiniband/hw/hns/hns_roce_main.c | 1 + drivers/infiniband/hw/hns/hns_roce_restrack.c | 56 +++++++++++++++++++ 4 files changed, 65 insertions(+), 6 deletions(-)
drivers/infiniband/hw/hns/hns_roce_device.h +2 -0
@@ -895,6 +895,7 @@ struct hns_roce_hw {void (*cleanup_eq)(struct hns_roce_dev *hr_dev);int (*write_srqc)(struct hns_roce_srq *srq,void *mb_buf);int (*query_cqc)(struct hns_roce_dev *hr_dev,u32 cqn,void *buffer); +int (*query_qpc)(struct hns_roce_dev *hr_dev,u32 qpn,void *buffer);const struct ib_device_ops *hns_roce_dev_ops;const struct ib_device_ops *hns_roce_dev_srq_ops; };
@@ -1226,6 +1227,7 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev);int hns_roce_fill_res_cq_entry (struct sk_buff *msg,struct ib_cq *ib_cq);int hns_roce_fill_res_cq_entry_raw (struct sk_buff *msg,struct ib_cq *ib_cq);int hns_roce_fill_res_qp_entry (struct sk_buff *msg,struct ib_qp *ib_qp); +int hns_roce_fill_res_qp_entry_raw (struct sk_buff *msg,struct ib_qp *ib_qp);struct hns_user_mmap_entry * hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, size_t length,
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +6 -6
@@ -5307,9 +5307,8 @@ static int to_ib_qp_st(enum hns_roce_v2_qp_state state)return (state <ARRAY_SIZE (map)) ? map[state] :-1 ; } -static int hns_roce_v2_query_qpc (struct hns_roce_dev *hr_dev, -struct hns_roce_qp *hr_qp, -struct hns_roce_v2_qp_context *hr_context) +static int hns_roce_v2_query_qpc (struct hns_roce_dev *hr_dev,u32 qpn, +void *buffer) {struct hns_roce_cmd_mailbox *mailbox;int ret;
@@ -5319,11 +5318,11 @@ static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,return PTR_ERR (mailbox); ret =hns_roce_cmd_mbox (hr_dev,0 , mailbox->dma,HNS_ROCE_CMD_QUERY_QPC , - hr_qp->qpn); + qpn);if (ret) goto out; -memcpy (hr_context, mailbox->buf, hr_dev->caps.qpc_sz); +memcpy (buffer, mailbox->buf, hr_dev->caps.qpc_sz); out:hns_roce_free_cmd_mailbox (hr_dev, mailbox);
@@ -5353,7 +5352,7 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, goto done; } - ret =hns_roce_v2_query_qpc (hr_dev, hr_qp, &context); + ret =hns_roce_v2_query_qpc (hr_dev, hr_qp->qpn, &context);if (ret) {ibdev_err (ibdev,"failed to query QPC, ret = %d.\n" , ret); ret = -EINVAL ;
@@ -6645,6 +6644,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { .cleanup_eq = hns_roce_v2_cleanup_eq_table, .write_srqc = hns_roce_v2_write_srqc, .query_cqc = hns_roce_v2_query_cqc, + .query_qpc = hns_roce_v2_query_qpc, .hns_roce_dev_ops = &hns_roce_v2_dev_ops, .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops, };
drivers/infiniband/hw/hns/hns_roce_main.c +1 -0
@@ -569,6 +569,7 @@ static const struct ib_device_ops hns_roce_dev_restrack_ops = { .fill_res_cq_entry = hns_roce_fill_res_cq_entry, .fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw, .fill_res_qp_entry = hns_roce_fill_res_qp_entry, + .fill_res_qp_entry_raw = hns_roce_fill_res_qp_entry_raw, };static int hns_roce_register_device (struct hns_roce_dev *hr_dev)
drivers/infiniband/hw/hns/hns_roce_restrack.c +56 -0
59 lines not on this page · raw message
Signature · 1 line
2.30.0
PATCH v2 6/7 RDMA/hns: Support MR's restrack ops for hns driver
In reply to Wenpeng LiangThe MR restrack attributes come from the queue information maintained by the driver.
For example:
$ rdma res show mr dev hns_0 mrn 6 -dd -jp
[ {
"ifindex": 4,
"ifname": "hns_0",
"mrn": 6,
"rkey": "300",
"lkey": "300",
"mrlen": 131072,
"pdn": 8,
"pid": 1524,
"comm": "ib_send_bw"
},
"drv_pbl_hop_num": 2,
"drv_ba_pg_shift": 14,
"drv_buf_pg_shift": 12
}--- drivers/infiniband/hw/hns/hns_roce_device.h | 1 + drivers/infiniband/hw/hns/hns_roce_main.c | 1 + drivers/infiniband/hw/hns/hns_roce_restrack.c | 30 +++++++++++++++++++ 3 files changed, 32 insertions(+)
drivers/infiniband/hw/hns/hns_roce_device.h +1 -0
@@ -1228,6 +1228,7 @@ int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);int hns_roce_fill_res_cq_entry_raw (struct sk_buff *msg,struct ib_cq *ib_cq);int hns_roce_fill_res_qp_entry (struct sk_buff *msg,struct ib_qp *ib_qp);int hns_roce_fill_res_qp_entry_raw (struct sk_buff *msg,struct ib_qp *ib_qp); +int hns_roce_fill_res_mr_entry (struct sk_buff *msg,struct ib_mr *ib_mr);struct hns_user_mmap_entry * hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, size_t length,
drivers/infiniband/hw/hns/hns_roce_main.c +1 -0
@@ -570,6 +570,7 @@ static const struct ib_device_ops hns_roce_dev_restrack_ops = { .fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw, .fill_res_qp_entry = hns_roce_fill_res_qp_entry, .fill_res_qp_entry_raw = hns_roce_fill_res_qp_entry_raw, + .fill_res_mr_entry = hns_roce_fill_res_mr_entry, };static int hns_roce_register_device (struct hns_roce_dev *hr_dev)
drivers/infiniband/hw/hns/hns_roce_restrack.c +30 -0
@@ -168,3 +168,33 @@ int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp)return ret; } + +int hns_roce_fill_res_mr_entry (struct sk_buff *msg,struct ib_mr *ib_mr) +{ +struct hns_roce_mr *hr_mr =to_hr_mr (ib_mr); +struct nlattr *table_attr; + + table_attr =nla_nest_start (msg,RDMA_NLDEV_ATTR_DRIVER ); +if (!table_attr) +return -EMSGSIZE ; + +if (rdma_nl_put_driver_u32_hex (msg,"pbl_hop_num" , hr_mr->pbl_hop_num)) + goto err; + +if (rdma_nl_put_driver_u32_hex (msg,"ba_pg_shift" , + hr_mr->pbl_mtr.hem_cfg.ba_pg_shift)) + goto err; + +if (rdma_nl_put_driver_u32_hex (msg,"buf_pg_shift" , + hr_mr->pbl_mtr.hem_cfg.buf_pg_shift)) + goto err; + +nla_nest_end (msg, table_attr); + +return 0 ; + +err: +nla_nest_cancel (msg, table_attr); + +return -EMSGSIZE ; +}
Signature · 1 line
2.30.0
PATCH v2 7/7 RDMA/hns: Support MR's restrack raw ops for hns driver
In reply to Wenpeng LiangThe MR raw restrack attributes come from the queue context maintained by the ROCEE.
For example:
$ rdma res show mr dev hns_0 mrn 6 -dd -jp -r
[ {
"ifindex": 4,
"ifname": "hns_0",
"data": [ 1,0,0,0,2,0,0,0,0,3,0,0,0,0,2,0,0,0,0,0,32,0,0,0,2,0,0,0,
2,0,0,0,0,0,0,0 ]
} ]--- drivers/infiniband/hw/hns/hns_roce_device.h | 2 ++ drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 29 +++++++++++++++++ drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 3 +- drivers/infiniband/hw/hns/hns_roce_main.c | 1 + drivers/infiniband/hw/hns/hns_roce_restrack.c | 31 +++++++++++++++++++ 5 files changed, 65 insertions(+), 1 deletion(-)
drivers/infiniband/hw/hns/hns_roce_device.h +2 -0
@@ -896,6 +896,7 @@ struct hns_roce_hw {int (*write_srqc)(struct hns_roce_srq *srq,void *mb_buf);int (*query_cqc)(struct hns_roce_dev *hr_dev,u32 cqn,void *buffer);int (*query_qpc)(struct hns_roce_dev *hr_dev,u32 qpn,void *buffer); +int (*query_mpt)(struct hns_roce_dev *hr_dev,u32 key,void *buffer);const struct ib_device_ops *hns_roce_dev_ops;const struct ib_device_ops *hns_roce_dev_srq_ops; };
@@ -1229,6 +1230,7 @@ int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);int hns_roce_fill_res_qp_entry (struct sk_buff *msg,struct ib_qp *ib_qp);int hns_roce_fill_res_qp_entry_raw (struct sk_buff *msg,struct ib_qp *ib_qp);int hns_roce_fill_res_mr_entry (struct sk_buff *msg,struct ib_mr *ib_mr); +int hns_roce_fill_res_mr_entry_raw (struct sk_buff *msg,struct ib_mr *ib_mr);struct hns_user_mmap_entry * hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, size_t length,
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +29 -0
@@ -5802,6 +5802,34 @@ static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn,return ret; } +int hns_roce_v2_query_mpt (struct hns_roce_dev *hr_dev,u32 key,void *buffer) +{ +struct hns_roce_v2_mpt_entry *context; +struct hns_roce_cmd_mailbox *mailbox; +int ret; + + mailbox =hns_roce_alloc_cmd_mailbox (hr_dev); +if (IS_ERR (mailbox)) +return PTR_ERR (mailbox); + + context = mailbox->buf; + ret =hns_roce_cmd_mbox (hr_dev,0 , mailbox->dma,HNS_ROCE_CMD_QUERY_MPT , +key_to_hw_index (key)); +if (ret) { +ibdev_err (&hr_dev->ib_dev, +"failed to process cmd when querying MPT, ret = %d.\n" , + ret); + goto err_mailbox; + } + +memcpy (buffer, context,sizeof (*context)); + +err_mailbox: +hns_roce_free_cmd_mailbox (hr_dev, mailbox); + +return ret; +} +static void hns_roce_irq_work_handle (struct work_struct *work) { struct hns_roce_work *irq_work =
@@ -6645,6 +6673,7 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { .write_srqc = hns_roce_v2_write_srqc, .query_cqc = hns_roce_v2_query_cqc, .query_qpc = hns_roce_v2_query_qpc, + .query_mpt = hns_roce_v2_query_mpt, .hns_roce_dev_ops = &hns_roce_v2_dev_ops, .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops, };
drivers/infiniband/hw/hns/hns_roce_hw_v2.h +2 -1
@@ -758,7 +758,8 @@ struct hns_roce_v2_mpt_entry {#define MPT_INNER_PA_VLD MPT_FIELD_LOC(71, 71)#define MPT_MW_BIND_QPN MPT_FIELD_LOC(95, 72)#define MPT_BOUND_LKEY MPT_FIELD_LOC(127, 96) -#define MPT_LEN MPT_FIELD_LOC(191, 128) +#define MPT_LEN_L MPT_FIELD_LOC(159, 128) +#define MPT_LEN_H MPT_FIELD_LOC(191, 160)#define MPT_LKEY MPT_FIELD_LOC(223, 192)#define MPT_VA MPT_FIELD_LOC(287, 224)#define MPT_PBL_SIZE MPT_FIELD_LOC(319, 288)
drivers/infiniband/hw/hns/hns_roce_main.c +1 -0
@@ -571,6 +571,7 @@ static const struct ib_device_ops hns_roce_dev_restrack_ops = { .fill_res_qp_entry = hns_roce_fill_res_qp_entry, .fill_res_qp_entry_raw = hns_roce_fill_res_qp_entry_raw, .fill_res_mr_entry = hns_roce_fill_res_mr_entry, + .fill_res_mr_entry_raw = hns_roce_fill_res_mr_entry_raw, };static int hns_roce_register_device (struct hns_roce_dev *hr_dev)
drivers/infiniband/hw/hns/hns_roce_restrack.c +31 -0
@@ -198,3 +198,34 @@ int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr)return -EMSGSIZE ; } + +int hns_roce_fill_res_mr_entry_raw (struct sk_buff *msg,struct ib_mr *ib_mr) +{ +struct hns_roce_dev *hr_dev =to_hr_dev (ib_mr->device); +struct hns_roce_mr *hr_mr =to_hr_mr (ib_mr); +struct hns_roce_v2_mpt_entry context; +u32 data[MAX_ENTRY_NUM ] = {}; +int offset =0 ; +int ret; + +if (!hr_dev->hw->query_mpt) +return -EINVAL ; + + ret = hr_dev->hw->query_mpt (hr_dev, hr_mr->key, &context); +if (ret) +return -EINVAL ; + + data[offset++] =hr_reg_read (&context,MPT_ST ); + data[offset++] =hr_reg_read (&context,MPT_PD ); + data[offset++] =hr_reg_read (&context,MPT_LKEY ); + data[offset++] =hr_reg_read (&context,MPT_LEN_L ); + data[offset++] =hr_reg_read (&context,MPT_LEN_H ); + data[offset++] =hr_reg_read (&context,MPT_PBL_SIZE ); + data[offset++] =hr_reg_read (&context,MPT_PBL_HOP_NUM ); + data[offset++] =hr_reg_read (&context,MPT_PBL_BA_PG_SZ ); + data[offset++] =hr_reg_read (&context,MPT_PBL_BUF_PG_SZ ); + + ret =nla_put (msg,RDMA_NLDEV_ATTR_RES_RAW , offset *sizeof (u32), data); + +return ret; +}
Signature · 1 line
2.30.0
Leon Romanovsky
AppliedQuoted 9 lines · Wenpeng Liang
The hns driver already supports CQ's restrack ops. Therefore, this patchset supports QP/MR's restrack ops and CQ/QP/MR's restrack raw ops. Among them, restrack ops dump the queue information maintained by the driver, and restrack raw ops dump the queue context maintained by ROCEE.
Changes since v1:
* Add support for restrack raw ops.
* Add the result dumped by rdmatool in the commit message.
* v1 Link: https://patchwork.kernel.org/project/linux-rdma/cover/[email protected]/
https://lore.kernel.org/all/[email protected]/
Quoted 10 lines
Wenpeng Liang (7):
RDMA/hns: Remove redundant DFX file and DFX ops structure RDMA/hns: Add or remove CQ's restrack attributes RDMA/hns: Support CQ's restrack raw ops for hns driver RDMA/hns: Support QP's restrack ops for hns driver RDMA/hns: Support QP's restrack raw ops for hns driver RDMA/hns: Support MR's restrack ops for hns driver RDMA/hns: Support MR's restrack raw ops for hns driver
Thanks, applied.