Extend nldev interface with contexts and SRQ
cover Extend nldev interface with contexts and SRQ
From: Leon Romanovsky <[email protected]>
This series from Neta extends nldev and restrack to provide an information about contexts and SRQs.
Thanks
Neta Ostrovsky (4):
RDMA/nldev: Return context information RDMA/restrack: Add support to get resource tracking for SRQ RDMA/nldev: Return SRQ information
RDMA/nldev: Add QP numbers to SRQ information
drivers/infiniband/core/nldev.c | 160 +++++++++++++++++++++++++++++ drivers/infiniband/core/restrack.c | 3 + drivers/infiniband/core/verbs.c | 7 ++ include/rdma/ib_verbs.h | 5 + include/rdma/restrack.h | 4 + include/uapi/rdma/rdma_netlink.h | 13 +++ 6 files changed, 192 insertions(+)
Signature · 1 line
2.30.2
PATCH 2/4 RDMA/restrack: Add support to get resource tracking for SRQ
From: Neta Ostrovsky <[email protected]>
In order to track SRQ resources, a new restrack object is initialized and added to the resource tracking database.
--- drivers/infiniband/core/restrack.c | 3 +++ drivers/infiniband/core/verbs.c | 7 +++++++ include/rdma/ib_verbs.h | 5 +++++ include/rdma/restrack.h | 4 ++++ 4 files changed, 19 insertions(+)
drivers/infiniband/core/restrack.c +3 -0
@@ -47,6 +47,7 @@ static const char *type2str(enum rdma_restrack_type type) [RDMA_RESTRACK_MR] = "MR", [RDMA_RESTRACK_CTX] = "CTX", [RDMA_RESTRACK_COUNTER] = "COUNTER", + [RDMA_RESTRACK_SRQ] = "SRQ", };return names[type];
@@ -141,6 +142,8 @@ static struct ib_device *res_to_dev(struct rdma_restrack_entry *res) return container_of(res, struct ib_ucontext, res)->device;case RDMA_RESTRACK_COUNTER : return container_of(res, struct rdma_counter, res)->device; +case RDMA_RESTRACK_SRQ : + return container_of(res, struct ib_srq, res)->device;default :WARN_ONCE (true,"Wrong resource tracking type %u\n" , res->type);return NULL ;
drivers/infiniband/core/verbs.c +7 -0
@@ -1039,8 +1039,12 @@ struct ib_srq *ib_create_srq_user(struct ib_pd *pd, }atomic_inc (&pd->usecnt); +rdma_restrack_new (&srq->res,RDMA_RESTRACK_SRQ ); +rdma_restrack_parent_name (&srq->res, &pd->res); + ret = pd->device->ops.create_srq (srq, srq_init_attr, udata);if (ret) { +rdma_restrack_put (&srq->res);atomic_dec (&srq->pd->usecnt);if (srq->srq_type ==IB_SRQT_XRC )atomic_dec (&srq->ext.xrc.xrcd->usecnt);
@@ -1050,6 +1054,8 @@ struct ib_srq *ib_create_srq_user(struct ib_pd *pd,return ERR_PTR (ret); } +rdma_restrack_add (&srq->res); +return srq; }EXPORT_SYMBOL (ib_create_srq_user);
@@ -1088,6 +1094,7 @@ int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata)atomic_dec (&srq->ext.xrc.xrcd->usecnt);if (ib_srq_has_cq (srq->srq_type))atomic_dec (&srq->ext.cq->usecnt); +rdma_restrack_del (&srq->res);kfree (srq);return ret;
include/rdma/ib_verbs.h +5 -0
@@ -1610,6 +1610,11 @@ struct ib_srq { } xrc; }; } ext; + +/* +* Implementation details of the RDMA core, don't use in drivers: +*/ +struct rdma_restrack_entry res; };enum ib_raw_packet_caps {
include/rdma/restrack.h +4 -0
@@ -49,6 +49,10 @@ enum rdma_restrack_type {* @RDMA_RESTRACK_COUNTER: Statistic Counter */ RDMA_RESTRACK_COUNTER , +/** +* @RDMA_RESTRACK_SRQ: Shared receive queue (SRQ) +*/ + RDMA_RESTRACK_SRQ,/** * @RDMA_RESTRACK_MAX: Last entry, used for array dclarations */
Signature · 1 line
2.30.2
PATCH 3/4 RDMA/nldev: Return SRQ information
In reply to Leon RomanovskyFrom: Neta Ostrovsky <[email protected]>
Extend the RDMA nldev return a SRQ information, like SRQ number, SRQ type, PD number, CQ number and process ID that created that SRQ.
Sample output:
$ rdma res show srq dev ibp8s0f0 srqn 0 type BASIC pdn 3 comm [ib_ipoib] dev ibp8s0f0 srqn 4 type BASIC pdn 9 pid 3581 comm ibv_srq_pingpon dev ibp8s0f0 srqn 5 type BASIC pdn 10 pid 3584 comm ibv_srq_pingpon dev ibp8s0f0 srqn 6 type BASIC pdn 11 pid 3590 comm ibv_srq_pingpon dev ibp8s0f1 srqn 0 type BASIC pdn 3 comm [ib_ipoib] dev ibp8s0f1 srqn 1 type BASIC pdn 4 pid 3586 comm ibv_srq_pingpon
--- drivers/infiniband/core/nldev.c | 42 ++++++++++++++++++++++++++++++++ include/uapi/rdma/rdma_netlink.h | 6 +++++ 2 files changed, 48 insertions(+)
drivers/infiniband/core/nldev.c +42 -0
@@ -132,6 +132,9 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { [RDMA_NLDEV_ATTR_RES_TYPE] = { .type = NLA_U8 }, [RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY]= { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_RES_USECNT] = { .type = NLA_U64 }, + [RDMA_NLDEV_ATTR_RES_SRQ] = { .type = NLA_NESTED }, + [RDMA_NLDEV_ATTR_RES_SRQN] = { .type = NLA_U32 }, + [RDMA_NLDEV_ATTR_RES_SRQ_ENTRY] = { .type = NLA_NESTED }, [RDMA_NLDEV_ATTR_SM_LID] = { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_SUBNET_PREFIX] = { .type = NLA_U64 }, [RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK] = { .type = NLA_U32 },
@@ -387,6 +390,7 @@ static int fill_res_info(struct sk_buff *msg, struct ib_device *device) [RDMA_RESTRACK_CM_ID] = "cm_id", [RDMA_RESTRACK_MR] = "mr", [RDMA_RESTRACK_CTX] = "ctx", + [RDMA_RESTRACK_SRQ] = "srq", };struct nlattr *table_attr;
@@ -719,6 +723,32 @@ static int fill_res_ctx_entry(struct sk_buff *msg, bool has_cap_net_admin,return fill_res_name_pid (msg, res); } +static int fill_res_srq_entry (struct sk_buff *msg,bool has_cap_net_admin, +struct rdma_restrack_entry *res,uint32_t port) +{ + struct ib_srq *srq = container_of(res, struct ib_srq, res); + +if (nla_put_u32 (msg,RDMA_NLDEV_ATTR_RES_SRQN , srq->res.id)) + goto err; + +if (nla_put_u8 (msg,RDMA_NLDEV_ATTR_RES_TYPE , srq->srq_type)) + goto err; + +if (nla_put_u32 (msg,RDMA_NLDEV_ATTR_RES_PDN , srq->pd->res.id)) + goto err; + +if (ib_srq_has_cq (srq->srq_type)) { +if (nla_put_u32 (msg,RDMA_NLDEV_ATTR_RES_CQN , + srq->ext.cq->res.id)) + goto err; + } + +return fill_res_name_pid (msg, res); + +err: +return -EMSGSIZE ; +} +static int fill_stat_counter_mode (struct sk_buff *msg,struct rdma_counter *counter) {
@@ -1258,6 +1288,13 @@ static const struct nldev_fill_res_entry fill_entries[RDMA_RESTRACK_MAX] = { .entry = RDMA_NLDEV_ATTR_RES_CTX_ENTRY, .id = RDMA_NLDEV_ATTR_RES_CTXN, }, + [RDMA_RESTRACK_SRQ] = { + .nldev_attr = RDMA_NLDEV_ATTR_RES_SRQ, + .flags = NLDEV_PER_DEV, + .entry = RDMA_NLDEV_ATTR_RES_SRQ_ENTRY, + .id = RDMA_NLDEV_ATTR_RES_SRQN, + }, + }; static int res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
@@ -1499,6 +1536,7 @@ RES_GET_FUNCS(mr, RDMA_RESTRACK_MR);RES_GET_FUNCS (mr_raw,RDMA_RESTRACK_MR );RES_GET_FUNCS (counter,RDMA_RESTRACK_COUNTER );RES_GET_FUNCS (ctx,RDMA_RESTRACK_CTX ); +RES_GET_FUNCS (srq,RDMA_RESTRACK_SRQ );static LIST_HEAD (link_ops);static DECLARE_RWSEM (link_ops_rwsem);
@@ -2166,6 +2204,10 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = { .doit = nldev_res_get_ctx_doit, .dump = nldev_res_get_ctx_dumpit, }, + [RDMA_NLDEV_CMD_RES_SRQ_GET] = { + .doit = nldev_res_get_srq_doit, + .dump = nldev_res_get_srq_dumpit, + }, [RDMA_NLDEV_CMD_SYS_GET] = { .doit = nldev_sys_get_doit, },
include/uapi/rdma/rdma_netlink.h +6 -0
@@ -295,6 +295,8 @@ enum rdma_nldev_command {RDMA_NLDEV_CMD_RES_CTX_GET ,/* can dump */ +RDMA_NLDEV_CMD_RES_SRQ_GET ,/* can dump */ + RDMA_NLDEV_NUM_OPS };
@@ -538,6 +540,10 @@ enum rdma_nldev_attr {RDMA_NLDEV_ATTR_RES_CTX ,/* nested table */ RDMA_NLDEV_ATTR_RES_CTX_ENTRY ,/* nested table */ +RDMA_NLDEV_ATTR_RES_SRQ ,/* nested table */ +RDMA_NLDEV_ATTR_RES_SRQ_ENTRY ,/* nested table */ + RDMA_NLDEV_ATTR_RES_SRQN, /* u32 */ +/* * Always the end */
Signature · 1 line
2.30.2
PATCH 4/4 RDMA/nldev: Add QP numbers to SRQ information
In reply to Leon RomanovskyFrom: Neta Ostrovsky <[email protected]>
Add QP numbers that are associated with the SRQ to the SRQ information. The QPs are displayed in a range form.
Sample output:
$ rdma res show srq dev ibp8s0f0 srqn 0 type BASIC pdn 3 comm [ib_ipoib] dev ibp8s0f0 srqn 4 type BASIC lqpn 125-128,130-140 pdn 9 pid 3581 comm ibv_srq_pingpon dev ibp8s0f0 srqn 5 type BASIC lqpn 141-156 pdn 10 pid 3584 comm ibv_srq_pingpon dev ibp8s0f0 srqn 6 type BASIC lqpn 157-172 pdn 11 pid 3590 comm ibv_srq_pingpon dev ibp8s0f1 srqn 0 type BASIC pdn 3 comm [ib_ipoib] dev ibp8s0f1 srqn 1 type BASIC lqpn 329-344 pdn 4 pid 3586 comm ibv_srq_pingpon
$ rdma res show srq lqpn 126-141 dev ibp8s0f0 srqn 4 type BASIC lqpn 126-128,130-140 pdn 9 pid 3581 comm ibv_srq_pingpon dev ibp8s0f0 srqn 5 type BASIC lqpn 141 pdn 10 pid 3584 comm ibv_srq_pingpon
$ rdma res show srq lqpn 127 dev ibp8s0f0 srqn 4 type BASIC lqpn 127 pdn 9 pid 3581 comm ibv_srq_pingpon
--- drivers/infiniband/core/nldev.c | 91 ++++++++++++++++++++++++++++++++ include/uapi/rdma/rdma_netlink.h | 2 + 2 files changed, 93 insertions(+)
drivers/infiniband/core/nldev.c +91 -0
@@ -135,6 +135,8 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { [RDMA_NLDEV_ATTR_RES_SRQ] = { .type = NLA_NESTED }, [RDMA_NLDEV_ATTR_RES_SRQN] = { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_RES_SRQ_ENTRY] = { .type = NLA_NESTED }, + [RDMA_NLDEV_ATTR_MIN_RANGE] = { .type = NLA_U32 }, + [RDMA_NLDEV_ATTR_MAX_RANGE] = { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_SM_LID] = { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_SUBNET_PREFIX] = { .type = NLA_U64 }, [RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK] = { .type = NLA_U32 },
@@ -723,6 +725,92 @@ static int fill_res_ctx_entry(struct sk_buff *msg, bool has_cap_net_admin,return fill_res_name_pid (msg, res); } +static int fill_res_range_qp_entry (struct sk_buff *msg,uint32_t min_range, +uint32_t max_range) +{ +struct nlattr *entry_attr; + +if (!min_range) +return 0 ; + + entry_attr =nla_nest_start (msg,RDMA_NLDEV_ATTR_RES_QP_ENTRY ); +if (!entry_attr) +return -EMSGSIZE ; + +if (min_range == max_range) { +if (nla_put_u32 (msg,RDMA_NLDEV_ATTR_RES_LQPN , min_range)) + goto err; + }else { +if (nla_put_u32 (msg,RDMA_NLDEV_ATTR_MIN_RANGE , min_range)) + goto err; +if (nla_put_u32 (msg,RDMA_NLDEV_ATTR_MAX_RANGE , max_range)) + goto err; + } +nla_nest_end (msg, entry_attr); +return 0 ; + +err: +nla_nest_cancel (msg, entry_attr); +return -EMSGSIZE ; +} + +static int fill_res_srq_qps (struct sk_buff *msg,struct ib_srq *srq) +{ +uint32_t min_range =0 , prev =0 ; +struct rdma_restrack_entry *res; +struct rdma_restrack_root *rt; +struct nlattr *table_attr; +struct ib_qp *qp =NULL ; +unsigned long id =0 ; + + table_attr =nla_nest_start (msg,RDMA_NLDEV_ATTR_RES_QP ); +if (!table_attr) +return -EMSGSIZE ; + + rt = &srq->device->res[RDMA_RESTRACK_QP ]; +xa_lock (&rt->xa); +xa_for_each (&rt->xa, id, res) { +if (!rdma_restrack_get (res)) +continue ; + + qp = container_of(res, struct ib_qp, res); +if (!qp->srq || (qp->srq->res.id != srq->res.id)) { +rdma_restrack_put (res); +continue ; + } + +if (qp->qp_num < prev) +/* qp_num should be ascending */ + goto err_loop; + +if (min_range ==0 ) { + min_range = qp->qp_num; + }else if (qp->qp_num > (prev +1 )) { +if (fill_res_range_qp_entry (msg, min_range, prev)) + goto err_loop; + + min_range = qp->qp_num; + } + prev = qp->qp_num; +rdma_restrack_put (res); + } + +xa_unlock (&rt->xa); + +if (fill_res_range_qp_entry (msg, min_range, prev)) + goto err; + +nla_nest_end (msg, table_attr); +return 0 ; + +err_loop: +rdma_restrack_put (res); +xa_unlock (&rt->xa); +err: +nla_nest_cancel (msg, table_attr); +return -EMSGSIZE ; +} +static int fill_res_srq_entry (struct sk_buff *msg,bool has_cap_net_admin,struct rdma_restrack_entry *res,uint32_t port) {
@@ -743,6 +831,9 @@ static int fill_res_srq_entry(struct sk_buff *msg, bool has_cap_net_admin, goto err; } +if (fill_res_srq_qps (msg, srq)) + goto err; +return fill_res_name_pid (msg, res); err:
include/uapi/rdma/rdma_netlink.h +2 -0
@@ -544,6 +544,8 @@ enum rdma_nldev_attr {RDMA_NLDEV_ATTR_RES_SRQ_ENTRY ,/* nested table */ RDMA_NLDEV_ATTR_RES_SRQN ,/* u32 */ +RDMA_NLDEV_ATTR_MIN_RANGE ,/* u32 */ + RDMA_NLDEV_ATTR_MAX_RANGE, /* u32 *//* * Always the end */
Signature · 1 line
2.30.2
PATCH 1/4 RDMA/nldev: Return context information
In reply to Leon RomanovskyFrom: Neta Ostrovsky <[email protected]>
Extend the RDMA nldev return a context information, like ctx number and process ID that created that context. This functionality is helpful to find orphan contexts that are not closed for some reason.
Sample output:
$ rdma res show ctx dev ibp8s0f0 ctxn 0 pid 980 comm ibv_rc_pingpong dev ibp8s0f0 ctxn 1 pid 981 comm ibv_rc_pingpong dev ibp8s0f0 ctxn 2 pid 992 comm ibv_rc_pingpong dev ibp8s0f1 ctxn 0 pid 984 comm ibv_rc_pingpong dev ibp8s0f1 ctxn 1 pid 987 comm ibv_rc_pingpong
$ rdma res show ctx dev ibp8s0f1 dev ibp8s0f1 ctxn 0 pid 984 comm ibv_rc_pingpong dev ibp8s0f1 ctxn 1 pid 987 comm ibv_rc_pingpong
--- drivers/infiniband/core/nldev.c | 27 +++++++++++++++++++++++++++ include/uapi/rdma/rdma_netlink.h | 5 +++++ 2 files changed, 32 insertions(+)
drivers/infiniband/core/nldev.c +27 -0
@@ -92,7 +92,9 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = { [RDMA_NLDEV_ATTR_RES_CQE] = { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_RES_CQN] = { .type = NLA_U32 }, [RDMA_NLDEV_ATTR_RES_CQ_ENTRY] = { .type = NLA_NESTED }, + [RDMA_NLDEV_ATTR_RES_CTX] = { .type = NLA_NESTED }, [RDMA_NLDEV_ATTR_RES_CTXN] = { .type = NLA_U32 }, + [RDMA_NLDEV_ATTR_RES_CTX_ENTRY] = { .type = NLA_NESTED }, [RDMA_NLDEV_ATTR_RES_DST_ADDR] = { .len = sizeof(struct __kernel_sockaddr_storage) }, [RDMA_NLDEV_ATTR_RES_IOVA] = { .type = NLA_U64 },
@@ -703,6 +705,20 @@ static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin, err:return -EMSGSIZE ; } +static int fill_res_ctx_entry (struct sk_buff *msg,bool has_cap_net_admin, +struct rdma_restrack_entry *res,uint32_t port) +{ + struct ib_ucontext *ctx = container_of(res, struct ib_ucontext, res); + +if (rdma_is_kernel_res (res)) +return 0 ; + +if (nla_put_u32 (msg,RDMA_NLDEV_ATTR_RES_CTXN , ctx->res.id)) +return -EMSGSIZE ; + +return fill_res_name_pid (msg, res); +} +static int fill_stat_counter_mode (struct sk_buff *msg,struct rdma_counter *counter) {
@@ -1236,6 +1252,12 @@ static const struct nldev_fill_res_entry fill_entries[RDMA_RESTRACK_MAX] = { .entry = RDMA_NLDEV_ATTR_STAT_COUNTER_ENTRY, .id = RDMA_NLDEV_ATTR_STAT_COUNTER_ID, }, + [RDMA_RESTRACK_CTX] = { + .nldev_attr = RDMA_NLDEV_ATTR_RES_CTX, + .flags = NLDEV_PER_DEV, + .entry = RDMA_NLDEV_ATTR_RES_CTX_ENTRY, + .id = RDMA_NLDEV_ATTR_RES_CTXN, + }, }; static int res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
@@ -1476,6 +1498,7 @@ RES_GET_FUNCS(pd, RDMA_RESTRACK_PD);RES_GET_FUNCS (mr,RDMA_RESTRACK_MR );RES_GET_FUNCS (mr_raw,RDMA_RESTRACK_MR );RES_GET_FUNCS (counter,RDMA_RESTRACK_COUNTER ); +RES_GET_FUNCS (ctx,RDMA_RESTRACK_CTX );static LIST_HEAD (link_ops);static DECLARE_RWSEM (link_ops_rwsem);
@@ -2139,6 +2162,10 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = { .doit = nldev_res_get_pd_doit, .dump = nldev_res_get_pd_dumpit, }, + [RDMA_NLDEV_CMD_RES_CTX_GET] = { + .doit = nldev_res_get_ctx_doit, + .dump = nldev_res_get_ctx_dumpit, + }, [RDMA_NLDEV_CMD_SYS_GET] = { .doit = nldev_sys_get_doit, },
include/uapi/rdma/rdma_netlink.h +5 -0
@@ -293,6 +293,8 @@ enum rdma_nldev_command {RDMA_NLDEV_CMD_RES_MR_GET_RAW , +RDMA_NLDEV_CMD_RES_CTX_GET ,/* can dump */ + RDMA_NLDEV_NUM_OPS };
@@ -533,6 +535,9 @@ enum rdma_nldev_attr {RDMA_NLDEV_ATTR_RES_RAW ,/* binary */ +RDMA_NLDEV_ATTR_RES_CTX ,/* nested table */ + RDMA_NLDEV_ATTR_RES_CTX_ENTRY, /* nested table */ +/* * Always the end */
Signature · 1 line
2.30.2
Quoted 5 lines · Leon Romanovsky
From: Leon Romanovsky <[email protected]>
This series from Neta extends nldev and restrack to provide an information about contexts and SRQs.
Quoted 7 lines
Thanks
Neta Ostrovsky (4):
RDMA/nldev: Return context information RDMA/restrack: Add support to get resource tracking for SRQ RDMA/nldev: Return SRQ information
RDMA/nldev: Add QP numbers to SRQ information
Applied to for-next, thanks
Jason