Control CQ adaptive moderation (RDMA DIM)
Subsystems (3)
cover Control CQ adaptive moderation (RDMA DIM)
From: Leon Romanovsky <[email protected]>
Hi,
This is supplementary part of RDMA DIM feature [1] accepted for the kernel v5.3. In this patch set Yamin extends rdmatool to get/set as a default this adaptive-moderation setting on IB device level and provides an information about DIM on/off status per-CQ.
Thanks
[1] https://lore.kernel.org/linux-rdma/[email protected]/
Yamin Friedman (2):
rdma: Control CQ adaptive moderation (DIM)
rdma: Document adaptive-moderation
man/man8/rdma-dev.8 | 16 ++++++++++++- rdma/dev.c | 55 ++++++++++++++++++++++++++++++++++++++++++++- rdma/rdma.h | 1 + rdma/res-cq.c | 15 +++++++++++++ rdma/utils.c | 6 +++++ 5 files changed, 91 insertions(+), 2 deletions(-)
--
2.20.1
PATCH 1/2 rdma: Control CQ adaptive moderation (DIM)
From: Yamin Friedman <[email protected]>
In order to set adaptive-moderation for an ib device the command is:
rdma dev set [DEV] adaptive-moderation [on|off]
rdma dev show -d
0: mlx5_0: node_type ca fw 16.25.0319 node_guid 248a:0703:00a5:29d0 sys_image_guid 248a:0703:00a5:29d0 adaptive-moderation on
caps: <BAD_PKEY_CNTR, BAD_QKEY_CNTR, AUTO_PATH_MIG, CHANGE_PHY_PORT, PORT_ACTIVE_EVENT, SYS_IMAGE_GUID, RC_RNR_NAK_GEN, MEM_WINDOW, XRC, MEM_MGT_EXTENSIONS, BLOCK_MULTICAST_LOOPBACK, MEM_WINDOW_TYPE_2B, RAW_IP_CSUM, CROSS_CHANNEL, MANAGED_FLOW_STEERING, SIGNATURE_HANDOVER, ON_DEMAND_PAGING, SG_GAPS_REG, RAW_SCATTER_FCS, PCI_WRITE_END_PADDING>
rdma resource show cq
dev mlx5_0 cqn 0 cqe 1023 users 4 poll-ctx UNBOUND_WORKQUEUE adaptive-moderation off comm [ib_core]
--- rdma/dev.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++- rdma/rdma.h | 1 + rdma/res-cq.c | 15 ++++++++++++++ rdma/utils.c | 6 ++++++ 4 files changed, 76 insertions(+), 1 deletion(-)
rdma/dev.c +54 -1
@@ -12,6 +12,7 @@ static int dev_help(struct rd *rd)pr_out ("Usage: %s dev show [DEV]\n" , rd->filename);pr_out (" %s dev set [DEV] name DEVNAME\n" , rd->filename);pr_out (" %s dev set [DEV] netns NSNAME\n" , rd->filename); +pr_out (" %s dev set [DEV] adaptive-moderation [on|off]\n" , rd->filename);return 0 ; }
@@ -167,6 +168,21 @@ static void dev_print_sys_image_guid(struct rd *rd, struct nlattr **tb)pr_out ("sys_image_guid %s " , str); } +static void dev_print_dim_setting (struct rd *rd,struct nlattr **tb) +{ +uint8_t dim_setting; + +if (!tb[RDMA_NLDEV_ATTR_DEV_DIM ]) +return ; + + dim_setting =mnl_attr_get_u8 (tb[RDMA_NLDEV_ATTR_DEV_DIM ]); +if (dim_setting >1 ) +return ; + +print_on_off (rd,"adaptive-moderation" , dim_setting); + +} +static const char *node_type_to_str (uint8_t node_type) {static const char *const node_type_str[] = {"unknown" ,"ca" ,
@@ -219,8 +235,10 @@ static int dev_parse_cb(const struct nlmsghdr *nlh, void *data)dev_print_fw (rd, tb);dev_print_node_guid (rd, tb);dev_print_sys_image_guid (rd, tb); -if (rd->show_details) +if (rd->show_details) { +dev_print_dim_setting (rd, tb);dev_print_caps (rd, tb); + }if (!rd->json_output)pr_out ("\n" );
@@ -308,12 +326,47 @@ done:return ret; } +static int dev_set_dim_sendmsg (struct rd *rd,uint8_t dim_setting) +{ +uint32_t seq; + +rd_prepare_msg (rd,RDMA_NLDEV_CMD_SET , &seq, + (NLM_F_REQUEST |NLM_F_ACK )); +mnl_attr_put_u32 (rd->nlh,RDMA_NLDEV_ATTR_DEV_INDEX , rd->dev_idx); +mnl_attr_put_u8 (rd->nlh,RDMA_NLDEV_ATTR_DEV_DIM , dim_setting); + +return rd_sendrecv_msg (rd, seq); +} + +static int dev_set_dim_off (struct rd *rd) +{ +return dev_set_dim_sendmsg (rd,0 ); +} + +static int dev_set_dim_on (struct rd *rd) +{ +return dev_set_dim_sendmsg (rd,1 ); +} + +static int dev_set_dim (struct rd *rd) +{ +const struct rd_cmd cmds[] = { + {NULL , dev_help}, + {"on" , dev_set_dim_on}, + {"off" , dev_set_dim_off}, + {0 } + }; + +return rd_exec_cmd (rd, cmds,"parameter" ); +} +static int dev_one_set (struct rd *rd) {const struct rd_cmd cmds[] = { {NULL , dev_help}, {"name" , dev_set_name}, {"netns" , dev_set_netns}, + {"adaptive-moderation" , dev_set_dim}, {0 } };
rdma/rdma.h +1 -0
@@ -136,6 +136,7 @@ int rd_attr_check(const struct nlattr *attr, int *typep);void print_driver_table (struct rd *rd,struct nlattr *tb);void newline (struct rd *rd);void newline_indent (struct rd *rd); +void print_on_off (struct rd *rd,const char *key_str,bool on);#define MAX_LINE_LENGTH 80#endif /* _RDMA_TOOL_H_ */
rdma/res-cq.c +15 -0
@@ -30,6 +30,20 @@ static void print_poll_ctx(struct rd *rd, uint8_t poll_ctx, struct nlattr *attr)pr_out ("poll-ctx %s " ,poll_ctx_to_str (poll_ctx)); } +static void print_cq_dim_setting (struct rd *rd,struct nlattr *attr) +{ +uint8_t dim_setting; + +if (!attr) +return ; + + dim_setting =mnl_attr_get_u8 (attr); +if (dim_setting >1 ) +return ; + +print_on_off (rd,"adaptive-moderation" , dim_setting); +} +static int res_cq_line (struct rd *rd,const char *name,int idx,struct nlattr **nla_line) {
@@ -97,6 +111,7 @@ static int res_cq_line(struct rd *rd, const char *name, int idx,res_print_uint (rd,"users" , users, nla_line[RDMA_NLDEV_ATTR_RES_USECNT ]);print_poll_ctx (rd, poll_ctx, nla_line[RDMA_NLDEV_ATTR_RES_POLL_CTX ]); +print_cq_dim_setting (rd, nla_line[RDMA_NLDEV_ATTR_DEV_DIM ]);res_print_uint (rd,"ctxn" , ctxn, nla_line[RDMA_NLDEV_ATTR_RES_CTXN ]);res_print_uint (rd,"pid" , pid, nla_line[RDMA_NLDEV_ATTR_RES_PID ]);print_comm (rd, comm, nla_line);
rdma/utils.c +6 -0
@@ -449,6 +449,7 @@ static const enum mnl_attr_data_type nldev_policy[RDMA_NLDEV_ATTR_MAX] = { [RDMA_NLDEV_ATTR_STAT_MODE] = MNL_TYPE_U32, [RDMA_NLDEV_ATTR_STAT_RES] = MNL_TYPE_U32, [RDMA_NLDEV_ATTR_STAT_AUTO_MODE_MASK] = MNL_TYPE_U32, + [RDMA_NLDEV_ATTR_DEV_DIM] = MNL_TYPE_U8, };int rd_attr_check (const struct nlattr *attr,int *typep)
@@ -789,6 +790,11 @@ static int print_driver_string(struct rd *rd, const char *key_str, } } +void print_on_off (struct rd *rd,const char *key_str,bool on) +{ +print_driver_string (rd, key_str, (on) ?"on" :"off" ); +} +static int print_driver_s32 (struct rd *rd,const char *key_str,int32_t val,enum rdma_nldev_print_type print_type) {
Signature · 1 line
2.20.1
PATCH 2/2 rdma: Document adaptive-moderation
In reply to Leon RomanovskyFrom: Yamin Friedman <[email protected]>
Add document of setting the adaptive-moderation for the ib device.
--- man/man8/rdma-dev.8 | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-)
man/man8/rdma-dev.8 +15 -1
@@ -34,11 +34,17 @@ rdma-dev \- RDMA device configuration .BR netns .BR NSNAME +.ti -8 +.B rdma dev set +.RI "[ " DEV " ]" +.BR adaptive-moderation +.BR [on/off] + .ti -8 .B rdma dev help .SH "DESCRIPTION" -.SS rdma dev set - rename RDMA device or set network namespace +.SS rdma dev set - rename RDMA device or set network namespace or set RDMA device adaptive-moderation .SS rdma dev show - display RDMA device attributes
@@ -70,6 +76,14 @@ Changes the network namespace of RDMA device to foo where foo is previously created using iproute2 ip command. .RE .PP +rdma dev set mlx5_3 adaptive-moderation [on/off] +.RS 4 +Sets the state of adaptive interrupt moderation for the RDMA device. +.RE +.RS 4 +This is a global setting for the RDMA device but the value is printed for each CQ individually because the state is constant from CQ allocation. +.RE +.PP .SH SEE ALSO .BR ip (8),
Signature · 1 line
2.20.1