Implement new netlink attribute for devlink-port function in iproute2
Subsystems (3)
cover Implement new netlink attribute for devlink-port function in iproute2
Patch implementing new netlink attribute for devlink-port function got merged to net-next.
https://lore.kernel.org/netdev/[email protected]/
Now there is a need to support these new attribute in the userspace tool. Implement roce and migratable port function attributes in devlink userspace tool. Update documentation.
Shay Drory (4):
devlink: Add uapi changes for roce and migratable port function attrs
devlink: Support setting port function roce cap
devlink: Support setting port function migratable cap
devlink: Add documentation for roce and migratable port function
attributes
devlink/devlink.c | 51 ++++++++++++++++++++++++++++++++++-- include/uapi/linux/devlink.h | 13 +++++++++ man/man8/devlink-port.8 | 24 +++++++++++++++++ 3 files changed, 86 insertions(+), 2 deletions(-)
Signature · 1 line
2.38.1
PATCH 1/4 devlink: Add uapi changes for roce and migratable port function attrs
Updating UAPI according to the changes merged to net-next.
https://lore.kernel.org/netdev/[email protected]/
--- include/uapi/linux/devlink.h | 13 +++++++++++++ 1 file changed, 13 insertions(+)
include/uapi/linux/devlink.h +13 -0
@@ -658,11 +658,24 @@ enum devlink_resource_unit { DEVLINK_RESOURCE_UNIT_ENTRY, }; +enum devlink_port_fn_attr_cap { +DEVLINK_PORT_FN_ATTR_CAP_ROCE_BIT , +DEVLINK_PORT_FN_ATTR_CAP_MIGRATABLE_BIT , + +/* Add new caps above */ + __DEVLINK_PORT_FN_ATTR_CAPS_MAX, +}; + +#define DEVLINK_PORT_FN_CAP_ROCE _BITUL(DEVLINK_PORT_FN_ATTR_CAP_ROCE_BIT) +#define DEVLINK_PORT_FN_CAP_MIGRATABLE \ + _BITUL(DEVLINK_PORT_FN_ATTR_CAP_MIGRATABLE_BIT) +enum devlink_port_function_attr {DEVLINK_PORT_FUNCTION_ATTR_UNSPEC ,DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR ,/* binary */ DEVLINK_PORT_FN_ATTR_STATE ,/* u8 */ DEVLINK_PORT_FN_ATTR_OPSTATE ,/* u8 */ +DEVLINK_PORT_FN_ATTR_CAPS ,/* bitfield32 */ __DEVLINK_PORT_FUNCTION_ATTR_MAX,DEVLINK_PORT_FUNCTION_ATTR_MAX = __DEVLINK_PORT_FUNCTION_ATTR_MAX -1
Signature · 1 line
2.38.1
PATCH 2/4 devlink: Support setting port function roce cap
In reply to Shay DrorySupport port function commands to enable / disable RoCE, this is used to control the port RoCE device capabilities.
When RoCE is disabled for a function of the port, function cannot create any RoCE specific resources (e.g GID table).
It also saves system memory utilization. For example disabling RoCE enable a VF/SF to save 1 Mbytes of system memory per function.
Example of a PCI VF port which supports a port function:
$ devlink port show pci/0000:06:00.0/2
pci/0000:06:00.0/2: type eth netdev enp6s0pf0vf1 flavour pcivf pfnum
0 vfnum 1
function:
hw_addr 00:00:00:00:00:00 roce enabled$ devlink port function set pci/0000:06:00.0/2 roce disable
$ devlink port show pci/0000:06:00.0/2
pci/0000:06:00.0/2: type eth netdev enp6s0pf0vf1 flavour pcivf pfnum
0 vfnum 1
function:
hw_addr 00:00:00:00:00:00 roce disabled--- devlink/devlink.c | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-)
devlink/devlink.c +33 -2
@@ -297,6 +297,7 @@ static void ifname_map_free(struct ifname_map *ifname_map)#define DL_OPT_SELFTESTS BIT(54)#define DL_OPT_PORT_FN_RATE_TX_PRIORITY BIT(55)#define DL_OPT_PORT_FN_RATE_TX_WEIGHT BIT(56) +#define DL_OPT_PORT_FN_CAPS BIT(57)struct dl_opts {uint64_t present;/* flags of present items */
@@ -362,6 +363,7 @@ struct dl_opts {uint32_t linecard_index;const char *linecard_type;bool selftests_opt[DEVLINK_ATTR_SELFTEST_ID_MAX +1 ]; +struct nla_bitfield32 port_fn_caps; };struct dl {
@@ -2116,6 +2118,18 @@ static int dl_argv_parse(struct dl *dl, uint64_t o_required,dl_arg_inc (dl); opts->linecard_type ="" ; o_found |=DL_OPT_LINECARD_TYPE ; + }else if (dl_argv_match (dl,"roce" ) && + (o_all &DL_OPT_PORT_FN_CAPS )) { +bool roce; + +dl_arg_inc (dl); + err =dl_argv_bool (dl, &roce); +if (err) +return err; + opts->port_fn_caps.selector |=DEVLINK_PORT_FN_CAP_ROCE ; +if (roce) + opts->port_fn_caps.value |=DEVLINK_PORT_FN_CAP_ROCE ; + o_found |=DL_OPT_PORT_FN_CAPS ; }else {pr_err ("Unknown option \"%s\"\n" ,dl_argv (dl));return -EINVAL ;
@@ -2146,6 +2160,10 @@ dl_function_attr_put(struct nlmsghdr *nlh, const struct dl_opts *opts)if (opts->present &DL_OPT_PORT_FUNCTION_STATE )mnl_attr_put_u8 (nlh,DEVLINK_PORT_FN_ATTR_STATE , opts->port_fn_state); +if (opts->present &DL_OPT_PORT_FN_CAPS ) +mnl_attr_put (nlh,DEVLINK_PORT_FN_ATTR_CAPS , +sizeof (opts->port_fn_caps), &opts->port_fn_caps); +mnl_attr_nest_end (nlh, nest); }
@@ -2336,7 +2354,8 @@ static void dl_opts_put(struct nlmsghdr *nlh, struct dl *dl)if (opts->present &DL_OPT_TRAP_POLICER_BURST )mnl_attr_put_u64 (nlh,DEVLINK_ATTR_TRAP_POLICER_BURST , opts->trap_policer_burst); -if (opts->present & (DL_OPT_PORT_FUNCTION_HW_ADDR |DL_OPT_PORT_FUNCTION_STATE )) +if (opts->present & (DL_OPT_PORT_FUNCTION_HW_ADDR |DL_OPT_PORT_FUNCTION_STATE | +DL_OPT_PORT_FN_CAPS ))dl_function_attr_put (nlh, opts);if (opts->present &DL_OPT_PORT_FLAVOUR )mnl_attr_put_u16 (nlh,DEVLINK_ATTR_PORT_FLAVOUR , opts->port_flavour);
@@ -4386,6 +4405,7 @@ static void cmd_port_help(void)pr_err (" devlink port split DEV/PORT_INDEX count COUNT\n" );pr_err (" devlink port unsplit DEV/PORT_INDEX\n" );pr_err (" devlink port function set DEV/PORT_INDEX [ hw_addr ADDR ] [ state { active | inactive } ]\n" ); +pr_err (" [ roce { enable | disable } ]\n" );pr_err (" devlink port function rate { help | show | add | del | set }\n" );pr_err (" devlink port param set DEV/PORT_INDEX name PARAMETER value VALUE cmode { permanent | driverinit | runtime }\n" );pr_err (" devlink port param show [DEV/PORT_INDEX name PARAMETER]\n" );
@@ -4499,6 +4519,15 @@ static void pr_out_port_function(struct dl *dl, struct nlattr **tb_port)print_string (PRINT_ANY ,"opstate" ," opstate %s" ,port_fn_opstate (state)); } +if (tb[DEVLINK_PORT_FN_ATTR_CAPS ]) { +struct nla_bitfield32 *port_fn_caps = +mnl_attr_get_payload (tb[DEVLINK_PORT_FN_ATTR_CAPS ]); + +if (port_fn_caps->selector &DEVLINK_PORT_FN_CAP_ROCE ) +print_string (PRINT_ANY ,"roce" ," roce %s" , + port_fn_caps->value &DEVLINK_PORT_FN_CAP_ROCE ? +"enable" :"disable" ); + }if (!dl->json_output)__pr_out_indent_dec ();
@@ -4693,6 +4722,7 @@ static int cmd_port_param_show(struct dl *dl)static void cmd_port_function_help (void ) {pr_err ("Usage: devlink port function set DEV/PORT_INDEX [ hw_addr ADDR ] [ state STATE ]\n" ); +pr_err (" [ roce { enable | disable } ]\n" );pr_err (" devlink port function rate { help | show | add | del | set }\n" ); }
@@ -4706,7 +4736,8 @@ static int cmd_port_function_set(struct dl *dl)return 0 ; } err =dl_argv_parse (dl,DL_OPT_HANDLEP , -DL_OPT_PORT_FUNCTION_HW_ADDR |DL_OPT_PORT_FUNCTION_STATE ); +DL_OPT_PORT_FUNCTION_HW_ADDR |DL_OPT_PORT_FUNCTION_STATE | +DL_OPT_PORT_FN_CAPS );if (err)return err;
Signature · 1 line
2.38.1
PATCH 3/4 devlink: Support setting port function migratable cap
In reply to Shay DrorySuppor port function commands to enable / disable migratable capability, this is used to set the port function as migratable.
Live migration is the process of transferring a live virtual machine from one physical host to another without disrupting its normal operation.
In order for a VM to be able to perform LM, all the VM components must be able to perform migration. e.g.: to be migratable.
In order for VF to be migratable, VF must be bound to VFIO driver with migration support.
When migratable capability is enable for a function of the port, the device is making the necessary preparations for the function to be migratable, which might include disabling features which cannot be migrated.
Example of LM with migratable function configuration:
Set migratable of the VF's port function.
$ devlink port show pci/0000:06:00.0/2
pci/0000:06:00.0/2: type eth netdev enp6s0pf0vf1 flavour pcivf pfnum 0
vfnum 1
function:
hw_addr 00:00:00:00:00:00 migratable disable$ devlink port function set pci/0000:06:00.0/2 migratable enable
$ devlink port show pci/0000:06:00.0/2
pci/0000:06:00.0/2: type eth netdev enp6s0pf0vf1 flavour pcivf pfnum 0
vfnum 1
function:
hw_addr 00:00:00:00:00:00 migratable enableBind VF to VFIO driver with migration support:
$ echo <pci_id> > /sys/bus/pci/devices/0000:08:00.0/driver/unbind $ echo mlx5_vfio_pci > /sys/bus/pci/devices/0000:08:00.0/driver_override $ echo <pci_id> > /sys/bus/pci/devices/0000:08:00.0/driver/bind
Attach VF to the VM.
Start the VM.
Perform LM.
--- devlink/devlink.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-)
devlink/devlink.c +18 -2
@@ -2130,6 +2130,18 @@ static int dl_argv_parse(struct dl *dl, uint64_t o_required,if (roce) opts->port_fn_caps.value |=DEVLINK_PORT_FN_CAP_ROCE ; o_found |=DL_OPT_PORT_FN_CAPS ; + }else if (dl_argv_match (dl,"migratable" ) && + (o_all &DL_OPT_PORT_FN_CAPS )) { +bool mig; + +dl_arg_inc (dl); + err =dl_argv_bool (dl, &mig); +if (err) +return err; + opts->port_fn_caps.selector |=DEVLINK_PORT_FN_CAP_MIGRATABLE ; +if (mig) + opts->port_fn_caps.value |=DEVLINK_PORT_FN_CAP_MIGRATABLE ; + o_found |=DL_OPT_PORT_FN_CAPS ; }else {pr_err ("Unknown option \"%s\"\n" ,dl_argv (dl));return -EINVAL ;
@@ -4405,7 +4417,7 @@ static void cmd_port_help(void)pr_err (" devlink port split DEV/PORT_INDEX count COUNT\n" );pr_err (" devlink port unsplit DEV/PORT_INDEX\n" );pr_err (" devlink port function set DEV/PORT_INDEX [ hw_addr ADDR ] [ state { active | inactive } ]\n" ); -pr_err (" [ roce { enable | disable } ]\n" ); +pr_err (" [ roce { enable | disable } ] [ migratable { enable | disable } ]\n" );pr_err (" devlink port function rate { help | show | add | del | set }\n" );pr_err (" devlink port param set DEV/PORT_INDEX name PARAMETER value VALUE cmode { permanent | driverinit | runtime }\n" );pr_err (" devlink port param show [DEV/PORT_INDEX name PARAMETER]\n" );
@@ -4527,6 +4539,10 @@ static void pr_out_port_function(struct dl *dl, struct nlattr **tb_port)print_string (PRINT_ANY ,"roce" ," roce %s" , port_fn_caps->value &DEVLINK_PORT_FN_CAP_ROCE ?"enable" :"disable" ); +if (port_fn_caps->selector &DEVLINK_PORT_FN_CAP_MIGRATABLE ) +print_string (PRINT_ANY ,"migratable" ," migratable %s" , + port_fn_caps->value &DEVLINK_PORT_FN_CAP_MIGRATABLE ? +"enable" :"disable" ); }if (!dl->json_output)
@@ -4722,7 +4738,7 @@ static int cmd_port_param_show(struct dl *dl)static void cmd_port_function_help (void ) {pr_err ("Usage: devlink port function set DEV/PORT_INDEX [ hw_addr ADDR ] [ state STATE ]\n" ); -pr_err (" [ roce { enable | disable } ]\n" ); +pr_err (" [ roce { enable | disable } ] [ migratable { enable | disable } ]\n" );pr_err (" devlink port function rate { help | show | add | del | set }\n" ); }
Signature · 1 line
2.38.1
PATCH 4/4 devlink: Add documentation for roce and migratable port function attributes
In reply to Shay DroryNew port function attributes roce and migratable were added. Update the man page for devlink-port to account for new attributes.
--- man/man8/devlink-port.8 | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+)
man/man8/devlink-port.8 +24 -0
@@ -71,6 +71,12 @@ devlink-port \- devlink port configuration .RI "[ " .BR state " { " active " | " inactive " }" .RI "]" +.RI "[ " +.BR roce " { " enable " | " disable " }" +.RI "]" +.RI "[ " +.BR migratable " { " enable " | " disable " }" +.RI "]" .ti -8 .BR "devlink port function rate "
@@ -208,6 +214,14 @@ New state of the function to change to. .I inactive - To inactivate the function and its device(s), set to inactive. +.TP +.BR roce " { " enable " | " disable " } " +Set the RoCE capability of the function. + +.TP +.BR migratable " { " enable " | " disable " } " +Set the migratable capability of the function. + .ti -8 .SS devlink port del - delete a devlink port .PP
@@ -327,6 +341,16 @@ Deactivate the function. This will initiate the function teardown which results in driver unload and device removal. .RE .PP +devlink port function set pci/0000:01:00.0/1 roce enable +.RS 4 +This will enable the RoCE functionality of the function. +.RE +.PP +devlink port function set pci/0000:01:00.0/1 migratable enable +.RS 4 +This will enable the migratable functionality of the function. +.RE +.PP devlink port function set pci/0000:01:00.0/1 hw_addr 00:00:00:11:22:33 state active .RS 4 Configure hardware address and also active the function. When a function is
Signature · 1 line
2.38.1
This series was applied to iproute2/iproute2-next.git (main) by David Ahern <[email protected]>:
Hello:
This series was applied to iproute2/iproute2-next.git (main) by David Ahern <[email protected]>:
Quoted 9 lines · On Sun, 11 Dec 2022 13:58:45 +0200 you wrote:
Patch implementing new netlink attribute for devlink-port function got merged to net-next.
https://lore.kernel.org/netdev/[email protected]/Now there is a need to support these new attribute in the userspace tool. Implement roce and migratable port function attributes in devlink userspace tool. Update documentation.
[...]
Here is the summary with links:
- [iproute2-next,1/4] devlink: Add uapi changes for roce and migratable port function attrs (no matching commit)
- [iproute2-next,2/4] devlink: Support setting port function roce cap
https://git.kernel.org/pub/scm/network/iproute2/iproute2-next.git/commit/?id=bb2eea918be7 - [iproute2-next,3/4] devlink: Support setting port function migratable cap https://git.kernel.org/pub/scm/network/iproute2/iproute2-next.git/commit/?id=32168d8a8894 - [iproute2-next,4/4] devlink: Add documentation for roce and migratable port function attributes https://git.kernel.org/pub/scm/network/iproute2/iproute2-next.git/commit/?id=fe036c3666a5
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html