Add 800Gb (XDR) speed support
Subsystems (5)
cover Add 800Gb (XDR) speed support
From: Leon Romanovsky <[email protected]>
Hi,
This series extends RDMA subsystem and mlx5_ib driver to support 800Gb (XDR) speed which was added to IBTA v1.7 specification.
Thanks
Or Har-Toov (2):
IB/core: Add support for XDR link speed
IB/mlx5: Expose XDR speed through MAD
Patrisious Haddad (4):
IB/mlx5: Add support for 800G_8X lane speed IB/mlx5: Rename 400G_8X speed to comply to naming convention IB/mlx5: Adjust mlx5 rate mapping to support 800Gb RDMA/ipoib: Add support for XDR speed in ethtool
drivers/infiniband/core/sysfs.c | 4 ++++ drivers/infiniband/core/uverbs_std_types_device.c | 3 ++- drivers/infiniband/core/verbs.c | 3 +++ drivers/infiniband/hw/mlx5/mad.c | 13 +++++++++++++ drivers/infiniband/hw/mlx5/main.c | 6 +++++- drivers/infiniband/hw/mlx5/qp.c | 2 +- drivers/infiniband/ulp/ipoib/ipoib_ethtool.c | 2 ++ drivers/net/ethernet/mellanox/mlx5/core/port.c | 3 ++- include/linux/mlx5/port.h | 3 ++- include/rdma/ib_mad.h | 2 ++ include/rdma/ib_verbs.h | 2 ++ include/uapi/rdma/ib_user_ioctl_verbs.h | 3 ++- 12 files changed, 40 insertions(+), 6 deletions(-)
Signature · 1 line
2.41.0
PATCH 1/6 IB/core: Add support for XDR link speed
From: Or Har-Toov <[email protected]>
Add new IBTA speed XDR, the new rate that was added to Infiniband spec as part of XDR and supporting signaling rate of 200Gb.
In order to report that value to rdma-core, add new u32 field to query_port response.
--- drivers/infiniband/core/sysfs.c | 4 ++++ drivers/infiniband/core/uverbs_std_types_device.c | 3 ++- drivers/infiniband/core/verbs.c | 3 +++ include/rdma/ib_verbs.h | 2 ++ include/uapi/rdma/ib_user_ioctl_verbs.h | 3 ++- 5 files changed, 13 insertions(+), 2 deletions(-)
drivers/infiniband/core/sysfs.c +4 -0
@@ -342,6 +342,10 @@ static ssize_t rate_show(struct ib_device *ibdev, u32 port_num, speed =" NDR" ; rate =1000 ;break ; +case IB_SPEED_XDR : + speed =" XDR" ; + rate =2000 ; +break ;case IB_SPEED_SDR :default :/* default to SDR for invalid rates */ speed =" SDR" ;
drivers/infiniband/core/uverbs_std_types_device.c +2 -1
@@ -203,6 +203,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_PORT)(copy_port_attr_to_resp (&attr, &resp.legacy_resp, ib_dev, port_num); resp.port_cap_flags2 = attr.port_cap_flags2; + resp.active_speed_ex = attr.active_speed;return uverbs_copy_to_struct_or_zero (attrs,UVERBS_ATTR_QUERY_PORT_RESP , &resp,sizeof (resp));
@@ -461,7 +462,7 @@ DECLARE_UVERBS_NAMED_METHOD(UVERBS_ATTR_PTR_OUT (UVERBS_ATTR_QUERY_PORT_RESP , UVERBS_ATTR_STRUCT(struct ib_uverbs_query_port_resp_ex, - reserved), + active_speed_ex), UA_MANDATORY)); DECLARE_UVERBS_NAMED_METHOD(
drivers/infiniband/core/verbs.c +3 -0
@@ -147,6 +147,7 @@ __attribute_const__ int ib_rate_to_mult(enum ib_rate rate)case IB_RATE_50_GBPS :return 20 ;case IB_RATE_400_GBPS :return 160 ;case IB_RATE_600_GBPS :return 240 ; +case IB_RATE_800_GBPS :return 320 ;default :return -1 ; } }
@@ -176,6 +177,7 @@ __attribute_const__ enum ib_rate mult_to_ib_rate(int mult)case 20 :return IB_RATE_50_GBPS ;case 160 :return IB_RATE_400_GBPS ;case 240 :return IB_RATE_600_GBPS ; +case 320 :return IB_RATE_800_GBPS ;default :return IB_RATE_PORT_CURRENT ; } }
@@ -205,6 +207,7 @@ __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate)case IB_RATE_50_GBPS :return 53125 ;case IB_RATE_400_GBPS :return 425000 ;case IB_RATE_600_GBPS :return 637500 ; +case IB_RATE_800_GBPS :return 850000 ;default :return -1 ; } }
include/rdma/ib_verbs.h +2 -0
@@ -561,6 +561,7 @@ enum ib_port_speed {IB_SPEED_EDR =32 ,IB_SPEED_HDR =64 ,IB_SPEED_NDR =128 , + IB_SPEED_XDR = 256, }; enum ib_stat_flag {
@@ -840,6 +841,7 @@ enum ib_rate {IB_RATE_50_GBPS =20 ,IB_RATE_400_GBPS =21 ,IB_RATE_600_GBPS =22 , + IB_RATE_800_GBPS = 23, };/**
include/uapi/rdma/ib_user_ioctl_verbs.h +2 -1
@@ -220,7 +220,8 @@ enum ib_uverbs_advise_mr_flag {struct ib_uverbs_query_port_resp_ex {struct ib_uverbs_query_port_resp legacy_resp;__u16 port_cap_flags2; -__u8 reserved[6 ]; +__u8 reserved[2 ]; +__u32 active_speed_ex; };struct ib_uverbs_qp_cap {
Signature · 1 line
2.41.0
PATCH 2/6 IB/mlx5: Expose XDR speed through MAD
In reply to Leon RomanovskyFrom: Or Har-Toov <[email protected]>
Under MAD query port, Report NDR speed when NDR is supported in the port capability mask.
--- drivers/infiniband/hw/mlx5/mad.c | 13 +++++++++++++ include/rdma/ib_mad.h | 2 ++ 2 files changed, 15 insertions(+)
drivers/infiniband/hw/mlx5/mad.c +13 -0
@@ -619,6 +619,19 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u32 port, } } +/* Check if extended speeds 2 (XDR/...) are supported */ +if (props->port_cap_flags &IB_PORT_CAP_MASK2_SUP && + props->port_cap_flags2 &IB_PORT_EXTENDED_SPEEDS2_SUP ) { + ext_active_speed = (out_mad->data[56 ] >>4 ) &0x6 ; + +switch (ext_active_speed) { +case 2 : +if (props->port_cap_flags2 &IB_PORT_LINK_SPEED_XDR_SUP ) + props->active_speed =IB_SPEED_XDR ; +break ; + } + } +/* If reported active speed is QDR, check if is FDR-10 */ if (props->active_speed ==4 ) { if (dev->port_caps[port - 1].ext_port_cap &
include/rdma/ib_mad.h +2 -0
@@ -277,6 +277,8 @@ enum ib_port_capability_mask2_bits {IB_PORT_LINK_WIDTH_2X_SUP =1 <<4 ,IB_PORT_LINK_SPEED_HDR_SUP =1 <<5 ,IB_PORT_LINK_SPEED_NDR_SUP =1 <<10 , +IB_PORT_EXTENDED_SPEEDS2_SUP =1 <<11 , + IB_PORT_LINK_SPEED_XDR_SUP = 1 << 12, };#define OPA_CLASS_PORT_INFO_PR_SUPPORT BIT(26)
Signature · 1 line
2.41.0
PATCH 3/6 IB/mlx5: Add support for 800G_8X lane speed
In reply to Leon RomanovskyFrom: Patrisious Haddad <[email protected]>
Add a check for 800G_8X speed when querying PTYS and report it back correctly when needed.
--- drivers/infiniband/hw/mlx5/main.c | 4 ++++ drivers/net/ethernet/mellanox/mlx5/core/port.c | 1 + include/linux/mlx5/port.h | 1 + 3 files changed, 6 insertions(+)
drivers/infiniband/hw/mlx5/main.c +4 -0
@@ -482,6 +482,10 @@ static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u16 *active_speed, *active_width =IB_WIDTH_4X ; *active_speed =IB_SPEED_NDR ;break ; +case MLX5E_PROT_MASK (MLX5E_800GAUI_8_800GBASE_CR8_KR8 ): + *active_width =IB_WIDTH_8X ; + *active_speed =IB_SPEED_NDR ; +break ;default :return -EINVAL ; }
drivers/net/ethernet/mellanox/mlx5/core/port.c +1 -0
@@ -1102,6 +1102,7 @@ static const u32 mlx5e_ext_link_speed[MLX5E_EXT_LINK_MODES_NUMBER] = { [MLX5E_100GAUI_1_100GBASE_CR_KR] = 100000, [MLX5E_200GAUI_2_200GBASE_CR2_KR2] = 200000, [MLX5E_400GAUI_4_400GBASE_CR4_KR4] = 400000, + [MLX5E_800GAUI_8_800GBASE_CR8_KR8] = 800000, }; int mlx5_port_query_eth_proto(struct mlx5_core_dev *dev, u8 port, bool ext,
include/linux/mlx5/port.h +1 -0
@@ -117,6 +117,7 @@ enum mlx5e_ext_link_mode {MLX5E_200GAUI_2_200GBASE_CR2_KR2 =13 ,MLX5E_400GAUI_8 =15 ,MLX5E_400GAUI_4_400GBASE_CR4_KR4 =16 , +MLX5E_800GAUI_8_800GBASE_CR8_KR8 =19 , MLX5E_EXT_LINK_MODES_NUMBER, };
Signature · 1 line
2.41.0
PATCH 4/6 IB/mlx5: Rename 400G_8X speed to comply to naming convention
In reply to Leon RomanovskyFrom: Patrisious Haddad <[email protected]>
Rename 400G_8X speed to comply to naming convention.
--- drivers/infiniband/hw/mlx5/main.c | 2 +- drivers/net/ethernet/mellanox/mlx5/core/port.c | 2 +- include/linux/mlx5/port.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-)
drivers/infiniband/hw/mlx5/main.c +1 -1
@@ -474,7 +474,7 @@ static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u16 *active_speed, *active_width =IB_WIDTH_2X ; *active_speed =IB_SPEED_NDR ;break ; -case MLX5E_PROT_MASK (MLX5E_400GAUI_8 ): +case MLX5E_PROT_MASK (MLX5E_400GAUI_8_400GBASE_CR8 ): *active_width =IB_WIDTH_8X ; *active_speed =IB_SPEED_HDR ;break ;
drivers/net/ethernet/mellanox/mlx5/core/port.c +1 -1
@@ -1098,7 +1098,7 @@ static const u32 mlx5e_ext_link_speed[MLX5E_EXT_LINK_MODES_NUMBER] = { [MLX5E_CAUI_4_100GBASE_CR4_KR4] = 100000, [MLX5E_100GAUI_2_100GBASE_CR2_KR2] = 100000, [MLX5E_200GAUI_4_200GBASE_CR4_KR4] = 200000, - [MLX5E_400GAUI_8] = 400000, + [MLX5E_400GAUI_8_400GBASE_CR8] = 400000, [MLX5E_100GAUI_1_100GBASE_CR_KR] = 100000, [MLX5E_200GAUI_2_200GBASE_CR2_KR2] = 200000, [MLX5E_400GAUI_4_400GBASE_CR4_KR4] = 400000,
include/linux/mlx5/port.h +1 -1
@@ -115,7 +115,7 @@ enum mlx5e_ext_link_mode {MLX5E_100GAUI_1_100GBASE_CR_KR =11 ,MLX5E_200GAUI_4_200GBASE_CR4_KR4 =12 ,MLX5E_200GAUI_2_200GBASE_CR2_KR2 =13 , -MLX5E_400GAUI_8 =15 , +MLX5E_400GAUI_8_400GBASE_CR8 =15 ,MLX5E_400GAUI_4_400GBASE_CR4_KR4 =16 ,MLX5E_800GAUI_8_800GBASE_CR8_KR8 =19 , MLX5E_EXT_LINK_MODES_NUMBER,
Signature · 1 line
2.41.0
PATCH 5/6 IB/mlx5: Adjust mlx5 rate mapping to support 800Gb
In reply to Leon RomanovskyFrom: Patrisious Haddad <[email protected]>
Adjust mlx5 function which maps the speed rate from IB spec values to internal driver values to be able to handle speeds up to 800Gb.
--- drivers/infiniband/hw/mlx5/qp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/infiniband/hw/mlx5/qp.c +1 -1
@@ -3438,7 +3438,7 @@ static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate)if (rate ==IB_RATE_PORT_CURRENT )return 0 ; -if (rate <IB_RATE_2_5_GBPS || rate >IB_RATE_600_GBPS ) +if (rate <IB_RATE_2_5_GBPS || rate >IB_RATE_800_GBPS )return -EINVAL ; stat_rate_support =MLX5_CAP_GEN (dev->mdev, stat_rate_support);
Signature · 1 line
2.41.0
PATCH 6/6 RDMA/ipoib: Add support for XDR speed in ethtool
In reply to Leon RomanovskyFrom: Patrisious Haddad <[email protected]>
The IBTA specification 1.7 has new speed - XDR, supporting signaling rate of 200Gb.
Ethtool support of IPoIB driver translates IB speed to signaling rate. Added translation of XDR IB type to rate of 200Gb Ethernet speed.
--- drivers/infiniband/ulp/ipoib/ipoib_ethtool.c | 2 ++ 1 file changed, 2 insertions(+)
drivers/infiniband/ulp/ipoib/ipoib_ethtool.c +2 -0
@@ -174,6 +174,8 @@ static inline int ib_speed_enum_to_int(int speed)return SPEED_50000 ;case IB_SPEED_NDR :return SPEED_100000 ; +case IB_SPEED_XDR : +return SPEED_200000 ; }return SPEED_UNKNOWN ;
Signature · 1 line
2.41.0
Quoted 13 lines · Leon Romanovsky
From: Leon Romanovsky <[email protected]>
Hi,
This series extends RDMA subsystem and
mlx5_ibdriver to support 800Gb (XDR) speed which was added to IBTA v1.7 specification.Thanks
Or Har-Toov (2):
IB/core: Add support for XDR link speed
IB/mlx5: Expose XDR speed through MAD
Quoted 6 lines
Patrisious Haddad (4):
IB/mlx5: Add support for 800G_8X lane speed IB/mlx5: Rename 400G_8X speed to comply to naming convention IB/mlx5: Adjust mlx5 rate mapping to support 800Gb RDMA/ipoib: Add support for XDR speed in ethtool
Quoted 14 lines
drivers/infiniband/core/sysfs.c | 4 ++++ drivers/infiniband/core/uverbs_std_types_device.c | 3 ++- drivers/infiniband/core/verbs.c | 3 +++ drivers/infiniband/hw/mlx5/mad.c | 13 +++++++++++++ drivers/infiniband/hw/mlx5/main.c | 6 +++++- drivers/infiniband/hw/mlx5/qp.c | 2 +- drivers/infiniband/ulp/ipoib/ipoib_ethtool.c | 2 ++ drivers/net/ethernet/mellanox/mlx5/core/port.c | 3 ++- include/linux/mlx5/port.h | 3 ++- include/rdma/ib_mad.h | 2 ++ include/rdma/ib_verbs.h | 2 ++ include/uapi/rdma/ib_user_ioctl_verbs.h | 3 ++- 12 files changed, 40 insertions(+), 6 deletions(-)
Leon Romanovsky
AppliedQuoted 8 lines · Leon Romanovsky
From: Leon Romanovsky <[email protected]>
Hi,
This series extends RDMA subsystem and
mlx5_ibdriver to support 800Gb (XDR) speed which was added to IBTA v1.7 specification.[...]
Applied, thanks!
[1/6] IB/core: Add support for XDR link speed
https://git.kernel.org/rdma/rdma/c/703289ce43f740 [2/6] IB/mlx5: Expose XDR speed through MAD https://git.kernel.org/rdma/rdma/c/561b4a3ac65597 [3/6] IB/mlx5: Add support for 800G_8X lane speed https://git.kernel.org/rdma/rdma/c/948f0bf5ad6ac1 [4/6] IB/mlx5: Rename 400G_8X speed to comply to naming convention https://git.kernel.org/rdma/rdma/c/b28ad32442bec2 [5/6] IB/mlx5: Adjust mlx5 rate mapping to support 800Gb https://git.kernel.org/rdma/rdma/c/4f4db190893fb8 [6/6] RDMA/ipoib: Add support for XDR speed in ethtool https://git.kernel.org/rdma/rdma/c/8dc0fd2f5693ab
Best regards,
Signature · 1 line
Leon Romanovsky <[email protected]>