RDMA/mlx5: Support TX port affinity for VF drivers in LAG mode
Subsystems (2)
PATCH 1/1 RDMA/mlx5: Support TX port affinity for VF drivers in LAG mode
From: Mark Zhang <[email protected]>
The mlx5 VF driver doesn't set QP tx port affinity because it doesn't know if the lag is active or not, since the "lag_active" works only for PF interfaces. In this case for VF interfaces only one lag is used which brings performance issue.
Add a lag_tx_port_affinity CAP bit; When it is enabled and "num_lag_ports > 1", then driver always set QP tx affinity, regardless of lag state.
--- drivers/infiniband/hw/mlx5/main.c | 2 +- drivers/infiniband/hw/mlx5/mlx5_ib.h | 7 +++++++ drivers/infiniband/hw/mlx5/qp.c | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-)
drivers/infiniband/hw/mlx5/main.c +1 -1
@@ -1984,7 +1984,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx, context->lib_caps = req.lib_caps;print_lib_caps (dev, context->lib_caps); -if (dev->lag_active) { +if (mlx5_ib_lag_should_assign_affinity (dev)) {u8 port =mlx5_core_native_port_num (dev->mdev) -1 ; atomic_set(&context->tx_port_affinity,
drivers/infiniband/hw/mlx5/mlx5_ib.h +7 -0
@@ -1553,4 +1553,11 @@ static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev,int mlx5_ib_enable_driver (struct ib_device *dev);int mlx5_ib_test_wc (struct mlx5_ib_dev *dev); + +static inline bool mlx5_ib_lag_should_assign_affinity (struct mlx5_ib_dev *dev) +{ +return dev->lag_active || + (MLX5_CAP_GEN (dev->mdev, num_lag_ports) && +MLX5_CAP_GEN (dev->mdev, lag_tx_port_affinity)); +}#endif /* MLX5_IB_H */
drivers/infiniband/hw/mlx5/qp.c +2 -1
@@ -3653,7 +3653,8 @@ static unsigned int get_tx_affinity(struct ib_qp *qp,struct mlx5_ib_qp_base *qp_base;unsigned int tx_affinity; -if (!(dev->lag_active &&qp_supports_affinity (qp))) +if (!(mlx5_ib_lag_should_assign_affinity (dev) && +qp_supports_affinity (qp)))return 0 ;if (mqp->flags &MLX5_IB_QP_CREATE_SQPN_QP1 )
Signature · 1 line
2.26.2
Quoted 38 lines · Leon Romanovsky
From: Mark Zhang <[email protected]>
The mlx5 VF driver doesn't set QP tx port affinity because it doesn't know if the lag is active or not, since the "
lag_active" works only for PF interfaces. In this case for VF interfaces only one lag is used which brings performance issue.Add a
lag_tx_port_affinityCAP bit; When it is enabled and "num_lag_ports> 1", then driver always set QP tx affinity, regardless of lag state.Signed-off-by Mark Zhang <[email protected]>Reviewed-by Maor Gottlieb <[email protected]>Signed-off-by Leon Romanovsky <[email protected]>--- drivers/infiniband/hw/mlx5/main.c | 2 +- drivers/infiniband/hw/mlx5/mlx5_ib.h | 7 +++++++ drivers/infiniband/hw/mlx5/qp.c | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-)drivers/infiniband/hw/mlx5/main.c +1
-1@@ -1984,7 +1984,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx, context->lib_caps = req.lib_caps;print_lib_caps (dev, context->lib_caps); -if (dev->lag_active) { +if (mlx5_ib_lag_should_assign_affinity (dev)) {u8 port =mlx5_core_native_port_num (dev->mdev) -1 ; atomic_set(&context->tx_port_affinity,drivers/infiniband/hw/mlx5/mlx5_ib.h +0
-0
drivers/infiniband/hw/mlx5/mlx5_ib.h static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev, · line 1561
Quoted 9 lines
drivers/infiniband/hw/mlx5/mlx5_ib.h +6
-0@@ -1553,4 +1553,11 @@ static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev,int mlx5_ib_enable_driver (struct ib_device *dev);int mlx5_ib_test_wc (struct mlx5_ib_dev *dev); + +static inline bool mlx5_ib_lag_should_assign_affinity (struct mlx5_ib_dev *dev) +{ +return dev->lag_active || + (MLX5_CAP_GEN (dev->mdev, num_lag_ports) && +MLX5_CAP_GEN (dev->mdev, lag_tx_port_affinity));
first of all in the commit message you write:
"When it is enabled and > "num_lag_ports > 1""
which isn't the case here as even num_lag_ports == 1 will pass.
In addition, even on a system without lag this cap can be 2 (on a 2 port NIC where the FW supports lag).
I assume/hope/think that somewhere in the FW there is a check that says:
"If user has set port_affinity but lag isn't active use the native port affinity" either way I think it needs to be documented in the commit message.
Mark
Quoted 17 lines
drivers/infiniband/hw/mlx5/mlx5_ib.h +1
-0+}#endif /* MLX5_IB_H */ drivers/infiniband/hw/mlx5/qp.c +2
-1@@ -3653,7 +3653,8 @@ static unsigned int get_tx_affinity(struct ib_qp *qp,struct mlx5_ib_qp_base *qp_base;unsigned int tx_affinity; -if (!(dev->lag_active &&qp_supports_affinity (qp))) +if (!(mlx5_ib_lag_should_assign_affinity (dev) && +qp_supports_affinity (qp)))return 0 ;if (mqp->flags &MLX5_IB_QP_CREATE_SQPN_QP1 )
Quoted 38 lines · Leon Romanovsky
From: Mark Zhang <[email protected]>
The mlx5 VF driver doesn't set QP tx port affinity because it doesn't know if the lag is active or not, since the "
lag_active" works only for PF interfaces. In this case for VF interfaces only one lag is used which brings performance issue.Add a
lag_tx_port_affinityCAP bit; When it is enabled and "num_lag_ports> 1", then driver always set QP tx affinity, regardless of lag state.Signed-off-by Mark Zhang <[email protected]>Reviewed-by Maor Gottlieb <[email protected]>Signed-off-by Leon Romanovsky <[email protected]>--- drivers/infiniband/hw/mlx5/main.c | 2 +- drivers/infiniband/hw/mlx5/mlx5_ib.h | 7 +++++++ drivers/infiniband/hw/mlx5/qp.c | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-)drivers/infiniband/hw/mlx5/main.c +1
-1@@ -1984,7 +1984,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx, context->lib_caps = req.lib_caps;print_lib_caps (dev, context->lib_caps); -if (dev->lag_active) { +if (mlx5_ib_lag_should_assign_affinity (dev)) {u8 port =mlx5_core_native_port_num (dev->mdev) -1 ; atomic_set(&context->tx_port_affinity,drivers/infiniband/hw/mlx5/mlx5_ib.h +0
-0
drivers/infiniband/hw/mlx5/mlx5_ib.h static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev, · line 1562
Quoted 10 lines
drivers/infiniband/hw/mlx5/mlx5_ib.h +7
-0@@ -1553,4 +1553,11 @@ static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev,int mlx5_ib_enable_driver (struct ib_device *dev);int mlx5_ib_test_wc (struct mlx5_ib_dev *dev); + +static inline bool mlx5_ib_lag_should_assign_affinity (struct mlx5_ib_dev *dev) +{ +return dev->lag_active || + (MLX5_CAP_GEN (dev->mdev, num_lag_ports) && +MLX5_CAP_GEN (dev->mdev, lag_tx_port_affinity)); +}
We did some investigation offline and it seems that this line should be changed to be
+static inline bool mlx5_ib_lag_should_assign_affinity(struct mlx5_ib_dev *dev) +{
+ return dev->lag_active ||
+ (MLX5_CAP_GEN(dev->mdev, num_lag_ports) > 1 &&
+ MLX5_CAP_GEN(dev->mdev, lag_tx_port_affinity));
+}
Thanks
Quoted 18 lines
drivers/infiniband/hw/mlx5/mlx5_ib.h +0
-0#endif /* MLX5_IB_H */ drivers/infiniband/hw/mlx5/qp.c +2
-2@@ -3653,7 +3653,8 @@ static unsigned int get_tx_affinity(struct ib_qp *qp,struct mlx5_ib_qp_base *qp_base;unsigned int tx_affinity; -if (!(dev->lag_active &&qp_supports_affinity (qp))) +if (!(mlx5_ib_lag_should_assign_affinity (dev) && +qp_supports_affinity (qp)))return 0 ;if (mqp->flags &MLX5_IB_QP_CREATE_SQPN_QP1 ) --2.26.2