Skip to contentSkip to conversation

RDMA/mlx5: Support TX port affinity for VF drivers in LAG mode

PATCH In review Leon Romanovsky 2 participants 3 messages May 26, 2020

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 RDMA/mlx5: Support TX port affinity for VF drivers in LAG mode

From: Mark Zhang <[email protected]>

The mlx5 VF driver doesn't set QP tx port affinity because it doesn't know if the lag is active or not, since the "lag_active" works only for PF interfaces. In this case for VF interfaces only one lag is used which brings performance issue.

Add a lag_tx_port_affinity CAP bit; When it is enabled and "num_lag_ports > 1", then driver always set QP tx affinity, regardless of lag state.

---
 drivers/infiniband/hw/mlx5/main.c    | 2 +-
 drivers/infiniband/hw/mlx5/mlx5_ib.h | 7 +++++++
 drivers/infiniband/hw/mlx5/qp.c      | 3 ++-
 3 files changed, 10 insertions(+), 2 deletions(-)
drivers/infiniband/hw/mlx5/main.c +1 -1
index 570c519ca530..4719da201382 100644
@@ -1984,7 +1984,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
 	context->lib_caps = req.lib_caps;
 	print_lib_caps(dev, context->lib_caps);
 
-	if (dev->lag_active) {
+	if (mlx5_ib_lag_should_assign_affinity(dev)) {
 		u8 port = mlx5_core_native_port_num(dev->mdev) - 1;
 
 		atomic_set(&context->tx_port_affinity,
drivers/infiniband/hw/mlx5/mlx5_ib.h +7 -0
index b486139b08ce..0f5a713ac2a9 100644
@@ -1553,4 +1553,11 @@ static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev,
 
 int mlx5_ib_enable_driver(struct ib_device *dev);
 int mlx5_ib_test_wc(struct mlx5_ib_dev *dev);
+
+static inline bool mlx5_ib_lag_should_assign_affinity(struct mlx5_ib_dev *dev)
+{
+	return dev->lag_active ||
+		(MLX5_CAP_GEN(dev->mdev, num_lag_ports) &&
+		 MLX5_CAP_GEN(dev->mdev, lag_tx_port_affinity));
+}
 #endif /* MLX5_IB_H */
drivers/infiniband/hw/mlx5/qp.c +2 -1
index 1988a0375696..9364a7a76ac2 100644
@@ -3653,7 +3653,8 @@ static unsigned int get_tx_affinity(struct ib_qp *qp,
 	struct mlx5_ib_qp_base *qp_base;
 	unsigned int tx_affinity;
 
-	if (!(dev->lag_active && qp_supports_affinity(qp)))
+	if (!(mlx5_ib_lag_should_assign_affinity(dev) &&
+	      qp_supports_affinity(qp)))
 		return 0;
 
 	if (mqp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
Signature · 1 line
2.26.2
#1 lore raw
Quoted 38 lines · Leon Romanovsky

From: Mark Zhang <[email protected]>

The mlx5 VF driver doesn't set QP tx port affinity because it doesn't know if the lag is active or not, since the "lag_active" works only for PF interfaces. In this case for VF interfaces only one lag is used which brings performance issue.

Add a lag_tx_port_affinity CAP bit; When it is enabled and "num_lag_ports > 1", then driver always set QP tx affinity, regardless of lag state.

---
 drivers/infiniband/hw/mlx5/main.c    | 2 +-
 drivers/infiniband/hw/mlx5/mlx5_ib.h | 7 +++++++
 drivers/infiniband/hw/mlx5/qp.c      | 3 ++-
 3 files changed, 10 insertions(+), 2 deletions(-)
drivers/infiniband/hw/mlx5/main.c +1 -1
index 570c519ca530..4719da201382 100644
@@ -1984,7 +1984,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
 	context->lib_caps = req.lib_caps;
 	print_lib_caps(dev, context->lib_caps);
 
-	if (dev->lag_active) {
+	if (mlx5_ib_lag_should_assign_affinity(dev)) {
 		u8 port = mlx5_core_native_port_num(dev->mdev) - 1;
 
 		atomic_set(&context->tx_port_affinity,
drivers/infiniband/hw/mlx5/mlx5_ib.h +0 -0
index b486139b08ce..0f5a713ac2a9 100644

drivers/infiniband/hw/mlx5/mlx5_ib.h static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev, · line 1561

Quoted 9 lines
drivers/infiniband/hw/mlx5/mlx5_ib.h +6 -0
@@ -1553,4 +1553,11 @@ static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev,
 
 int mlx5_ib_enable_driver(struct ib_device *dev);
 int mlx5_ib_test_wc(struct mlx5_ib_dev *dev);
+
+static inline bool mlx5_ib_lag_should_assign_affinity(struct mlx5_ib_dev *dev)
+{
+	return dev->lag_active ||
+		(MLX5_CAP_GEN(dev->mdev, num_lag_ports) &&
+		 MLX5_CAP_GEN(dev->mdev, lag_tx_port_affinity));

first of all in the commit message you write:
"When it is enabled and > "num_lag_ports > 1""
which isn't the case here as even num_lag_ports == 1 will pass.

In addition, even on a system without lag this cap can be 2 (on a 2 port NIC where the FW supports lag).

I assume/hope/think that somewhere in the FW there is a check that says:
"If user has set port_affinity but lag isn't active use the native port affinity" either way I think it needs to be documented in the commit message.

Mark

Quoted 17 lines
drivers/infiniband/hw/mlx5/mlx5_ib.h +1 -0
+}
 #endif /* MLX5_IB_H */
drivers/infiniband/hw/mlx5/qp.c +2 -1
index 1988a0375696..9364a7a76ac2 100644
@@ -3653,7 +3653,8 @@ static unsigned int get_tx_affinity(struct ib_qp *qp,
 	struct mlx5_ib_qp_base *qp_base;
 	unsigned int tx_affinity;
 
-	if (!(dev->lag_active && qp_supports_affinity(qp)))
+	if (!(mlx5_ib_lag_should_assign_affinity(dev) &&
+	      qp_supports_affinity(qp)))
 		return 0;
 
 	if (mqp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
#2 lore raw
In reply to Leon Romanovsky
Quoted 38 lines · Leon Romanovsky

From: Mark Zhang <[email protected]>

The mlx5 VF driver doesn't set QP tx port affinity because it doesn't know if the lag is active or not, since the "lag_active" works only for PF interfaces. In this case for VF interfaces only one lag is used which brings performance issue.

Add a lag_tx_port_affinity CAP bit; When it is enabled and "num_lag_ports > 1", then driver always set QP tx affinity, regardless of lag state.

---
 drivers/infiniband/hw/mlx5/main.c    | 2 +-
 drivers/infiniband/hw/mlx5/mlx5_ib.h | 7 +++++++
 drivers/infiniband/hw/mlx5/qp.c      | 3 ++-
 3 files changed, 10 insertions(+), 2 deletions(-)
drivers/infiniband/hw/mlx5/main.c +1 -1
index 570c519ca530..4719da201382 100644
@@ -1984,7 +1984,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
 	context->lib_caps = req.lib_caps;
 	print_lib_caps(dev, context->lib_caps);
 
-	if (dev->lag_active) {
+	if (mlx5_ib_lag_should_assign_affinity(dev)) {
 		u8 port = mlx5_core_native_port_num(dev->mdev) - 1;
 
 		atomic_set(&context->tx_port_affinity,
drivers/infiniband/hw/mlx5/mlx5_ib.h +0 -0
index b486139b08ce..0f5a713ac2a9 100644

drivers/infiniband/hw/mlx5/mlx5_ib.h static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev, · line 1562

Quoted 10 lines
drivers/infiniband/hw/mlx5/mlx5_ib.h +7 -0
@@ -1553,4 +1553,11 @@ static inline bool mlx5_ib_can_use_umr(struct mlx5_ib_dev *dev,
 
 int mlx5_ib_enable_driver(struct ib_device *dev);
 int mlx5_ib_test_wc(struct mlx5_ib_dev *dev);
+
+static inline bool mlx5_ib_lag_should_assign_affinity(struct mlx5_ib_dev *dev)
+{
+	return dev->lag_active ||
+		(MLX5_CAP_GEN(dev->mdev, num_lag_ports) &&
+		 MLX5_CAP_GEN(dev->mdev, lag_tx_port_affinity));
+}

We did some investigation offline and it seems that this line should be changed to be
+static inline bool mlx5_ib_lag_should_assign_affinity(struct mlx5_ib_dev *dev) +{
+ return dev->lag_active ||
+ (MLX5_CAP_GEN(dev->mdev, num_lag_ports) > 1 &&
+ MLX5_CAP_GEN(dev->mdev, lag_tx_port_affinity));
+}

Thanks

Quoted 18 lines
drivers/infiniband/hw/mlx5/mlx5_ib.h +0 -0
 #endif /* MLX5_IB_H */
drivers/infiniband/hw/mlx5/qp.c +2 -2
index 1988a0375696..9364a7a76ac2 100644
@@ -3653,7 +3653,8 @@ static unsigned int get_tx_affinity(struct ib_qp *qp,
 	struct mlx5_ib_qp_base *qp_base;
 	unsigned int tx_affinity;
 
-	if (!(dev->lag_active && qp_supports_affinity(qp)))
+	if (!(mlx5_ib_lag_should_assign_affinity(dev) &&
+	      qp_supports_affinity(qp)))
 		return 0;
 
 	if (mqp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
--

2.26.2