net/mlx5: Define interface bits for fencing UMR wqe

PATCH New Max Gurtovoy 5 participants 17 messages May 28, 2017 – Jun 1, 2017

Subsystems (4)

InfiniBand NVMe Mellanox mlx5 core Mellanox MLX5 IB driver

lore

Share & tools

lore

Max Gurtovoy

author

2017-05-28 07:53 UTC #0 lore raw

PATCH v2 1/2 net/mlx5: Define interface bits for fencing UMR wqe

HW can implement UMR wqe re-transmission in various ways. Thus, add HCA cap to distinguish the needed fence for UMR to make sure that the wqe wouldn't fail on mkey checks.

Signed-off-by Max Gurtovoy <maxg at mellanox.com>

Acked-by Leon Romanovsky <leon at kernel.org>

---

No changes from v1 in this patch (PATCH 1/2).

---
 include/linux/mlx5/mlx5_ifc.h |   10 +++++++++-
 1 files changed, 9 insertions(+), 1 deletions(-)

include/linux/mlx5/mlx5_ifc.h +9 -1

index 32de072..edafedb 100644
@@ -766,6 +766,12 @@ enum {
 	MLX5_CAP_PORT_TYPE_ETH = 0x1,
 };
 
+enum {
+	MLX5_CAP_UMR_FENCE_STRONG	= 0x0,
+	MLX5_CAP_UMR_FENCE_SMALL	= 0x1,
+	MLX5_CAP_UMR_FENCE_NONE		= 0x2,
+};
+
 struct mlx5_ifc_cmd_hca_cap_bits {
 	u8         reserved_at_0[0x80];

@@ -875,7 +881,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 	u8         reserved_at_202[0x1];
 	u8         ipoib_enhanced_offloads[0x1];
 	u8         ipoib_basic_offloads[0x1];
-	u8         reserved_at_205[0xa];
+	u8         reserved_at_205[0x5];
+	u8         umr_fence[0x2];
+	u8         reserved_at_20c[0x3];
 	u8         drain_sigerr[0x1];
 	u8         cmdif_checksum[0x2];
 	u8         sigerr_cqe[0x1];

Signature · 1 line

1.7.1

Max Gurtovoy

author

2017-05-28 07:53 UTC #1 lore raw

PATCH v2 2/2 IB/mlx5: set UMR wqe fence according to HCA cap

Cache the needed umr_fence and set the wqe ctrl segmennt accordingly.

Signed-off-by Max Gurtovoy <maxg at mellanox.com>

Acked-by Leon Romanovsky <leon at kernel.org>

---

Changes from v1:
- updated MLX5_FENCE_MODE_STRONG_ORDERING to be default value in mlx5_get_umr_fence.

---
 drivers/infiniband/hw/mlx5/main.c    |   14 ++++++++++++++
 drivers/infiniband/hw/mlx5/mlx5_ib.h |    1 +
 drivers/infiniband/hw/mlx5/qp.c      |   15 +++++++--------
 3 files changed, 22 insertions(+), 8 deletions(-)

drivers/infiniband/hw/mlx5/main.c +14 -0

index d45772d..83d1f9b 100644
@@ -2979,6 +2979,18 @@ static int create_umr_res(struct mlx5_ib_dev *dev)
 	return ret;
 }
 
+static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
+{
+	switch (umr_fence_cap) {
+	case MLX5_CAP_UMR_FENCE_NONE:
+		return MLX5_FENCE_MODE_NONE;
+	case MLX5_CAP_UMR_FENCE_SMALL:
+		return MLX5_FENCE_MODE_INITIATOR_SMALL;
+	default:
+		return MLX5_FENCE_MODE_STRONG_ORDERING;
+	}
+}
+
 static int create_dev_resources(struct mlx5_ib_resources *devr)
 {
 	struct ib_srq_init_attr attr;

@@ -3693,6 +3705,8 @@ static void mlx5_ib_free_rdma_netdev(struct net_device *netdev)
 
 	mlx5_ib_internal_fill_odp_caps(dev);
 
+	dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
+
 	if (MLX5_CAP_GEN(mdev, imaicl)) {
 		dev->ib_dev.alloc_mw		= mlx5_ib_alloc_mw;
 		dev->ib_dev.dealloc_mw		= mlx5_ib_dealloc_mw;

drivers/infiniband/hw/mlx5/mlx5_ib.h +1 -0

index 38c877b..0e08a58 100644
@@ -654,6 +654,7 @@ struct mlx5_ib_dev {
 	struct mlx5_ib_port	*port;
 	struct mlx5_sq_bfreg     bfreg;
 	struct mlx5_sq_bfreg     fp_bfreg;
+	u8				umr_fence;
 };
 
 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)

drivers/infiniband/hw/mlx5/qp.c +7 -8

index 93959e1..876a429 100644
@@ -3738,11 +3738,10 @@ static void dump_wqe(struct mlx5_ib_qp *qp, int idx, int size_16)
 	}
 }
 
-static u8 get_fence(u8 fence, struct ib_send_wr *wr)
+static u8 get_fence(u8 fence, struct ib_send_wr *wr, struct mlx5_ib_dev *dev)
 {
-	if (unlikely(wr->opcode == IB_WR_LOCAL_INV &&
-		     wr->send_flags & IB_SEND_FENCE))
-		return MLX5_FENCE_MODE_STRONG_ORDERING;
+	if (wr->opcode == IB_WR_LOCAL_INV || wr->opcode == IB_WR_REG_MR)
+		return dev->umr_fence;
 
 	if (unlikely(fence)) {
 		if (wr->send_flags & IB_SEND_FENCE)

@@ -3928,7 +3927,7 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 				}
 
 				finish_wqe(qp, ctrl, size, idx, wr->wr_id,
-					   nreq, get_fence(fence, wr),
+					   nreq, get_fence(fence, wr, dev),
 					   next_fence, MLX5_OPCODE_UMR);
 				/*
 				 * SET_PSV WQEs are not signaled and solicited

@@ -3955,7 +3954,7 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 				}
 
 				finish_wqe(qp, ctrl, size, idx, wr->wr_id,
-					   nreq, get_fence(fence, wr),
+					   nreq, get_fence(fence, wr, dev),
 					   next_fence, MLX5_OPCODE_SET_PSV);
 				err = begin_wqe(qp, &seg, &ctrl, wr,
 						&idx, &size, nreq);

@@ -3977,7 +3976,7 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 				}
 
 				finish_wqe(qp, ctrl, size, idx, wr->wr_id,
-					   nreq, get_fence(fence, wr),
+					   nreq, get_fence(fence, wr, dev),
 					   next_fence, MLX5_OPCODE_SET_PSV);
 				num_sge = 0;
 				goto skip_psv;

@@ -4090,7 +4089,7 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 		}
 
 		finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,
-			   get_fence(fence, wr), next_fence,
+			   get_fence(fence, wr, dev), next_fence,
 			   mlx5_ib_opcode[wr->opcode]);
 skip_psv:
 		if (0)

Signature · 1 line

1.7.1

Christoph Hellwig

maintainer

2017-05-28 09:07 UTC #2 lore raw

PATCH v2 2/2 IB/mlx5: set UMR wqe fence according to HCA cap

Quoted 2 lines · Max Gurtovoy

Cache the needed umr_fence and set the wqe ctrl segmennt accordingly.

Looks good,

Reviewed-by Christoph Hellwig <hch at lst.de>

But that whole fence logic looks awkward to me. Does the following patch to reorder it make sense to you?

drivers/infiniband/hw/mlx5/mlx5_ib.h +1 -1

index 0e08a58de673..bdcf25410c99 100644
@@ -349,7 +349,7 @@ struct mlx5_ib_qp {
 	struct mlx5_ib_wq	rq;
 
 	u8			sq_signal_bits;
-	u8			fm_cache;
+	u8			next_fence;
 	struct mlx5_ib_wq	sq;
 
 	/* serialize qp state modifications

drivers/infiniband/hw/mlx5/qp.c +23 ~~-35~~

130 lines not on this page · raw message

Max Gurtovoy

author

2017-05-28 09:53 UTC #3 lore raw

PATCH v2 2/2 IB/mlx5: set UMR wqe fence according to HCA cap

Quoted 7 lines · Christoph Hellwig

Quoted 2 lines · Max Gurtovoy
Cache the needed umr_fence and set the wqe ctrl segmennt accordingly.
Looks good,
Reviewed-by Christoph Hellwig <hch at lst.de>

Thanks.

Quoted 4 lines

But that whole fence logic looks awkward to me. Does the following patch to reorder it make sense to you?

Yes it make sense to me.
Sagi/Leon, any comments ?

Leon Romanovsky

2017-05-29 10:05 UTC #4 lore raw

PATCH v2 2/2 IB/mlx5: set UMR wqe fence according to HCA cap

Quoted 14 lines · Max Gurtovoy

Quoted 7 lines · Christoph Hellwig
Quoted 2 lines · Max Gurtovoy
Cache the needed umr_fence and set the wqe ctrl segmennt accordingly.
Looks good,
Reviewed-by Christoph Hellwig <hch at lst.de>
Thanks.
Quoted 1 line

Quoted 7 lines

Quoted 3 lines
But that whole fence logic looks awkward to me. Does the following patch to reorder it make sense to you?
Yes it make sense to me.
Sagi/Leon, any comments ?

Max,

Do you see any performance impact for IB_WR_RDMA_READ, IB_WR_RDMA_WRITE and IB_WR_RDMA_WRITE_WITH_IMM flows? They don't need fences and such change can cause to performance losses.

Thanks

Quoted 6 lines

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majordomo at vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 833 bytes
Desc: not available
URL: <http://lists.infradead.org/pipermail/linux-nvme/attachments/20170529/b2c6f7d0/attachment.sig>

Max Gurtovoy

author

2017-05-29 12:21 UTC #5 lore raw

PATCH v2 2/2 IB/mlx5: set UMR wqe fence according to HCA cap

Quoted 23 lines · Leon Romanovsky

Quoted 14 lines · Max Gurtovoy
Quoted 7 lines · Christoph Hellwig
Quoted 2 lines · Max Gurtovoy
Cache the needed umr_fence and set the wqe ctrl segmennt accordingly.
Looks good,
Reviewed-by Christoph Hellwig <hch at lst.de>
Thanks.
Quoted 1 line
Quoted 7 lines
Quoted 3 lines
But that whole fence logic looks awkward to me. Does the following patch to reorder it make sense to you?
Yes it make sense to me.
Sagi/Leon, any comments ?

Quoted 8 lines

Max,
Do you see any performance impact for IB_WR_RDMA_READ, IB_WR_RDMA_WRITE and IB_WR_RDMA_WRITE_WITH_IMM flows? They don't need fences and such change can cause to performance losses.
Thanks

We don't fence those WR's.
Christoph just re-write it to be more intuitive code. I don't see logic difference, am I wrong here ?

Quoted 7 lines

Quoted 7 lines
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majordomo at vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html

Leon Romanovsky

2017-05-29 16:06 UTC #6 lore raw

PATCH v2 2/2 IB/mlx5: set UMR wqe fence according to HCA cap

Quoted 32 lines · Max Gurtovoy

Quoted 29 lines · Leon Romanovsky
Quoted 14 lines · Max Gurtovoy
Quoted 7 lines · Christoph Hellwig
Quoted 2 lines · Max Gurtovoy
Cache the needed umr_fence and set the wqe ctrl segmennt accordingly.
Looks good,
Reviewed-by Christoph Hellwig <hch at lst.de>
Thanks.
Quoted 1 line
Quoted 7 lines
Quoted 3 lines
But that whole fence logic looks awkward to me. Does the following patch to reorder it make sense to you?
Yes it make sense to me.
Sagi/Leon, any comments ?
Max,
Do you see any performance impact for IB_WR_RDMA_READ, IB_WR_RDMA_WRITE and IB_WR_RDMA_WRITE_WITH_IMM flows? They don't need fences and such change can cause to performance losses.

Quoted 6 lines

Quoted 2 lines
Thanks
We don't fence those WR's.
Christoph just re-write it to be more intuitive code. I don't see logic difference, am I wrong here ?

A little bit, before Christoph's suggestion, we calculated fence for the paths which need such fence, after we will calculate for all paths.

Thanks

Quoted 8 lines

Quoted 8 lines
Quoted 8 lines
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majordomo at vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html

Sagi Grimberg

maintainer

2017-05-30 10:51 UTC #7 lore raw

PATCH v2 2/2 IB/mlx5: set UMR wqe fence according to HCA cap

Leon,

Quoted 6 lines

Quoted 3 lines
We don't fence those WR's.
Christoph just re-write it to be more intuitive code. I don't see logic difference, am I wrong here ?
A little bit, before Christoph's suggestion, we calculated fence for the paths which need such fence, after we will calculate for all paths.

Every WQE posted to a send queue must include a fence bit. All work request posted on the send queue calculate the required fence, this used to happen in finish_wqe call-sites with get_fence(), Christoph just inlin'ed it.

Leon Romanovsky

2017-05-30 17:15 UTC #8 lore raw

PATCH v2 2/2 IB/mlx5: set UMR wqe fence according to HCA cap

Quoted 6 lines · Sagi Grimberg

Leon,
Quoted 4 lines
Quoted 3 lines
We don't fence those WR's.
Christoph just re-write it to be more intuitive code. I don't see logic difference, am I wrong here ?

Quoted 7 lines

Quoted 2 lines
A little bit, before Christoph's suggestion, we calculated fence for the paths which need such fence, after we will calculate for all paths.
Every WQE posted to a send queue must include a fence bit. All work request posted on the send queue calculate the required fence, this used to happen in finish_wqe call-sites with get_fence(), Christoph just inlin'ed it.

Sagi,

Thanks, I found my mistake, I saw that IB_SEND_INLINE WR doesn't call to get_fence and we have an if() which can skip finish_wqe, so I thought that finish_wqe isn't called always, however it was for error path.

Thanks again.
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 833 bytes
Desc: not available
URL: <http://lists.infradead.org/pipermail/linux-nvme/attachments/20170530/8489129f/attachment.sig>

Sagi Grimberg

maintainer

2017-05-30 10:48 UTC #9 lore raw

PATCH v2 2/2 IB/mlx5: set UMR wqe fence according to HCA cap

In reply to Max Gurtovoy

drivers/infiniband/hw/mlx5/main.c static int create_umr_res(struct mlx5_ib_dev *dev) · line 2992

Quoted 11 lines

drivers/infiniband/hw/mlx5/main.c +11 -0

+static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
+{
+	switch (umr_fence_cap) {
+	case MLX5_CAP_UMR_FENCE_NONE:
+		return MLX5_FENCE_MODE_NONE;
+	case MLX5_CAP_UMR_FENCE_SMALL:
+		return MLX5_FENCE_MODE_INITIATOR_SMALL;
+	default:
+		return MLX5_FENCE_MODE_STRONG_ORDERING;
+	}
+}

Where are the MLX5_CAP_UMR defines declared? Am I missing something?

What is the value of MLX5_CAP_UMR_FENCE_NONE? if its zero then this change is not backwards compatible with older FW.

Max Gurtovoy

author

2017-05-30 11:15 UTC #10 lore raw

PATCH v2 2/2 IB/mlx5: set UMR wqe fence according to HCA cap

Quoted 14 lines · Sagi Grimberg

drivers/infiniband/hw/mlx5/main.c static int create_umr_res(struct mlx5_ib_dev *dev) · line 2992
Quoted 11 lines
drivers/infiniband/hw/mlx5/main.c +11 -0
+static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
+{
+    switch (umr_fence_cap) {
+    case MLX5_CAP_UMR_FENCE_NONE:
+        return MLX5_FENCE_MODE_NONE;
+    case MLX5_CAP_UMR_FENCE_SMALL:
+        return MLX5_FENCE_MODE_INITIATOR_SMALL;
+    default:
+        return MLX5_FENCE_MODE_STRONG_ORDERING;
+    }
+}
Where are the MLX5_CAP_UMR defines declared? Am I missing something?

Quoted 2 lines

What is the value of MLX5_CAP_UMR_FENCE_NONE? if its zero then this change is not backwards compatible with older FW.

they are declared in patch 1/2:

+enum {
+ MLX5_CAP_UMR_FENCE_STRONG = 0x0,
+ MLX5_CAP_UMR_FENCE_SMALL = 0x1,
+ MLX5_CAP_UMR_FENCE_NONE = 0x2,
+};
+

Thanks,
Max.

Sagi Grimberg

maintainer

2017-05-30 11:22 UTC #11 lore raw

PATCH v2 2/2 IB/mlx5: set UMR wqe fence according to HCA cap

Review

Quoted 8 lines

they are declared in patch 1/2:

(quoted diff) +6 -0

+enum {
+    MLX5_CAP_UMR_FENCE_STRONG    = 0x0,
+    MLX5_CAP_UMR_FENCE_SMALL    = 0x1,
+    MLX5_CAP_UMR_FENCE_NONE        = 0x2,
+};
+

Missed that, thanks.

What's FENCE_NONE anyway? no umr fence needed at all?

Max Gurtovoy

author

2017-05-30 11:24 UTC #12 lore raw

PATCH v2 2/2 IB/mlx5: set UMR wqe fence according to HCA cap

Quoted 11 lines · Sagi Grimberg

Review
Quoted 8 lines
they are declared in patch 1/2:
(quoted diff) +6 -0
+enum {
+    MLX5_CAP_UMR_FENCE_STRONG    = 0x0,
+    MLX5_CAP_UMR_FENCE_SMALL    = 0x1,
+    MLX5_CAP_UMR_FENCE_NONE        = 0x2,
+};
+
Missed that, thanks.

Quoted 1 line

What's FENCE_NONE anyway? no umr fence needed at all?

Yes.

Sagi Grimberg

maintainer

2017-05-30 14:28 UTC #13 lore raw

PATCH v2 2/2 IB/mlx5: set UMR wqe fence according to HCA cap

Quoted 2 lines

Yes.

So for the series:

Reviewed-by Sagi Grimberg <sagi at grimberg.me>

Doug, can you mark this for stable?

Max, I'm not sure how far back this needs a fix,
probably since day 1.

Max Gurtovoy

author

2017-05-30 20:41 UTC #14 lore raw

PATCH v2 2/2 IB/mlx5: set UMR wqe fence according to HCA cap

Quoted 11 lines · Sagi Grimberg

Quoted 2 lines
Yes.
So for the series:
Reviewed-by Sagi Grimberg <sagi at grimberg.me>
Doug, can you mark this for stable?
Max, I'm not sure how far back this needs a fix,
probably since day 1.

Yes, I guess we should.

Christoph Hellwig

maintainer

2017-05-28 09:14 UTC #15 lore raw

PATCH v2 1/2 net/mlx5: Define interface bits for fencing UMR wqe

In reply to Max Gurtovoy

Quoted 9 lines · Max Gurtovoy

HW can implement UMR wqe re-transmission in various ways. Thus, add HCA cap to distinguish the needed fence for UMR to make sure that the wqe wouldn't fail on mkey checks.
Signed-off-by Max Gurtovoy <maxg at mellanox.com>
Acked-by Leon Romanovsky <leon at kernel.org>
---
No changes from v1 in this patch (PATCH 1/2).

Still:

Reviewed-by Christoph Hellwig <hch at lst.de>

Doug Ledford

2017-06-01 22:51 UTC #16 lore raw

PATCH v2 1/2 net/mlx5: Define interface bits for fencing UMR wqe

In reply to Max Gurtovoy

Quoted 6 lines · Max Gurtovoy

HW can implement UMR wqe re-transmission in various ways. Thus, add HCA cap to distinguish the needed fence for UMR to make sure that the wqe wouldn't fail on mkey checks.
Signed-off-by Max Gurtovoy <maxg at mellanox.com>
Acked-by Leon Romanovsky <leon at kernel.org>

Series, with Christoph's fixup, applied.

Signature · 4 lines

Doug Ledford <dledford at redhat.com>
? ? GPG KeyID: B826A3330E572FDD
? ?
Key fingerprint = AE6B 1BDA 122B 23B4 265B ?1274 B826 A333 0E57 2FDD