Skip to contentSkip to conversation

RDMA/rxe: For invalidate compare according to set keys in mr

PATCH Applied Md Haris Iqbal 4 participants 7 messages Jul 7, 2022 – Jul 22, 2022

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 RDMA/rxe: For invalidate compare according to set keys in mr

The 'rkey' input can be an lkey or rkey, and in rxe the lkey or rkey have the same value, including the variant bits.

So, if mr->rkey is set, compare the invalidate key with it, otherwise compare with the mr->lkey.

Since we already did a lookup on the non-varient bits to get this far, the check's only purpose is to confirm that the wqe has the correct variant bits.

Fixes 001345339f4c ("RDMA/rxe: Separate HW and SW l/rkeys")
---
 drivers/infiniband/sw/rxe/rxe_loc.h |  2 +-
 drivers/infiniband/sw/rxe/rxe_mr.c  | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -1
index 0e022ae1b8a5..37484a559d20 100644
@@ -77,7 +77,7 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 			 enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
-int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey);
+int rxe_invalidate_mr(struct rxe_qp *qp, u32 key);
 int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
 int rxe_mr_set_page(struct ib_mr *ibmr, u64 addr);
 int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
drivers/infiniband/sw/rxe/rxe_mr.c +6 -6
index fc3942e04a1f..3add52129006 100644
@@ -576,22 +576,22 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
 	return mr;
 }
 
-int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey)
+int rxe_invalidate_mr(struct rxe_qp *qp, u32 key)
 {
 	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
 	struct rxe_mr *mr;
 	int ret;
 
-	mr = rxe_pool_get_index(&rxe->mr_pool, rkey >> 8);
+	mr = rxe_pool_get_index(&rxe->mr_pool, key >> 8);
 	if (!mr) {
-		pr_err("%s: No MR for rkey %#x\n", __func__, rkey);
+		pr_err("%s: No MR for key %#x\n", __func__, key);
 		ret = -EINVAL;
 		goto err;
 	}
 
-	if (rkey != mr->rkey) {
-		pr_err("%s: rkey (%#x) doesn't match mr->rkey (%#x)\n",
-			__func__, rkey, mr->rkey);
+	if (mr->rkey ? (key != mr->rkey) : (key != mr->lkey)) {
+		pr_err("%s: wr key (%#x) doesn't match mr key (%#x)\n",
+			__func__, key, (mr->rkey ? mr->rkey : mr->lkey));
 		ret = -EINVAL;
 		goto err_drop_ref;
 	}
Signature · 1 line
2.25.1
#1 lore raw
Quoted 66 lines · Md Haris Iqbal

The 'rkey' input can be an lkey or rkey, and in rxe the lkey or rkey have the same value, including the variant bits.

So, if mr->rkey is set, compare the invalidate key with it, otherwise compare with the mr->lkey.

Since we already did a lookup on the non-varient bits to get this far, the check's only purpose is to confirm that the wqe has the correct variant bits.

Fixes 001345339f4c ("RDMA/rxe: Separate HW and SW l/rkeys")
---
 drivers/infiniband/sw/rxe/rxe_loc.h |  2 +-
 drivers/infiniband/sw/rxe/rxe_mr.c  | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -1
index 0e022ae1b8a5..37484a559d20 100644
@@ -77,7 +77,7 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
                         enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
-int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey);
+int rxe_invalidate_mr(struct rxe_qp *qp, u32 key);
 int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
 int rxe_mr_set_page(struct ib_mr *ibmr, u64 addr);
 int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
drivers/infiniband/sw/rxe/rxe_mr.c +6 -7
index fc3942e04a1f..3add52129006 100644
@@ -576,22 +576,22 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
        return mr;
 }
 
-int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey)
+int rxe_invalidate_mr(struct rxe_qp *qp, u32 key)
 {
        struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
        struct rxe_mr *mr;
        int ret;
 
-       mr = rxe_pool_get_index(&rxe->mr_pool, rkey >> 8);
+       mr = rxe_pool_get_index(&rxe->mr_pool, key >> 8);
        if (!mr) {
-               pr_err("%s: No MR for rkey %#x\n", __func__, rkey);
+               pr_err("%s: No MR for key %#x\n", __func__, key);
                ret = -EINVAL;
                goto err;
        }
 
-       if (rkey != mr->rkey) {
-               pr_err("%s: rkey (%#x) doesn't match mr->rkey (%#x)\n",
-                       __func__, rkey, mr->rkey);
+       if (mr->rkey ? (key != mr->rkey) : (key != mr->lkey)) {
+               pr_err("%s: wr key (%#x) doesn't match mr key (%#x)\n",
+                       __func__, key, (mr->rkey ? mr->rkey : mr->lkey));
                ret = -EINVAL;
                goto err_drop_ref;
        }
--
Quoted 1 line

2.25.1

Ping.

Quoted 1 line
#2 lore raw
Quoted 58 lines · haris iqbal
Quoted 56 lines · Md Haris Iqbal

The 'rkey' input can be an lkey or rkey, and in rxe the lkey or rkey have the same value, including the variant bits.

So, if mr->rkey is set, compare the invalidate key with it, otherwise compare with the mr->lkey.

Since we already did a lookup on the non-varient bits to get this far, the check's only purpose is to confirm that the wqe has the correct variant bits.

Fixes 001345339f4c ("RDMA/rxe: Separate HW and SW l/rkeys")
---
 drivers/infiniband/sw/rxe/rxe_loc.h |  2 +-
 drivers/infiniband/sw/rxe/rxe_mr.c  | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -1
index 0e022ae1b8a5..37484a559d20 100644
@@ -77,7 +77,7 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
                         enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
-int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey);
+int rxe_invalidate_mr(struct rxe_qp *qp, u32 key);
 int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
 int rxe_mr_set_page(struct ib_mr *ibmr, u64 addr);
 int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
drivers/infiniband/sw/rxe/rxe_mr.c +3 -3
index fc3942e04a1f..3add52129006 100644
@@ -576,22 +576,22 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
        return mr;
 }
 
-int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey)
+int rxe_invalidate_mr(struct rxe_qp *qp, u32 key)
 {
        struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
        struct rxe_mr *mr;
        int ret;
 
-       mr = rxe_pool_get_index(&rxe->mr_pool, rkey >> 8);
+       mr = rxe_pool_get_index(&rxe->mr_pool, key >> 8);
        if (!mr) {
-               pr_err("%s: No MR for rkey %#x\n", __func__, rkey);
+               pr_err("%s: No MR for key %#x\n", __func__, key);
                ret = -EINVAL;
                goto err;
        }
 
Quoted 13 lines
Quoted 11 lines
drivers/infiniband/sw/rxe/rxe_mr.c +3 -4
-       if (rkey != mr->rkey) {
-               pr_err("%s: rkey (%#x) doesn't match mr->rkey (%#x)\n",
-                       __func__, rkey, mr->rkey);
+       if (mr->rkey ? (key != mr->rkey) : (key != mr->lkey)) {
+               pr_err("%s: wr key (%#x) doesn't match mr key (%#x)\n",
+                       __func__, key, (mr->rkey ? mr->rkey : mr->lkey));
                ret = -EINVAL;
                goto err_drop_ref;
        }
--

2.25.1

Ping.

Ping.

Does this need more discussions or changes?

Quoted 2 lines
Quoted 2 lines
#3 lore raw
Quoted 71 lines · haris iqbal
Quoted 58 lines · haris iqbal
Quoted 56 lines · Md Haris Iqbal

The 'rkey' input can be an lkey or rkey, and in rxe the lkey or rkey have the same value, including the variant bits.

So, if mr->rkey is set, compare the invalidate key with it, otherwise compare with the mr->lkey.

Since we already did a lookup on the non-varient bits to get this far, the check's only purpose is to confirm that the wqe has the correct variant bits.

Fixes 001345339f4c ("RDMA/rxe: Separate HW and SW l/rkeys")
---
 drivers/infiniband/sw/rxe/rxe_loc.h |  2 +-
 drivers/infiniband/sw/rxe/rxe_mr.c  | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -1
index 0e022ae1b8a5..37484a559d20 100644
@@ -77,7 +77,7 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
                         enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
-int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey);
+int rxe_invalidate_mr(struct rxe_qp *qp, u32 key);
 int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
 int rxe_mr_set_page(struct ib_mr *ibmr, u64 addr);
 int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
drivers/infiniband/sw/rxe/rxe_mr.c +3 -3
index fc3942e04a1f..3add52129006 100644
@@ -576,22 +576,22 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
        return mr;
 }
 
-int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey)
+int rxe_invalidate_mr(struct rxe_qp *qp, u32 key)
 {
        struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
        struct rxe_mr *mr;
        int ret;
 
-       mr = rxe_pool_get_index(&rxe->mr_pool, rkey >> 8);
+       mr = rxe_pool_get_index(&rxe->mr_pool, key >> 8);
        if (!mr) {
-               pr_err("%s: No MR for rkey %#x\n", __func__, rkey);
+               pr_err("%s: No MR for key %#x\n", __func__, key);
                ret = -EINVAL;
                goto err;
        }
 
Quoted 11 lines

drivers/infiniband/sw/rxe/rxe_mr.c struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key, · line 597

Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_mr.c +3 -3
-       if (rkey != mr->rkey) {
-               pr_err("%s: rkey (%#x) doesn't match mr->rkey (%#x)\n",
-                       __func__, rkey, mr->rkey);
+       if (mr->rkey ? (key != mr->rkey) : (key != mr->lkey)) {
+               pr_err("%s: wr key (%#x) doesn't match mr key (%#x)\n",
+                       __func__, key, (mr->rkey ? mr->rkey : mr->lkey));
                ret = -EINVAL;
                goto err_drop_ref;
        }

Ping.

Quoted 3 lines

Ping.

Does this need more discussions or changes?

I was hoping Bob would say something but I am happy with it..

Jason

#4 lore raw
Quoted 73 lines · Jason Gunthorpe
Quoted 71 lines · haris iqbal
Quoted 69 lines · haris iqbal
Quoted 55 lines · Md Haris Iqbal

The 'rkey' input can be an lkey or rkey, and in rxe the lkey or rkey have the same value, including the variant bits.

So, if mr->rkey is set, compare the invalidate key with it, otherwise compare with the mr->lkey.

Since we already did a lookup on the non-varient bits to get this far, the check's only purpose is to confirm that the wqe has the correct variant bits.

Fixes 001345339f4c ("RDMA/rxe: Separate HW and SW l/rkeys")
---
 drivers/infiniband/sw/rxe/rxe_loc.h |  2 +-
 drivers/infiniband/sw/rxe/rxe_mr.c  | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -1
index 0e022ae1b8a5..37484a559d20 100644
@@ -77,7 +77,7 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
                         enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
-int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey);
+int rxe_invalidate_mr(struct rxe_qp *qp, u32 key);
 int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
 int rxe_mr_set_page(struct ib_mr *ibmr, u64 addr);
 int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
drivers/infiniband/sw/rxe/rxe_mr.c +3 -3
index fc3942e04a1f..3add52129006 100644
@@ -576,22 +576,22 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
        return mr;
 }
 
-int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey)
+int rxe_invalidate_mr(struct rxe_qp *qp, u32 key)
 {
        struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
        struct rxe_mr *mr;
        int ret;
 
-       mr = rxe_pool_get_index(&rxe->mr_pool, rkey >> 8);
+       mr = rxe_pool_get_index(&rxe->mr_pool, key >> 8);
        if (!mr) {
-               pr_err("%s: No MR for rkey %#x\n", __func__, rkey);
+               pr_err("%s: No MR for key %#x\n", __func__, key);
                ret = -EINVAL;
                goto err;
        }

drivers/infiniband/sw/rxe/rxe_mr.c struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key, · line 597

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +3 -3
 
-       if (rkey != mr->rkey) {
-               pr_err("%s: rkey (%#x) doesn't match mr->rkey (%#x)\n",
-                       __func__, rkey, mr->rkey);
+       if (mr->rkey ? (key != mr->rkey) : (key != mr->lkey)) {
+               pr_err("%s: wr key (%#x) doesn't match mr key (%#x)\n",
+                       __func__, key, (mr->rkey ? mr->rkey : mr->lkey));
                ret = -EINVAL;
                goto err_drop_ref;
        }

Ping.

Quoted 5 lines
Quoted 3 lines

Ping.

Does this need more discussions or changes?

I was hoping Bob would say something but I am happy with it..

Thanks Jason for the response.
We can wait for Bob to share his thoughts.

Quoted 2 lines

Jason

Reviewed-by
#5 lore raw
Quoted 74 lines · haris iqbal
Quoted 73 lines · Jason Gunthorpe
Quoted 71 lines · haris iqbal
Quoted 69 lines · haris iqbal
Quoted 55 lines · Md Haris Iqbal

The 'rkey' input can be an lkey or rkey, and in rxe the lkey or rkey have the same value, including the variant bits.

So, if mr->rkey is set, compare the invalidate key with it, otherwise compare with the mr->lkey.

Since we already did a lookup on the non-varient bits to get this far, the check's only purpose is to confirm that the wqe has the correct variant bits.

Fixes 001345339f4c ("RDMA/rxe: Separate HW and SW l/rkeys")
---
 drivers/infiniband/sw/rxe/rxe_loc.h |  2 +-
 drivers/infiniband/sw/rxe/rxe_mr.c  | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -1
index 0e022ae1b8a5..37484a559d20 100644
@@ -77,7 +77,7 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
                         enum rxe_mr_lookup_type type);
 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length);
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
-int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey);
+int rxe_invalidate_mr(struct rxe_qp *qp, u32 key);
 int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
 int rxe_mr_set_page(struct ib_mr *ibmr, u64 addr);
 int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
drivers/infiniband/sw/rxe/rxe_mr.c +3 -3
index fc3942e04a1f..3add52129006 100644
@@ -576,22 +576,22 @@ struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
        return mr;
 }
 
-int rxe_invalidate_mr(struct rxe_qp *qp, u32 rkey)
+int rxe_invalidate_mr(struct rxe_qp *qp, u32 key)
 {
        struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
        struct rxe_mr *mr;
        int ret;
 
-       mr = rxe_pool_get_index(&rxe->mr_pool, rkey >> 8);
+       mr = rxe_pool_get_index(&rxe->mr_pool, key >> 8);
        if (!mr) {
-               pr_err("%s: No MR for rkey %#x\n", __func__, rkey);
+               pr_err("%s: No MR for key %#x\n", __func__, key);
                ret = -EINVAL;
                goto err;
        }

drivers/infiniband/sw/rxe/rxe_mr.c struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key, · line 597

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_mr.c +3 -3
 
-       if (rkey != mr->rkey) {
-               pr_err("%s: rkey (%#x) doesn't match mr->rkey (%#x)\n",
-                       __func__, rkey, mr->rkey);
+       if (mr->rkey ? (key != mr->rkey) : (key != mr->lkey)) {
+               pr_err("%s: wr key (%#x) doesn't match mr key (%#x)\n",
+                       __func__, key, (mr->rkey ? mr->rkey : mr->lkey));
                ret = -EINVAL;
                goto err_drop_ref;
        }

Ping.

Quoted 11 lines
Quoted 5 lines
Quoted 3 lines

Ping.

Does this need more discussions or changes?

I was hoping Bob would say something but I am happy with it..

Thanks Jason for the response.
We can wait for Bob to share his thoughts.

Quoted 2 lines

JasonFine by me

maintainer Applied
#6 lore raw
In reply to Md Haris Iqbal
Quoted 10 lines · Md Haris Iqbal

The 'rkey' input can be an lkey or rkey, and in rxe the lkey or rkey have the same value, including the variant bits.

So, if mr->rkey is set, compare the invalidate key with it, otherwise compare with the mr->lkey.

Since we already did a lookup on the non-varient bits to get this far, the check's only purpose is to confirm that the wqe has the correct variant bits.

Quoted 8 lines
Fixes 001345339f4c ("RDMA/rxe: Separate HW and SW l/rkeys")
---
 drivers/infiniband/sw/rxe/rxe_loc.h |  2 +-
 drivers/infiniband/sw/rxe/rxe_mr.c  | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

Applied to for-next, thanks

Jason