Enable rcu locking of verbs objects
Subsystems (2)
cover Enable rcu locking of verbs objects
This patch set consists of two patches. The first adds code to rdma-core to optionally use kfree_rcu instead of kfree for three of the verbs objects which are looked up by their indices in the rxe driver and which are freed in the destroy verbs in rdma-core. The second patch adds rcu_head to the private data in the rxe driver and sets the offsets to these structs. This allows the rxe driver to correctly use rcu locking on these objects.
Bob Pearson (2):
RDMA/core: Support drivers use of rcu locking
RDMA/rxe: Enable rcu locking of indexed objects
drivers/infiniband/core/uverbs_main.c | 2 +- drivers/infiniband/core/verbs.c | 6 +++--- drivers/infiniband/sw/rxe/rxe_pool.h | 1 + drivers/infiniband/sw/rxe/rxe_verbs.c | 6 +++++- include/rdma/ib_verbs.h | 24 ++++++++++++++++++++++++ 5 files changed, 34 insertions(+), 5 deletions(-)
base-commit: f877f22ac1e9bf1f9aded3765b0012851e1dc4c5
Signature · 1 line
2.39.2
PATCH 1/2 RDMA/core: Support drivers use of rcu locking
This patch allows drivers to optionally include struct rcu_head in their private object data structs and have rdma-core use kfree_rcu to free the objects.
The offsets of the rcu_heads are stored in fields in struct ib_device_ops and a macro RDMA_KFREE_RCU is introduced which calls (an open coded) kfree_rcu instead of kfree if the value is non- zero. Currently the supported object types are AH, QP and MW.
--- drivers/infiniband/core/uverbs_main.c | 2 +- drivers/infiniband/core/verbs.c | 6 +++--- include/rdma/ib_verbs.h | 24 ++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-)
drivers/infiniband/core/uverbs_main.c +1 -1
@@ -112,7 +112,7 @@ int uverbs_dealloc_mw(struct ib_mw *mw)return ret;atomic_dec (&pd->usecnt); -kfree (mw); +RDMA_KFREE_RCU (mw);return ret; }
drivers/infiniband/core/verbs.c +3 -3
@@ -982,7 +982,7 @@ int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata)if (sgid_attr)rdma_put_gid_attr (sgid_attr); -kfree (ah); +RDMA_KFREE_RCU (ah);return ret; }EXPORT_SYMBOL (rdma_destroy_ah_user);
@@ -1970,7 +1970,7 @@ int ib_close_qp(struct ib_qp *qp)atomic_dec (&real_qp->usecnt);if (qp->qp_sec)ib_close_shared_qp_security (qp->qp_sec); -kfree (qp); +RDMA_KFREE_RCU (qp);return 0 ; }
@@ -2041,7 +2041,7 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata)ib_destroy_qp_security_end (sec);rdma_restrack_del (&qp->res); -kfree (qp); +RDMA_KFREE_RCU (qp);return ret; }EXPORT_SYMBOL (ib_destroy_qp_user);
include/rdma/ib_verbs.h +24 -0
@@ -2684,8 +2684,32 @@ struct ib_device_ops {DECLARE_RDMA_OBJ_SIZE (ib_srq);DECLARE_RDMA_OBJ_SIZE (ib_ucontext);DECLARE_RDMA_OBJ_SIZE (ib_xrcd); + +/* if non-zero holds offset of rcu_head in object */ +ssize_t rcu_offset_ah; +ssize_t rcu_offset_qp; +ssize_t rcu_offset_mw; + }; +/* drivers may optionally use rcu locking on verbs objects +* by including struct rcu_head in their private data and +* setting rcu_offset_xx in the ib_driver_ops struct where +* xx is one of ah, qp or mw. +*/ +static inline void __rdma_kfree_rcu (void *obj,ssize_t rcu_offset) +{ +if (rcu_offset) +kvfree_call_rcu ((struct rcu_head *)((u8 *)obj + rcu_offset), + obj); +else +kfree (obj); +} + +/* obj must be one of ah, qp, or mw */ +#define RDMA_KFREE_RCU (obj) __rdma_kfree_rcu(obj, \ + obj->device->ops.rcu_offset_##obj) +struct ib_core_device {/* device must be the first element in structure until, * union of ib_core_device and device exists in ib_device.
Signature · 1 line
2.39.2
Jason Gunthorpe
maintainerRe: [PATCH for-next 1/2] RDMA/core: Support drivers use of rcu locking
Quoted 65 lines · Bob Pearson
This patch allows drivers to optionally include struct
rcu_headin their private object data structs and have rdma-core usekfree_rcuto free the objects.The offsets of the
rcu_headsare stored in fields in structib_device_opsand a macroRDMA_KFREE_RCUis introduced which calls (an open coded)kfree_rcuinstead of kfree if the value is non- zero. Currently the supported object types are AH, QP and MW.Signed-off-by Bob Pearson <[email protected]>--- drivers/infiniband/core/uverbs_main.c | 2 +- drivers/infiniband/core/verbs.c | 6 +++--- include/rdma/ib_verbs.h | 24 ++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-)drivers/infiniband/core/uverbs_main.c +1
-1@@ -112,7 +112,7 @@ int uverbs_dealloc_mw(struct ib_mw *mw)return ret;atomic_dec (&pd->usecnt); -kfree (mw); +RDMA_KFREE_RCU (mw);return ret; }drivers/infiniband/core/verbs.c +3
-3@@ -982,7 +982,7 @@ int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata)if (sgid_attr)rdma_put_gid_attr (sgid_attr); -kfree (ah); +RDMA_KFREE_RCU (ah);return ret; }EXPORT_SYMBOL (rdma_destroy_ah_user);@@ -1970,7 +1970,7 @@ int ib_close_qp(struct ib_qp *qp)atomic_dec (&real_qp->usecnt);if (qp->qp_sec)ib_close_shared_qp_security (qp->qp_sec); -kfree (qp); +RDMA_KFREE_RCU (qp);return 0 ; }@@ -2041,7 +2041,7 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata)ib_destroy_qp_security_end (sec);rdma_restrack_del (&qp->res); -kfree (qp); +RDMA_KFREE_RCU (qp);return ret; }EXPORT_SYMBOL (ib_destroy_qp_user);include/rdma/ib_verbs.h +0
-0
include/rdma/ib_verbs.h struct ib_device_ops { · line 2691
Quoted 8 lines
include/rdma/ib_verbs.h +5
-0@@ -2684,8 +2684,32 @@ struct ib_device_ops {DECLARE_RDMA_OBJ_SIZE (ib_srq);DECLARE_RDMA_OBJ_SIZE (ib_ucontext);DECLARE_RDMA_OBJ_SIZE (ib_xrcd); + +/* if non-zero holds offset of rcu_head in object */ +ssize_t rcu_offset_ah; +ssize_t rcu_offset_qp; +ssize_t rcu_offset_mw;
Just size_t, this isn't negative
include/rdma/ib_verbs.h struct ib_device_ops { · line 2700
Quoted 1 line
include/rdma/ib_verbs.h +1
-0+static inline void __rdma_kfree_rcu (void *obj,ssize_t rcu_offset)
Here too
Jason
PATCH 2/2 RDMA/rxe: Enable rcu locking of indexed objects
In reply to Bob PearsonMake rcu_read locking of critical sections with the indexed verbs objects be protected from early freeing of those objects. The AH, QP, MR and MW objects are looked up from their indices contained in received packets or WQEs during I/O processing. Make these objects be freed using kfree_rcu to avoid races.
--- drivers/infiniband/sw/rxe/rxe_pool.h | 1 + drivers/infiniband/sw/rxe/rxe_verbs.c | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-)
drivers/infiniband/sw/rxe/rxe_pool.h +1 -0
@@ -25,6 +25,7 @@ struct rxe_pool_elem {struct kref ref_cnt;struct list_head list;struct completion complete; +struct rcu_head rcu;u32 index; };
drivers/infiniband/sw/rxe/rxe_verbs.c +5 -1
@@ -1395,7 +1395,7 @@ static int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)if (cleanup_err)rxe_err_mr (mr,"cleanup failed, err = %d" , cleanup_err); -kfree_rcu_mightsleep (mr); +kfree_rcu (mr, elem.rcu);return 0 ; err_out:
@@ -1494,6 +1494,10 @@ static const struct ib_device_ops rxe_dev_ops = {INIT_RDMA_OBJ_SIZE (ib_srq, rxe_srq, ibsrq),INIT_RDMA_OBJ_SIZE (ib_ucontext, rxe_ucontext, ibuc),INIT_RDMA_OBJ_SIZE (ib_mw, rxe_mw, ibmw), + + .rcu_offset_ah = offsetof(struct rxe_ah, elem.rcu), + .rcu_offset_qp = offsetof(struct rxe_qp, elem.rcu), + .rcu_offset_mw = offsetof(struct rxe_mw, elem.rcu), };int rxe_register_device (struct rxe_dev *rxe,const char *ibdev_name)
Signature · 1 line
2.39.2
Re: [PATCH for-next 2/2] RDMA/rxe: Enable rcu locking of indexed objects
Quoted 39 lines · Bob Pearson
Make
rcu_readlocking of critical sections with the indexed verbs objects be protected from early freeing of those objects. The AH, QP, MR and MW objects are looked up from their indices contained in received packets or WQEs during I/O processing. Make these objects be freed usingkfree_rcuto avoid races.Signed-off-by Bob Pearson <[email protected]>--- drivers/infiniband/sw/rxe/rxe_pool.h | 1 + drivers/infiniband/sw/rxe/rxe_verbs.c | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-)drivers/infiniband/sw/rxe/rxe_pool.h +1
-0@@ -25,6 +25,7 @@ struct rxe_pool_elem {struct kref ref_cnt;struct list_head list;struct completion complete; +struct rcu_head rcu;u32 index; };drivers/infiniband/sw/rxe/rxe_verbs.c +1
-1@@ -1395,7 +1395,7 @@ static int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)if (cleanup_err)rxe_err_mr (mr,"cleanup failed, err = %d" , cleanup_err); -kfree_rcu_mightsleep (mr); +kfree_rcu (mr, elem.rcu);return 0 ; err_out:
drivers/infiniband/sw/rxe/rxe_verbs.c static const struct ib_device_ops rxe_dev_ops = { · line 1500
Quoted 7 lines
drivers/infiniband/sw/rxe/rxe_verbs.c +4
-0@@ -1494,6 +1494,10 @@ static const struct ib_device_ops rxe_dev_ops = {INIT_RDMA_OBJ_SIZE (ib_srq, rxe_srq, ibsrq),INIT_RDMA_OBJ_SIZE (ib_ucontext, rxe_ucontext, ibuc),INIT_RDMA_OBJ_SIZE (ib_mw, rxe_mw, ibmw), + + .rcu_offset_ah = offsetof(struct rxe_ah, elem.rcu), + .rcu_offset_qp = offsetof(struct rxe_qp, elem.rcu), + .rcu_offset_mw = offsetof(struct rxe_mw, elem.rcu),
In your commits, you mentioned that ah, qp, mw and mr will be protected. But here, only ah, qp and mw are set. Not sure if mr is also protected and set in other place. Except that, I am fine with it.
Zhu Yanjun
Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_verbs.c +0
-1};int rxe_register_device (struct rxe_dev *rxe,const char *ibdev_name) --2.39.2
Re: [PATCH for-next 2/2] RDMA/rxe: Enable rcu locking of indexed objects
Quoted 53 lines · Zhu Yanjun
Quoted 39 lines · Bob Pearson
Make
rcu_readlocking of critical sections with the indexed verbs objects be protected from early freeing of those objects. The AH, QP, MR and MW objects are looked up from their indices contained in received packets or WQEs during I/O processing. Make these objects be freed usingkfree_rcuto avoid races.Signed-off-by Bob Pearson <[email protected]>--- drivers/infiniband/sw/rxe/rxe_pool.h | 1 + drivers/infiniband/sw/rxe/rxe_verbs.c | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-)drivers/infiniband/sw/rxe/rxe_pool.h +1
-0@@ -25,6 +25,7 @@ struct rxe_pool_elem {struct kref ref_cnt;struct list_head list;struct completion complete; +struct rcu_head rcu;u32 index; };drivers/infiniband/sw/rxe/rxe_verbs.c +1
-1@@ -1395,7 +1395,7 @@ static int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)if (cleanup_err)rxe_err_mr (mr,"cleanup failed, err = %d" , cleanup_err); -kfree_rcu_mightsleep (mr); +kfree_rcu (mr, elem.rcu);return 0 ; err_out:drivers/infiniband/sw/rxe/rxe_verbs.c static const struct ib_device_ops rxe_dev_ops = { · line 1500
Quoted 7 lines
drivers/infiniband/sw/rxe/rxe_verbs.c +4
-0@@ -1494,6 +1494,10 @@ static const struct ib_device_ops rxe_dev_ops = {INIT_RDMA_OBJ_SIZE (ib_srq, rxe_srq, ibsrq),INIT_RDMA_OBJ_SIZE (ib_ucontext, rxe_ucontext, ibuc),INIT_RDMA_OBJ_SIZE (ib_mw, rxe_mw, ibmw), + + .rcu_offset_ah = offsetof(struct rxe_ah, elem.rcu), + .rcu_offset_qp = offsetof(struct rxe_qp, elem.rcu), + .rcu_offset_mw = offsetof(struct rxe_mw, elem.rcu),In your commits, you mentioned that ah, qp, mw and mr will be protected. But here, only ah, qp and mw are set. Not sure if mr is also protected and set in other place. Except that, I am fine with it.
Quoted 10 lines
Acked-by Zhu Yanjun <[email protected]>Zhu Yanjun
Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_verbs.c +0
-1};int rxe_register_device (struct rxe_dev *rxe,const char *ibdev_name) --2.39.2
All of the verbs objects except MR are allocated and freed by rdma-core. MR is allocated and freed by the drivers. So for MR the kfree_rcu call is made in rxe_dereg_mr(). I changed it from kfree_rcu_mightsleep to the kfree_rcu variant to match the three others.
Bob
Re: [PATCH for-next 2/2] RDMA/rxe: Enable rcu locking of indexed objects
In reply to Bob PearsonQuoted 5 lines · Bob Pearson
Make
rcu_readlocking of critical sections with the indexed verbs objects be protected from early freeing of those objects. The AH, QP, MR and MW objects are looked up from their indices contained in received packets or WQEs during I/O processing. Make these objects be freed usingkfree_rcuto avoid races.
Sorry, how use of RCU avoid races?
Thanks
Re: [PATCH for-next 2/2] RDMA/rxe: Enable rcu locking of indexed objects
Quoted 10 lines · Leon Romanovsky
Quoted 5 lines · Bob Pearson
Make
rcu_readlocking of critical sections with the indexed verbs objects be protected from early freeing of those objects. The AH, QP, MR and MW objects are looked up from their indices contained in received packets or WQEs during I/O processing. Make these objects be freed usingkfree_rcuto avoid races.Sorry, how use of RCU avoid races?
Thanks
The races are between destroy/deallocate/dereg verbs API calls and packets arriving or completing send or deferred processing of wqes. Packets and wqes contain indices/keys/numbers that refer to objects. The rxe driver maintains xarrays for each type of object that allow to lookup the address of the object from its index and then take a reference to protect the pointer. The destroy verbs defer completion until the reference count falls to zero and then removes the entry in the xarray. These operations need to be atomic. One alternative is to use spinlocks to protect them but that places a load on performance under heavy load which is typically dominated by the lookup function since objects tend to have a long lifetime. rcu readlocks are a better alternative but depend on the deferred destruction of the objects used in the rcu critical section. In certain benchmarks the use of rcu locking has a very large performance advantage over spinlocks. For example 100s of QPs running over a 200 gbit/s network.
Bob
Re: [PATCH for-next 2/2] RDMA/rxe: Enable rcu locking of indexed objects
Quoted 12 lines · Bob Pearson
Quoted 10 lines · Leon Romanovsky
Quoted 5 lines · Bob Pearson
Make
rcu_readlocking of critical sections with the indexed verbs objects be protected from early freeing of those objects. The AH, QP, MR and MW objects are looked up from their indices contained in received packets or WQEs during I/O processing. Make these objects be freed usingkfree_rcuto avoid races.Sorry, how use of RCU avoid races?
Thanks
Quoted 9 lines
The races are between destroy/deallocate/dereg verbs API calls and packets arriving or completing send or deferred processing of wqes. Packets and wqes contain indices/keys/numbers that refer to objects. The rxe driver maintains xarrays for each type of object that allow to lookup the address of the object from its index and then take a reference to protect the pointer. The destroy verbs defer completion until the reference count falls to zero and then removes the entry in the xarray. These operations need to be atomic. One alternative is to use spinlocks to protect them but that places a load on performance under heavy load which is typically dominated by the lookup function since objects tend to have a long lifetime. rcu readlocks are a better alternative but depend on the deferred destruction of the objects used in the rcu critical section.
You rarely can replace locks with RCU without careful design changes.
Thanks
Jason Gunthorpe
maintainerRe: [PATCH for-next 2/2] RDMA/rxe: Enable rcu locking of indexed objects
In reply to Bob PearsonQuoted 38 lines · Bob Pearson
Make
rcu_readlocking of critical sections with the indexed verbs objects be protected from early freeing of those objects. The AH, QP, MR and MW objects are looked up from their indices contained in received packets or WQEs during I/O processing. Make these objects be freed usingkfree_rcuto avoid races.Signed-off-by Bob Pearson <[email protected]>--- drivers/infiniband/sw/rxe/rxe_pool.h | 1 + drivers/infiniband/sw/rxe/rxe_verbs.c | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-)drivers/infiniband/sw/rxe/rxe_pool.h +1
-0@@ -25,6 +25,7 @@ struct rxe_pool_elem {struct kref ref_cnt;struct list_head list;struct completion complete; +struct rcu_head rcu;u32 index; };drivers/infiniband/sw/rxe/rxe_verbs.c +1
-1@@ -1395,7 +1395,7 @@ static int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)if (cleanup_err)rxe_err_mr (mr,"cleanup failed, err = %d" , cleanup_err); -kfree_rcu_mightsleep (mr); +kfree_rcu (mr, elem.rcu);return 0 ; err_out:
drivers/infiniband/sw/rxe/rxe_verbs.c static const struct ib_device_ops rxe_dev_ops = { · line 1501
Quoted 8 lines
drivers/infiniband/sw/rxe/rxe_verbs.c +4
-0@@ -1494,6 +1494,10 @@ static const struct ib_device_ops rxe_dev_ops = {INIT_RDMA_OBJ_SIZE (ib_srq, rxe_srq, ibsrq),INIT_RDMA_OBJ_SIZE (ib_ucontext, rxe_ucontext, ibuc),INIT_RDMA_OBJ_SIZE (ib_mw, rxe_mw, ibmw), + + .rcu_offset_ah = offsetof(struct rxe_ah, elem.rcu), + .rcu_offset_qp = offsetof(struct rxe_qp, elem.rcu), + .rcu_offset_mw = offsetof(struct rxe_mw, elem.rcu), };
This isn't quite the right calculation.. It works because the ib struct is at the top of these structs
Do it like this:
/*
* When used the core will RCU free the object using the driver provided
* rcu member.
*/
#define INIT_RDMA_OBJ_SIZE_RCU(ib_struct, drv_struct, member, rcu_member) \ .rcu_offset_##ib_struct = \ (offsetof(struct drv_struct, rcu_member) - \ offsetof(struct drv_struct, member) + \ BUILD_BUG_ON_ZERO(offsetof(struct drv_struct, rcu_member) > \ offsetof(struct drv_struct, member))), \ INIT_RDMA_OBJ_SIZE(ib_struct, drv_struct, member),
(and maybe the prior patch could stick with the ssize_t and we could remove the BUILD_BUG_ON_ZERO?)
Jason