RDMA/rxe: Prefetching pages with explicit ODP
Subsystems (2)
v3 cover RDMA/rxe: Prefetching pages with explicit ODP
There is ibv_advise_mr(3) that can be used by applications to optimize memory access. This series enables the feature on rxe driver, which has already been available in mlx5.
There is a tiny change on the rdma-core util. cf. https://github.com/linux-rdma/rdma-core/pull/1605
Daisuke Matsuda (2):
RDMA/rxe: Implement synchronous prefetch for ODP MRs RDMA/rxe: Enable asynchronous prefetch for ODP MRs
drivers/infiniband/sw/rxe/rxe.c | 7 ++ drivers/infiniband/sw/rxe/rxe_loc.h | 10 ++ drivers/infiniband/sw/rxe/rxe_odp.c | 162 ++++++++++++++++++++++++++++ 3 files changed, 179 insertions(+)
Signature · 1 line
2.43.0
PATCH v3 1/2 RDMA/rxe: Implement synchronous prefetch for ODP MRs
Minimal implementation of ibv_advise_mr(3) requires synchronous calls being successful with the IBV_ADVISE_MR_FLAG_FLUSH flag. Asynchronous requests, which are best-effort, will be supported subsequently.
--- drivers/infiniband/sw/rxe/rxe.c | 7 +++ drivers/infiniband/sw/rxe/rxe_loc.h | 10 ++++ drivers/infiniband/sw/rxe/rxe_odp.c | 82 +++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+)
drivers/infiniband/sw/rxe/rxe.c +7 -0
@@ -34,6 +34,10 @@ void rxe_dealloc(struct ib_device *ib_dev)mutex_destroy (&rxe->usdev_lock); } +static const struct ib_device_ops rxe_ib_dev_odp_ops = { + .advise_mr = rxe_ib_advise_mr, +}; +/* initialize rxe device parameters */ static void rxe_init_device_param (struct rxe_dev *rxe,struct net_device *ndev) {
@@ -103,6 +107,9 @@ static void rxe_init_device_param(struct rxe_dev *rxe, struct net_device *ndev) rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_SRQ_RECV ; rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_FLUSH ; rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_ATOMIC_WRITE ; + +/* set handler for ODP prefetching API - ibv_advise_mr(3) */ +ib_set_device_ops (&rxe->ib_dev, &rxe_ib_dev_odp_ops); } }
drivers/infiniband/sw/rxe/rxe_loc.h +10 -0
@@ -197,6 +197,9 @@ enum resp_states rxe_odp_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,int rxe_odp_flush_pmem_iova (struct rxe_mr *mr,u64 iova,unsigned int length);enum resp_states rxe_odp_do_atomic_write (struct rxe_mr *mr,u64 iova,u64 value); +int rxe_ib_advise_mr (struct ib_pd *pd,enum ib_uverbs_advise_mr_advice advice, +u32 flags,struct ib_sge *sg_list,u32 num_sge, +struct uverbs_attr_bundle *attrs);#else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ static inline int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
@@ -225,6 +228,13 @@ static inline enum resp_states rxe_odp_do_atomic_write(struct rxe_mr *mr, {return RESPST_ERR_UNSUPPORTED_OPCODE ; } +static inline int rxe_ib_advise_mr (struct ib_pd *pd,enum ib_uverbs_advise_mr_advice advice, +u32 flags,struct ib_sge *sg_list,u32 num_sge, +struct uverbs_attr_bundle *attrs) +{ +return -EOPNOTSUPP ; +} +#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ #endif /* RXE_LOC_H */
drivers/infiniband/sw/rxe/rxe_odp.c +82 -0
85 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v3 2/2 RDMA/rxe: Enable asynchronous prefetch for ODP MRs
In reply to Daisuke MatsudaCalling ibv_advise_mr(3) with flags other than IBV_ADVISE_MR_FLAG_FLUSH invokes asynchronous request. It is best-effort, and thus can safely be deferred to the system-wide workqueue.
--- drivers/infiniband/sw/rxe/rxe_odp.c | 84 ++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-)
drivers/infiniband/sw/rxe/rxe_odp.c +82 -2
@@ -425,6 +425,73 @@ enum resp_states rxe_odp_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)return RESPST_NONE ; } +struct prefetch_mr_work { +struct work_struct work; +u32 pf_flags; +u32 num_sge; +struct { +u64 io_virt; +struct rxe_mr *mr; +size_t length; + } frags[]; +}; + +static void rxe_ib_prefetch_mr_work (struct work_struct *w) +{ +struct prefetch_mr_work *work = + container_of(w, struct prefetch_mr_work, work); +int ret; +u32 i; + +/* We rely on IB/core that work is executed if we have num_sge != 0 only. */ +WARN_ON (!work->num_sge); +for (i =0 ; i < work->num_sge; ++i) { +struct ib_umem_odp *umem_odp; + + ret =rxe_odp_do_pagefault_and_lock (work->frags[i].mr, work->frags[i].io_virt, + work->frags[i].length, work->pf_flags); +if (ret <0 ) { +rxe_dbg_mr (work->frags[i].mr,"failed to prefetch the mr\n" ); +continue ; + } + + umem_odp =to_ib_umem_odp (work->frags[i].mr->umem); +mutex_unlock (&umem_odp->umem_mutex); + } + +kvfree (work); +} + +static int rxe_init_prefetch_work (struct ib_pd *ibpd, +enum ib_uverbs_advise_mr_advice advice, +u32 pf_flags,struct prefetch_mr_work *work, +struct ib_sge *sg_list,u32 num_sge) +{ + struct rxe_pd *pd = container_of(ibpd, struct rxe_pd, ibpd); +u32 i; + +INIT_WORK (&work->work, rxe_ib_prefetch_mr_work); + work->pf_flags = pf_flags; + +for (i =0 ; i < num_sge; ++i) { +struct rxe_mr *mr; + + mr =lookup_mr (pd,IB_ACCESS_LOCAL_WRITE , + sg_list[i].lkey,RXE_LOOKUP_LOCAL ); +if (IS_ERR (mr)) { + work->num_sge = i; +return PTR_ERR (mr); + } + work->frags[i].io_virt = sg_list[i].addr; + work->frags[i].length = sg_list[i].length; + work->frags[i].mr = mr; + +rxe_put (mr); + } + work->num_sge = num_sge; +return 0 ; +} + static int rxe_ib_prefetch_sg_list(struct ib_pd *ibpd, enum ib_uverbs_advise_mr_advice advice, u32 pf_flags, struct ib_sge *sg_list,
@@ -475,6 +542,8 @@ static int rxe_ib_advise_mr_prefetch(struct ib_pd *ibpd, u32 flags, struct ib_sge *sg_list, u32 num_sge) { u32 pf_flags = RXE_PAGEFAULT_DEFAULT; +struct prefetch_mr_work *work; +int rc;if (advice ==IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH ) pf_flags |=RXE_PAGEFAULT_RDONLY ;
@@ -487,8 +556,19 @@ static int rxe_ib_advise_mr_prefetch(struct ib_pd *ibpd,return rxe_ib_prefetch_sg_list (ibpd, advice, pf_flags, sg_list, num_sge); -/* Asynchronous call is to be added in the next patch */ -return -EOPNOTSUPP ; +/* Asynchronous call is "best-effort" and allowed to fail */ + work =kvzalloc (struct_size (work, frags, num_sge),GFP_KERNEL ); +if (!work) +return -ENOMEM ; + + rc =rxe_init_prefetch_work (ibpd, advice, pf_flags, work, sg_list, num_sge); +if (rc) { +kvfree (work); +return rc; + } +queue_work (system_unbound_wq, &work->work); + +return 0 ; } int rxe_ib_advise_mr(struct ib_pd *ibpd,
Signature · 1 line
2.43.0
Re: [PATCH for-next v3 2/2] RDMA/rxe: Enable asynchronous prefetch for ODP MRs
Quoted 17 lines · Daisuke Matsuda
Calling
ibv_advise_mr(3)with flags other thanIBV_ADVISE_MR_FLAG_FLUSHinvokes asynchronous request. It is best-effort, and thus can safely be deferred to the system-wide workqueue.Signed-off-by Daisuke Matsuda <[email protected]>Reviewed-by Zhu Yanjun <[email protected]>--- drivers/infiniband/sw/rxe/rxe_odp.c | 84 ++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-)drivers/infiniband/sw/rxe/rxe_odp.c +0
-0@@ -425,6 +425,73 @@ enum resp_states rxe_odp_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)return RESPST_NONE ; }
<...>
drivers/infiniband/sw/rxe/rxe_odp.c enum resp_states rxe_odp_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) · line 468
Quoted 4 lines
drivers/infiniband/sw/rxe/rxe_odp.c +4
-0+static int rxe_init_prefetch_work (struct ib_pd *ibpd, +enum ib_uverbs_advise_mr_advice advice, +u32 pf_flags,struct prefetch_mr_work *work, +struct ib_sge *sg_list,u32 num_sge)
There is no need one-time called function. It can be embedded into rxe_ib_advise_mr_prefetch().
Quoted 1 line
+{
<...>
Quoted 3 lines
drivers/infiniband/sw/rxe/rxe_odp.c +0
-0@@ -475,6 +542,8 @@ static int rxe_ib_advise_mr_prefetch(struct ib_pd *ibpd, u32 flags, struct ib_sge *sg_list, u32 num_sge) {
<...>
drivers/infiniband/sw/rxe/rxe_odp.c static int rxe_ib_advise_mr_prefetch(struct ib_pd *ibpd, · line 569
Quoted 1 line
drivers/infiniband/sw/rxe/rxe_odp.c +1
-0+queue_work (system_unbound_wq, &work->work);
How do you ensure that this work isn't running after RXE is destroyed?
Thanks
Quoted 8 lines
drivers/infiniband/sw/rxe/rxe_odp.c +2
-0+ +return 0 ; } int rxe_ib_advise_mr(struct ib_pd *ibpd,Signature · 1 line
2.43.0
Re: [PATCH for-next v3 2/2] RDMA/rxe: Enable asynchronous prefetch for ODP MRs
Quoted 21 lines · Leon Romanovsky
Quoted 17 lines · Daisuke Matsuda
Calling
ibv_advise_mr(3)with flags other thanIBV_ADVISE_MR_FLAG_FLUSHinvokes asynchronous request. It is best-effort, and thus can safely be deferred to the system-wide workqueue.Signed-off-by Daisuke Matsuda <[email protected]>Reviewed-by Zhu Yanjun <[email protected]>--- drivers/infiniband/sw/rxe/rxe_odp.c | 84 ++++++++++++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-)drivers/infiniband/sw/rxe/rxe_odp.c +0
-0@@ -425,6 +425,73 @@ enum resp_states rxe_odp_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)return RESPST_NONE ; }<...>
Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_odp.c enum resp_states rxe_odp_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) · line 468
Quoted 4 lines
drivers/infiniband/sw/rxe/rxe_odp.c +4
-0+static int rxe_init_prefetch_work (struct ib_pd *ibpd, +enum ib_uverbs_advise_mr_advice advice, +u32 pf_flags,struct prefetch_mr_work *work, +struct ib_sge *sg_list,u32 num_sge)There is no need one-time called function. It can be embedded into
rxe_ib_advise_mr_prefetch().
Certainly.
Quoted 11 lines
Quoted 2 lines
+{
<...>
Quoted 3 lines
drivers/infiniband/sw/rxe/rxe_odp.c +0
-0@@ -475,6 +542,8 @@ static int rxe_ib_advise_mr_prefetch(struct ib_pd *ibpd, u32 flags, struct ib_sge *sg_list, u32 num_sge) {<...>
Quoted 3 lines
drivers/infiniband/sw/rxe/rxe_odp.c static int rxe_ib_advise_mr_prefetch(struct ib_pd *ibpd, · line 569
Quoted 1 line
drivers/infiniband/sw/rxe/rxe_odp.c +1
-0+queue_work (system_unbound_wq, &work->work);How do you ensure that this work isn't running after RXE is destroyed?
I think we can use per-pd reference counter in struct rxe_pd. I will fix it in v4.
Thanks,
Daisuke
Quoted 11 lines
Thanks
Quoted 8 lines
drivers/infiniband/sw/rxe/rxe_odp.c +2
-0+ +return 0 ; } int rxe_ib_advise_mr(struct ib_pd *ibpd,Signature · 1 line
2.43.0