RDMA/rxe: Prefetching pages with explicit ODP
Subsystems (2)
v4 cover RDMA/rxe: Prefetching pages with explicit ODP
There is ibv_advise_mr(3) that can be used by applications to optimize memory access. This series enables the feature on rxe driver, which has already been available in mlx5.
There is a tiny change on the rdma-core util. cf. https://github.com/linux-rdma/rdma-core/pull/1605
Currently, there are regressions in the for-next tree. When running ODP tests, ib_uverbs and rxe can get stuck respectively. Therefore, to test this series, please apply the patches to commit e56b4eab9cde ("RDMA/siw: Remove unused siw_mem_add"). The regressions are under investigation.
cf. https://lore.kernel.org/linux-rdma/20250522083257.GM7435@unreal/T/#t
v2->v3:
- Added return -EOPNOTSUPP; (1st patch)
- Corrected type definitions mismatch (1st patch)
v3->v4:
- merged rxe_init_prefetch_work() to rxe_ib_advise_mr_prefetch() (2nd patch)
- Modified to use per-MR counter to ensure RXE is not destroyed while the asynchronous task is running (2nd patch)
Daisuke Matsuda (2):
RDMA/rxe: Implement synchronous prefetch for ODP MRs RDMA/rxe: Enable asynchronous prefetch for ODP MRs
drivers/infiniband/sw/rxe/rxe.c | 7 ++ drivers/infiniband/sw/rxe/rxe_loc.h | 10 ++ drivers/infiniband/sw/rxe/rxe_odp.c | 159 ++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+)
Signature · 1 line
2.43.0
PATCH v4 1/2 RDMA/rxe: Implement synchronous prefetch for ODP MRs
Minimal implementation of ibv_advise_mr(3) requires synchronous calls being successful with the IBV_ADVISE_MR_FLAG_FLUSH flag. Asynchronous requests, which are best-effort, will be supported subsequently.
--- drivers/infiniband/sw/rxe/rxe.c | 7 +++ drivers/infiniband/sw/rxe/rxe_loc.h | 10 ++++ drivers/infiniband/sw/rxe/rxe_odp.c | 82 +++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+)
drivers/infiniband/sw/rxe/rxe.c +7 -0
@@ -34,6 +34,10 @@ void rxe_dealloc(struct ib_device *ib_dev)mutex_destroy (&rxe->usdev_lock); } +static const struct ib_device_ops rxe_ib_dev_odp_ops = { + .advise_mr = rxe_ib_advise_mr, +}; +/* initialize rxe device parameters */ static void rxe_init_device_param (struct rxe_dev *rxe,struct net_device *ndev) {
@@ -103,6 +107,9 @@ static void rxe_init_device_param(struct rxe_dev *rxe, struct net_device *ndev) rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_SRQ_RECV ; rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_FLUSH ; rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_ATOMIC_WRITE ; + +/* set handler for ODP prefetching API - ibv_advise_mr(3) */ +ib_set_device_ops (&rxe->ib_dev, &rxe_ib_dev_odp_ops); } }
drivers/infiniband/sw/rxe/rxe_loc.h +10 -0
@@ -197,6 +197,9 @@ enum resp_states rxe_odp_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,int rxe_odp_flush_pmem_iova (struct rxe_mr *mr,u64 iova,unsigned int length);enum resp_states rxe_odp_do_atomic_write (struct rxe_mr *mr,u64 iova,u64 value); +int rxe_ib_advise_mr (struct ib_pd *pd,enum ib_uverbs_advise_mr_advice advice, +u32 flags,struct ib_sge *sg_list,u32 num_sge, +struct uverbs_attr_bundle *attrs);#else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ static inline int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
@@ -225,6 +228,13 @@ static inline enum resp_states rxe_odp_do_atomic_write(struct rxe_mr *mr, {return RESPST_ERR_UNSUPPORTED_OPCODE ; } +static inline int rxe_ib_advise_mr (struct ib_pd *pd,enum ib_uverbs_advise_mr_advice advice, +u32 flags,struct ib_sge *sg_list,u32 num_sge, +struct uverbs_attr_bundle *attrs) +{ +return -EOPNOTSUPP ; +} +#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ #endif /* RXE_LOC_H */
drivers/infiniband/sw/rxe/rxe_odp.c +82 -0
85 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v4 2/2 RDMA/rxe: Enable asynchronous prefetch for ODP MRs
In reply to Daisuke MatsudaCalling ibv_advise_mr(3) with flags other than IBV_ADVISE_MR_FLAG_FLUSH invokes an asynchronous request. It is best-effort, and thus can safely be deferred to the system-wide workqueue.
The reference counter in rxe_mr is used to ensure that the MRs persist and that rxe is not terminated until the queued work is done.
--- drivers/infiniband/sw/rxe/rxe_odp.c | 81 ++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-)
drivers/infiniband/sw/rxe/rxe_odp.c +79 -2
@@ -423,6 +423,46 @@ enum resp_states rxe_odp_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)return RESPST_NONE ; } +struct prefetch_mr_work { +struct work_struct work; +u32 pf_flags; +u32 num_sge; +struct { +u64 io_virt; +struct rxe_mr *mr; +size_t length; + } frags[]; +}; + +static void rxe_ib_prefetch_mr_work (struct work_struct *w) +{ +struct prefetch_mr_work *work = + container_of(w, struct prefetch_mr_work, work); +int ret; +u32 i; + +/* We rely on IB/core that work is executed if we have num_sge != 0 only. */ +WARN_ON (!work->num_sge); +for (i =0 ; i < work->num_sge; ++i) { +struct ib_umem_odp *umem_odp; + + ret =rxe_odp_do_pagefault_and_lock (work->frags[i].mr, work->frags[i].io_virt, + work->frags[i].length, work->pf_flags); +if (ret <0 ) { +rxe_dbg_mr (work->frags[i].mr,"failed to prefetch the mr\n" ); + goto deref; + } + + umem_odp =to_ib_umem_odp (work->frags[i].mr->umem); +mutex_unlock (&umem_odp->umem_mutex); + +deref: +rxe_put (work->frags[i].mr); + } + +kvfree (work); +} + static int rxe_ib_prefetch_sg_list(struct ib_pd *ibpd, enum ib_uverbs_advise_mr_advice advice, u32 pf_flags, struct ib_sge *sg_list,
@@ -472,7 +512,11 @@ static int rxe_ib_advise_mr_prefetch(struct ib_pd *ibpd,enum ib_uverbs_advise_mr_advice advice, u32 flags, struct ib_sge *sg_list, u32 num_sge) { + struct rxe_pd *pd = container_of(ibpd, struct rxe_pd, ibpd);u32 pf_flags =RXE_PAGEFAULT_DEFAULT ; +struct prefetch_mr_work *work; +struct rxe_mr *mr; +u32 i;if (advice ==IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH ) pf_flags |=RXE_PAGEFAULT_RDONLY ;
@@ -485,8 +529,41 @@ static int rxe_ib_advise_mr_prefetch(struct ib_pd *ibpd,return rxe_ib_prefetch_sg_list (ibpd, advice, pf_flags, sg_list, num_sge); -/* Asynchronous call is to be added in the next patch */ -return -EOPNOTSUPP ; +/* Asynchronous call is "best-effort" and allowed to fail */ + work =kvzalloc (struct_size (work, frags, num_sge),GFP_KERNEL ); +if (!work) +return -ENOMEM ; + +INIT_WORK (&work->work, rxe_ib_prefetch_mr_work); + work->pf_flags = pf_flags; + work->num_sge = num_sge; + +for (i =0 ; i < num_sge; ++i) { +/* takes a reference, which will be released in the queued work */ + mr =lookup_mr (pd,IB_ACCESS_LOCAL_WRITE , + sg_list[i].lkey,RXE_LOOKUP_LOCAL ); +if (IS_ERR (mr)) + goto err; + + work->frags[i].io_virt = sg_list[i].addr; + work->frags[i].length = sg_list[i].length; + work->frags[i].mr = mr; + } + +queue_work (system_unbound_wq, &work->work); + +return 0 ; + + err: +/* rollback reference counts for the invalid request */ +while (i >0 ) { + i--; +rxe_put (work->frags[i].mr); + } + +kvfree (work); + +return PTR_ERR (mr); } int rxe_ib_advise_mr(struct ib_pd *ibpd,
Signature · 1 line
2.43.0
Leon Romanovsky
AppliedQuoted 8 lines · Daisuke Matsuda
There is
ibv_advise_mr(3)that can be used by applications to optimize memory access. This series enables the feature on rxe driver, which has already been available in mlx5.There is a tiny change on the rdma-core util. cf. https://github.com/linux-rdma/rdma-core/pull/1605
[...]
Applied, thanks!
[1/2] RDMA/rxe: Implement synchronous prefetch for ODP MRs
https://git.kernel.org/rdma/rdma/c/3576b0df1588a0 [2/2] RDMA/rxe: Enable asynchronous prefetch for ODP MRs https://git.kernel.org/rdma/rdma/c/9284bc34c77399
Best regards,
Signature · 1 line
Leon Romanovsky <[email protected]>