Clean the mlx5 MR deregistration logic
Subsystems (2)
cover Clean the mlx5 MR deregistration logic
From: Leon Romanovsky <[email protected]>
Hi,
The following patchset is a cleanup of mlx5_ib_mr dereg logic.
Thanks
Jason Gunthorpe (4):
RDMA/mlx5: Zero out ODP related items in the mlx5_ib_mr RDMA/mlx5: Use a union inside mlx5_ib_mr RDMA/mlx5: Consolidate MR destruction to mlx5_ib_dereg_mr() RDMA/mlx5: Rename mlx5_mr_cache_invalidate() to revoke_mr()
drivers/infiniband/core/umem_dmabuf.c | 4 + drivers/infiniband/hw/mlx5/mlx5_ib.h | 106 +++++++++-------- drivers/infiniband/hw/mlx5/mr.c | 157 ++++++++++++-------------- drivers/infiniband/hw/mlx5/odp.c | 152 ++++--------------------- 4 files changed, 160 insertions(+), 259 deletions(-)
--
2.29.2
PATCH 1/4 RDMA/mlx5: Zero out ODP related items in the mlx5_ib_mr
From: Jason Gunthorpe <[email protected]>
All of the ODP code assumes when it calls mlx5_mr_cache_alloc() the ODP related fields are zero'd. This is true if the MR was just allocated, but if the MR is recycled through the cache then the values are never zero'd.
This causes a bug in the odp_stats, they don't reset when the MR is reallocated, also is_odp_implicit is never 0'd.
So we can use memset on a block of the mlx5_ib_mr reorganize the structure to put all the data that can be zero'd by the cache at the end.
It is organized as an anonymous struct because the next patch will make this a union.
Delete the unused smr_info. Don't set the kernel only desc_size on the user path. No longer any need to zero mr->parent before freeing it, the memset() will get it now.
--- drivers/infiniband/hw/mlx5/mlx5_ib.h | 103 ++++++++++++++++----------- drivers/infiniband/hw/mlx5/mr.c | 14 ++-- drivers/infiniband/hw/mlx5/odp.c | 1 - 3 files changed, 66 insertions(+), 52 deletions(-)
drivers/infiniband/hw/mlx5/mlx5_ib.h +60 -43
@@ -547,11 +547,6 @@ static inline const struct mlx5_umr_wr *umr_wr(const struct ib_send_wr *wr) return container_of(wr, struct mlx5_umr_wr, wr); } -struct mlx5_shared_mr_info { -int mr_id; -struct ib_umem *umem; -}; -enum mlx5_ib_cq_pr_flags {MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD =1 <<0 , };
@@ -654,47 +649,69 @@ struct mlx5_ib_dm { atomic64_add(value, &((mr)->odp_stats.counter_name))struct mlx5_ib_mr { -struct ib_mr ibmr; -void *descs; -dma_addr_t desc_map; -int ndescs; -int data_length; -int meta_ndescs; -int meta_length; -int max_descs; -int desc_size; -int access_mode; -unsigned int page_shift; -struct mlx5_core_mkey mmkey; -struct ib_umem *umem; -struct mlx5_shared_mr_info *smr_info; -struct list_head list; -struct mlx5_cache_ent *cache_ent; -u32 out[MLX5_ST_SZ_DW (create_mkey_out)]; -struct mlx5_core_sig_ctx *sig; -void *descs_alloc; -int access_flags;/* Needed for rereg MR */ - -struct mlx5_ib_mr *parent; -/* Needed for IB_MR_TYPE_INTEGRITY */ -struct mlx5_ib_mr *pi_mr; -struct mlx5_ib_mr *klm_mr; -struct mlx5_ib_mr *mtt_mr; -u64 data_iova; -u64 pi_iova; - -/* For ODP and implicit */ -struct xarray implicit_children; -union { -struct list_head elm; -struct work_struct work; - } odp_destroy; -struct ib_odp_counters odp_stats; -bool is_odp_implicit; +struct ib_mr ibmr; +struct mlx5_core_mkey mmkey; -struct mlx5_async_work cb_work; +/* User MR data */ +struct mlx5_cache_ent *cache_ent; +struct ib_umem *umem; + +/* This is zero'd when the MR is allocated */ +struct { +/* Used only while the MR is in the cache */ +struct { +u32 out[MLX5_ST_SZ_DW (create_mkey_out)]; +struct mlx5_async_work cb_work; +/* Cache list element */ +struct list_head list; + }; + +/* Used only by kernel MRs (umem == NULL) */ +struct { +void *descs; +void *descs_alloc; +dma_addr_t desc_map; +int max_descs; +int ndescs; +int desc_size; +int access_mode; + +/* For Kernel IB_MR_TYPE_INTEGRITY */ +struct mlx5_core_sig_ctx *sig; +struct mlx5_ib_mr *pi_mr; +struct mlx5_ib_mr *klm_mr; +struct mlx5_ib_mr *mtt_mr; +u64 data_iova; +u64 pi_iova; +int meta_ndescs; +int meta_length; +int data_length; + }; + +/* Used only by User MRs (umem != NULL) */ +struct { +unsigned int page_shift; +/* Current access_flags */ +int access_flags; + +/* For User ODP */ +struct mlx5_ib_mr *parent; +struct xarray implicit_children; +union { +struct work_struct work; + } odp_destroy; +struct ib_odp_counters odp_stats; +bool is_odp_implicit; + }; + }; }; +/* Zero the fields in the mr that are variant depending on usage */ +static inline void mlx5_clear_mr (struct mlx5_ib_mr *mr) +{ +memset (mr->out,0 ,sizeof (*mr) - offsetof(struct mlx5_ib_mr , out)); +} +static inline bool is_odp_mr (struct mlx5_ib_mr *mr) { return IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && mr->umem &&
drivers/infiniband/hw/mlx5/mr.c +6 -8
@@ -590,6 +590,8 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, ent->available_mrs--;queue_adjust_cache_locked (ent);spin_unlock_irq (&ent->lock); + +mlx5_clear_mr (mr); } mr->access_flags = access_flags;return mr;
@@ -615,16 +617,14 @@ static struct mlx5_ib_mr *get_cache_mr(struct mlx5_cache_ent *req_ent) ent->available_mrs--;queue_adjust_cache_locked (ent);spin_unlock_irq (&ent->lock); -break ; +mlx5_clear_mr (mr); +return mr; }queue_adjust_cache_locked (ent);spin_unlock_irq (&ent->lock); } - -if (!mr) - req_ent->miss++; - -return mr; + req_ent->miss++; +return NULL ; }static void detach_mr_from_cache (struct mlx5_ib_mr *mr)
@@ -993,8 +993,6 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd, mr->ibmr.pd = pd; mr->umem = umem; - mr->access_flags = access_flags; - mr->desc_size =sizeof (struct mlx5_mtt ); mr->mmkey.iova = iova; mr->mmkey.size = umem->length; mr->mmkey.pd =to_mpd (pd)->pdn;
drivers/infiniband/hw/mlx5/odp.c +0 -1
@@ -227,7 +227,6 @@ static void free_implicit_child_mr(struct mlx5_ib_mr *mr, bool need_imr_xlt)dma_fence_odp_mr (mr); - mr->parent =NULL ;mlx5_mr_cache_free (mr_to_mdev (mr), mr);ib_umem_odp_release (odp); }
--
2.29.2
PATCH 4/4 RDMA/mlx5: Rename mlx5_mr_cache_invalidate() to revoke_mr()
In reply to Leon RomanovskyFrom: Jason Gunthorpe <[email protected]>
Now that this is only used in a few places in mr.c give it a sensible name. It has nothing to do with the cache and can be invoked on any MR. DMA is stopped and the user cannot touch the MR any further once it completes.
--- drivers/infiniband/hw/mlx5/mr.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-)
drivers/infiniband/hw/mlx5/mr.c +6 -6
@@ -1640,14 +1640,14 @@ struct ib_mr *mlx5_ib_reg_user_mr_dmabuf(struct ib_pd *pd, u64 offset, }/** -* mlx5_mr_cache_invalidate - Fence all DMA on the MR +* revoke_mr - Fence all DMA on the MR * @mr: The MR to fence * * Upon return the NIC will not be doing any DMA to the pages under the MR, -* and any DMA inprogress will be completed. Failure of this function +* and any DMA in progress will be completed. Failure of this function * indicates the HW has failed catastrophically. */ -static int mlx5_mr_cache_invalidate (struct mlx5_ib_mr *mr) +static int revoke_mr (struct mlx5_ib_mr *mr) {struct mlx5_umr_wr umrwr = {};
@@ -1741,7 +1741,7 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd,* with it. This ensure the change is atomic relative to any use of the * MR. */ - err =mlx5_mr_cache_invalidate (mr); + err =revoke_mr (mr);if (err)return err;
@@ -1820,7 +1820,7 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,* Only one active MR can refer to a umem at one time, revoke * the old MR before assigning the umem to the new one. */ - err =mlx5_mr_cache_invalidate (mr); + err =revoke_mr (mr);if (err)return ERR_PTR (err); umem = mr->umem;
@@ -1965,7 +1965,7 @@ int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)/* Stop DMA */ if (mr->cache_ent) { -if (mlx5_mr_cache_invalidate (mr)) { +if (revoke_mr (mr)) {spin_lock_irq (&mr->cache_ent->lock); mr->cache_ent->total_mrs--;spin_unlock_irq (&mr->cache_ent->lock);
Signature · 1 line
2.29.2
PATCH 3/4 RDMA/mlx5: Consolidate MR destruction to mlx5_ib_dereg_mr()
In reply to Leon RomanovskyFrom: Jason Gunthorpe <[email protected]>
Now that the SRCU stuff has been removed the entire MR destroy logic can be made a lot simpler. Currently there are many different ways to destroy a MR and it makes it really hard to do this task correctly. Route all destruction through mlx5_ib_dereg_mr() and make it work for all situations.
Since it turns out all the different MR types do basically the same thing this removes a lot of knowledge of MR internals from ODP and leaves ODP just exporting an operation to clean up children.
This fixes a few weird corner cases bugs and firmly uses the correct ordering of the MR destruction:
- Stop parallel access to the mkey via the ODP xarray
- Stop DMA
- Release the umem
- Clean up ODP children
- Free/Recycle the MR
--- drivers/infiniband/core/umem_dmabuf.c | 4 + drivers/infiniband/hw/mlx5/mlx5_ib.h | 5 +- drivers/infiniband/hw/mlx5/mr.c | 133 +++++++++++------------ drivers/infiniband/hw/mlx5/odp.c | 151 ++++---------------------- 4 files changed, 90 insertions(+), 203 deletions(-)
drivers/infiniband/core/umem_dmabuf.c +4 -0
@@ -168,6 +168,10 @@ void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf) {struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf; +dma_resv_lock (dmabuf->resv,NULL ); +ib_umem_dmabuf_unmap_pages (umem_dmabuf); +dma_resv_unlock (dmabuf->resv); +dma_buf_detach (dmabuf, umem_dmabuf->attach);dma_buf_put (dmabuf);kfree (umem_dmabuf);
drivers/infiniband/hw/mlx5/mlx5_ib.h +1 -4
@@ -1285,8 +1285,7 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,struct ib_udata *udata, int access_flags);void mlx5_ib_free_implicit_mr (struct mlx5_ib_mr *mr); -void mlx5_ib_fence_odp_mr (struct mlx5_ib_mr *mr); -void mlx5_ib_fence_dmabuf_mr (struct mlx5_ib_mr *mr); +void mlx5_ib_free_odp_mr (struct mlx5_ib_mr *mr);struct ib_mr *mlx5_ib_rereg_user_mr (struct ib_mr *ib_mr,int flags,u64 start,u64 length,u64 virt_addr,int access_flags,struct ib_pd *pd,struct ib_udata *udata);
@@ -1334,8 +1333,6 @@ int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);struct mlx5_ib_mr *mlx5_mr_cache_alloc (struct mlx5_ib_dev *dev,unsigned int entry,int access_flags); -void mlx5_mr_cache_free (struct mlx5_ib_dev *dev,struct mlx5_ib_mr *mr); -int mlx5_mr_cache_invalidate (struct mlx5_ib_mr *mr);int mlx5_ib_check_mr_status (struct ib_mr *ibmr,u32 check_mask,struct ib_mr_status *mr_status);
drivers/infiniband/hw/mlx5/mr.c +62 -71
207 lines not on this page · raw message
drivers/infiniband/hw/mlx5/odp.c +23 -128
204 lines not on this page · raw message
Signature · 1 line
2.29.2
PATCH 2/4 RDMA/mlx5: Use a union inside mlx5_ib_mr
In reply to Leon RomanovskyFrom: Jason Gunthorpe <[email protected]>
The struct mlx5_ib_mr can be used for three different things, but only one at a time:
- In the user MR cache
- As a kernel MR
- As a user MR
Overlay the three things into a single union with the following rules:
- If the mr is found on the cache_ent->head list then it is a cache MR and umem == NULL. The entire union is zero after the MR is removed from the cache.
- If umem != NULL or type == IB_MR_TYPE_USER then it is a user MR.
- If umem == NULL then it is a kernel MR
This reduces the size of struct mlx5_ib_mr to 552 bytes from 702.
The only place the three flows overlap in the code is during dereg, so add a few extra checks along there.
--- drivers/infiniband/hw/mlx5/mlx5_ib.h | 2 +- drivers/infiniband/hw/mlx5/mr.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-)
drivers/infiniband/hw/mlx5/mlx5_ib.h +1 -1
@@ -657,7 +657,7 @@ struct mlx5_ib_mr {struct ib_umem *umem;/* This is zero'd when the MR is allocated */ - struct { + union { /* Used only while the MR is in the cache */ struct { u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
drivers/infiniband/hw/mlx5/mr.c +2 -2
@@ -1929,7 +1929,7 @@ mlx5_alloc_priv_descs(struct ib_device *device,static void mlx5_free_priv_descs (struct mlx5_ib_mr *mr) { -if (mr->descs) { +if (!mr->umem && mr->descs) {struct ib_device *device = mr->ibmr.device;int size = mr->max_descs * mr->desc_size;struct mlx5_ib_dev *dev =to_mdev (device);
@@ -1943,7 +1943,7 @@ mlx5_free_priv_descs(struct mlx5_ib_mr *mr)static void clean_mr (struct mlx5_ib_dev *dev,struct mlx5_ib_mr *mr) { -if (mr->sig) { +if (mr->ibmr.type ==IB_MR_TYPE_INTEGRITY ) {if (mlx5_core_destroy_psv (dev->mdev, mr->sig->psv_memory.psv_idx)) mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
Signature · 1 line
2.29.2
Quoted 6 lines · Leon Romanovsky
From: Leon Romanovsky <[email protected]>
Hi,
The following patchset is a cleanup of
mlx5_ib_mrdereg logic.
Quoted 7 lines
Thanks
Jason Gunthorpe (4):
RDMA/mlx5: Zero out ODP related items in themlx5_ib_mrRDMA/mlx5: Use a union insidemlx5_ib_mrRDMA/mlx5: Consolidate MR destruction tomlx5_ib_dereg_mr()RDMA/mlx5: Renamemlx5_mr_cache_invalidate()torevoke_mr()
Applied to for-next
Jason