MR cache enhancements
Subsystems (2)
v2 cover MR cache enhancements
From: Leon Romanovsky <[email protected]>
Changelog:
v2:
* Susbset of previously sent patches
v1: https://lore.kernel.org/all/[email protected]
* Based on DM revert https://lore.kernel.org/all/[email protected]
v0: https://lore.kernel.org/all/[email protected]
---------------------------------------------------------
Hi,
This series from Aharon cleans a little bit MR logic.
Thanks
Aharon Landau (5):
RDMA/mlx5: Remove redundant work in struct mlx5_cache_ent RDMA/mlx5: Fix the flow of a miss in the allocation of a cache ODP MR RDMA/mlx5: Merge similar flows of allocating MR from the cache RDMA/mlx5: Store ndescs instead of the translation table size RDMA/mlx5: Reorder calls to pcie_relaxed_ordering_enabled()
drivers/infiniband/hw/mlx5/mlx5_ib.h | 6 +- drivers/infiniband/hw/mlx5/mr.c | 104 ++++++++++----------------- drivers/infiniband/hw/mlx5/odp.c | 19 ++--- 3 files changed, 52 insertions(+), 77 deletions(-)
Signature · 1 line
2.35.1
PATCH v2 1/5 RDMA/mlx5: Remove redundant work in struct mlx5_cache_ent
From: Aharon Landau <[email protected]>
delayed_cache_work_func() and the cache_work_func() are both wrappers of __cache_work_func(). Instead of having a special not delayed work, use the delayed work with delay = 0.
--- drivers/infiniband/hw/mlx5/mlx5_ib.h | 1 - drivers/infiniband/hw/mlx5/mr.c | 16 +++------------- 2 files changed, 3 insertions(+), 14 deletions(-)
drivers/infiniband/hw/mlx5/mlx5_ib.h +0 -1
@@ -788,7 +788,6 @@ struct mlx5_cache_ent {u32 miss;struct mlx5_ib_dev *dev; -struct work_struct work;struct delayed_work dwork; };
drivers/infiniband/hw/mlx5/mr.c +3 -13
@@ -465,14 +465,14 @@ static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent)return ;if (ent->available_mrs < ent->limit) { ent->fill_to_high_water = true; -queue_work (ent->dev->cache.wq, &ent->work); +mod_delayed_work (ent->dev->cache.wq, &ent->dwork,0 ); }else if (ent->fill_to_high_water && ent->available_mrs + ent->pending <2 * ent->limit) {/* * Once we start populating due to hitting a low water mark * continue until we pass the high water mark. */ -queue_work (ent->dev->cache.wq, &ent->work); +mod_delayed_work (ent->dev->cache.wq, &ent->dwork,0 ); }else if (ent->available_mrs ==2 * ent->limit) { ent->fill_to_high_water = false; }else if (ent->available_mrs >2 * ent->limit) {
@@ -482,7 +482,7 @@ static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent)queue_delayed_work (ent->dev->cache.wq, &ent->dwork,msecs_to_jiffies (1000 )); else - queue_work(ent->dev->cache.wq, &ent->work); + mod_delayed_work(ent->dev->cache.wq, &ent->dwork, 0); } }
@@ -558,14 +558,6 @@ static void delayed_cache_work_func(struct work_struct *work)__cache_work_func (ent); } -static void cache_work_func (struct work_struct *work) -{ -struct mlx5_cache_ent *ent; - - ent = container_of(work, struct mlx5_cache_ent, work); -__cache_work_func (ent); -} -/* Allocate a special entry from the cache */ struct mlx5_ib_mr *mlx5_mr_cache_alloc (struct mlx5_ib_dev *dev,unsigned int entry,int access_flags)
@@ -726,7 +718,6 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev) ent->dev = dev; ent->limit =0 ; -INIT_WORK (&ent->work, cache_work_func);INIT_DELAYED_WORK (&ent->dwork, delayed_cache_work_func);if (i >MR_CACHE_LAST_STD_ENTRY ) {
@@ -770,7 +761,6 @@ int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)spin_lock_irq (&ent->lock); ent->disabled = true;spin_unlock_irq (&ent->lock); -cancel_work_sync (&ent->work);cancel_delayed_work_sync (&ent->dwork); }
Signature · 1 line
2.35.1
PATCH v2 2/5 RDMA/mlx5: Fix the flow of a miss in the allocation of a cache ODP MR
In reply to Leon RomanovskyFrom: Aharon Landau <[email protected]>
When an ODP MR cache entry is empty and trying to allocate it, increment the ent->miss counter and call to queue_adjust_cache_locked() to verify the entry is balanced.
--- drivers/infiniband/hw/mlx5/mr.c | 2 ++ 1 file changed, 2 insertions(+)
drivers/infiniband/hw/mlx5/mr.c +2 -0
@@ -577,6 +577,8 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, ent = &cache->ent[entry];spin_lock_irq (&ent->lock);if (list_empty (&ent->head)) { +queue_adjust_cache_locked (ent); + ent->miss++;spin_unlock_irq (&ent->lock); mr =create_cache_mr (ent);if (IS_ERR (mr))
Signature · 1 line
2.35.1
PATCH v2 3/5 RDMA/mlx5: Merge similar flows of allocating MR from the cache
In reply to Leon RomanovskyFrom: Aharon Landau <[email protected]>
When allocating a MR from the cache, the driver calls to get_cache_mr(), and in case of failure, retries with create_cache_mr(). This is the flow of mlx5_mr_cache_alloc(), so use it instead.
--- drivers/infiniband/hw/mlx5/mlx5_ib.h | 3 +- drivers/infiniband/hw/mlx5/mr.c | 47 +++------------------------- drivers/infiniband/hw/mlx5/odp.c | 11 +++++-- 3 files changed, 15 insertions(+), 46 deletions(-)
drivers/infiniband/hw/mlx5/mlx5_ib.h +2 -1
@@ -1343,7 +1343,8 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);int mlx5_mr_cache_cleanup (struct mlx5_ib_dev *dev);struct mlx5_ib_mr *mlx5_mr_cache_alloc (struct mlx5_ib_dev *dev, -unsigned int entry,int access_flags); +struct mlx5_cache_ent *ent, +int access_flags);int mlx5_ib_check_mr_status (struct ib_mr *ibmr,u32 check_mask,struct ib_mr_status *mr_status);
drivers/infiniband/hw/mlx5/mr.c +5 -42
76 lines not on this page · raw message
drivers/infiniband/hw/mlx5/odp.c +8 -3
@@ -407,6 +407,7 @@ static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev,static struct mlx5_ib_mr *implicit_get_child_mr (struct mlx5_ib_mr *imr,unsigned long idx) { +struct mlx5_ib_dev *dev =mr_to_mdev (imr);struct ib_umem_odp *odp;struct mlx5_ib_mr *mr;struct mlx5_ib_mr *ret;
@@ -418,13 +419,14 @@ static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr,if (IS_ERR (odp))return ERR_CAST (odp); - mr =mlx5_mr_cache_alloc ( -mr_to_mdev (imr),MLX5_IMR_MTT_CACHE_ENTRY , imr->access_flags); + mr =mlx5_mr_cache_alloc (dev, &dev->cache.ent[MLX5_IMR_MTT_CACHE_ENTRY ], + imr->access_flags);if (IS_ERR (mr)) {ib_umem_odp_release (odp);return mr; } + mr->access_flags = imr->access_flags; mr->ibmr.pd = imr->ibmr.pd; mr->ibmr.device = &mr_to_mdev (imr)->ib_dev; mr->umem = &odp->umem;
@@ -493,12 +495,15 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,if (IS_ERR (umem_odp))return ERR_CAST (umem_odp); - imr =mlx5_mr_cache_alloc (dev,MLX5_IMR_KSM_CACHE_ENTRY , access_flags); + imr =mlx5_mr_cache_alloc (dev, + &dev->cache.ent[MLX5_IMR_KSM_CACHE_ENTRY ], + access_flags);if (IS_ERR (imr)) {ib_umem_odp_release (umem_odp);return imr; } + imr->access_flags = access_flags; imr->ibmr.pd = &pd->ibpd; imr->ibmr.iova =0 ; imr->umem = &umem_odp->umem;
Signature · 1 line
2.35.1
PATCH v2 4/5 RDMA/mlx5: Store ndescs instead of the translation table size
In reply to Leon RomanovskyFrom: Aharon Landau <[email protected]>
Currently, ent->xlt stores the translation table size. This data should not be stored in the cache entry but be written directly to the mailbox. Store ndescs instead, and deduce the translation table size from it according to the access mode.
Later in the series, ent->ndescs will help to search for entries.
--- drivers/infiniband/hw/mlx5/mlx5_ib.h | 2 +- drivers/infiniband/hw/mlx5/mr.c | 25 ++++++++++++++++++++++--- drivers/infiniband/hw/mlx5/odp.c | 8 ++------ 3 files changed, 25 insertions(+), 10 deletions(-)
drivers/infiniband/hw/mlx5/mlx5_ib.h +1 -1
@@ -763,9 +763,9 @@ struct mlx5_cache_ent {char name[4 ];u32 order; -u32 xlt;u32 access_mode;u32 page; +unsigned int ndescs; u8 disabled:1; u8 fill_to_high_water:1;
drivers/infiniband/hw/mlx5/mr.c +22 -3
@@ -176,6 +176,25 @@ static void create_mkey_callback(int status, struct mlx5_async_work *context)spin_unlock_irqrestore (&ent->lock, flags); } +static int get_mkc_octo_size (unsigned int access_mode,unsigned int ndescs) +{ +int ret =0 ; + +switch (access_mode) { +case MLX5_MKC_ACCESS_MODE_MTT : + ret =DIV_ROUND_UP (ndescs,MLX5_IB_UMR_OCTOWORD / +sizeof (struct mlx5_mtt )); +break ; +case MLX5_MKC_ACCESS_MODE_KSM : + ret =DIV_ROUND_UP (ndescs,MLX5_IB_UMR_OCTOWORD / +sizeof (struct mlx5_klm )); +break ; +default : +WARN_ON (1 ); + } +return ret; +} +static struct mlx5_ib_mr *alloc_cache_mr (struct mlx5_cache_ent *ent,void *mkc) {struct mlx5_ib_mr *mr;
@@ -191,7 +210,8 @@ static struct mlx5_ib_mr *alloc_cache_mr(struct mlx5_cache_ent *ent, void *mkc)MLX5_SET (mkc, mkc, access_mode_1_0, ent->access_mode &0x3 );MLX5_SET (mkc, mkc, access_mode_4_2, (ent->access_mode >>2 ) &0x7 ); -MLX5_SET (mkc, mkc, translations_octword_size, ent->xlt); +MLX5_SET (mkc, mkc, translations_octword_size, +get_mkc_octo_size (ent->access_mode, ent->ndescs));MLX5_SET (mkc, mkc, log_page_size, ent->page);return mr; }
@@ -701,8 +721,7 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)continue ; ent->page =PAGE_SHIFT ; - ent->xlt = (1 << ent->order) *sizeof (struct mlx5_mtt ) / -MLX5_IB_UMR_OCTOWORD ; + ent->ndescs =1 << ent->order; ent->access_mode =MLX5_MKC_ACCESS_MODE_MTT ; if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) && !dev->is_rep && mlx5_core_is_pf(dev->mdev) &&
drivers/infiniband/hw/mlx5/odp.c +2 -6
@@ -1598,18 +1598,14 @@ void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent)switch (ent->order -2 ) {case MLX5_IMR_MTT_CACHE_ENTRY : ent->page =PAGE_SHIFT ; - ent->xlt =MLX5_IMR_MTT_ENTRIES * -sizeof (struct mlx5_mtt ) / -MLX5_IB_UMR_OCTOWORD ; + ent->ndescs =MLX5_IMR_MTT_ENTRIES ; ent->access_mode =MLX5_MKC_ACCESS_MODE_MTT ; ent->limit =0 ;break ;case MLX5_IMR_KSM_CACHE_ENTRY : ent->page =MLX5_KSM_PAGE_SHIFT ; - ent->xlt = mlx5_imr_ksm_entries * -sizeof (struct mlx5_klm ) / -MLX5_IB_UMR_OCTOWORD ; + ent->ndescs = mlx5_imr_ksm_entries; ent->access_mode =MLX5_MKC_ACCESS_MODE_KSM ; ent->limit =0 ;break ;
Signature · 1 line
2.35.1
PATCH v2 5/5 RDMA/mlx5: Reorder calls to pcie_relaxed_ordering_enabled()
In reply to Leon RomanovskyFrom: Aharon Landau <[email protected]>
The mkc is the key for the mkey cache, hence, created in each attempt to get a cache mkey, while pcie_relaxed_ordering_enabled() is called during the setting of the mkc, but used only for cases where IB_ACCESS_RELAXED_ORDERING is set.
pcie_relaxed_ordering_enabled() is an expensive call (26 us). Reorder the code so the driver will call it only when it is needed.
--- drivers/infiniband/hw/mlx5/mr.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)
drivers/infiniband/hw/mlx5/mr.c +7 -7
@@ -68,7 +68,6 @@ static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr, struct ib_pd *pd) {struct mlx5_ib_dev *dev =to_mdev (pd->device); -bool ro_pci_enabled =pcie_relaxed_ordering_enabled (dev->mdev->pdev);MLX5_SET (mkc, mkc, a, !!(acc &IB_ACCESS_REMOTE_ATOMIC ));MLX5_SET (mkc, mkc, rw, !!(acc &IB_ACCESS_REMOTE_WRITE ));
@@ -76,12 +75,13 @@ static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr,MLX5_SET (mkc, mkc, lw, !!(acc &IB_ACCESS_LOCAL_WRITE ));MLX5_SET (mkc, mkc, lr,1 ); -if (MLX5_CAP_GEN (dev->mdev, relaxed_ordering_write)) -MLX5_SET (mkc, mkc, relaxed_ordering_write, - (acc &IB_ACCESS_RELAXED_ORDERING ) && ro_pci_enabled); -if (MLX5_CAP_GEN (dev->mdev, relaxed_ordering_read)) -MLX5_SET (mkc, mkc, relaxed_ordering_read, - (acc &IB_ACCESS_RELAXED_ORDERING ) && ro_pci_enabled); +if ((acc &IB_ACCESS_RELAXED_ORDERING ) && +pcie_relaxed_ordering_enabled (dev->mdev->pdev)) { +if (MLX5_CAP_GEN (dev->mdev, relaxed_ordering_write)) +MLX5_SET (mkc, mkc, relaxed_ordering_write,1 ); +if (MLX5_CAP_GEN (dev->mdev, relaxed_ordering_read)) +MLX5_SET (mkc, mkc, relaxed_ordering_read,1 ); + }MLX5_SET (mkc, mkc, pd,to_mpd (pd)->pdn);MLX5_SET (mkc, mkc, qpn,0xffffff );
Signature · 1 line
2.35.1
Quoted 14 lines · Leon Romanovsky
From: Leon Romanovsky <[email protected]>
Changelog:
v2:
* Susbset of previously sent patches
v1: https://lore.kernel.org/all/[email protected]
* Based on DM revert https://lore.kernel.org/all/[email protected]
v0: https://lore.kernel.org/all/[email protected]---------------------------------------------------------
Hi,This series from Aharon cleans a little bit MR logic.
Quoted 8 lines
Thanks
Aharon Landau (5):
RDMA/mlx5: Remove redundant work in structmlx5_cache_entRDMA/mlx5: Fix the flow of a miss in the allocation of a cache ODP MR RDMA/mlx5: Merge similar flows of allocating MR from the cache RDMA/mlx5: Store ndescs instead of the translation table size RDMA/mlx5: Reorder calls topcie_relaxed_ordering_enabled()
Applied to for-next, thanks
Jason