{IB,net}/mlx5: Setup mkey variant before mr create command invocation
Subsystems (5)
PATCH 1/12 {IB,net}/mlx5: Setup mkey variant before mr create command invocation
In reply to a message not in the archiveFrom: Saeed Mahameed <[email protected]>
On reg_mr_callback() mlx5_ib is recalculating the mkey variant which is wrong and will lead to using a different key variant than the one submitted to firmware on create mkey command invocation.
To fix this, we store the mkey variant before invoking the firmware command and use it later on completion (reg_mr_callback).
--- drivers/infiniband/hw/mlx5/mr.c | 7 ++----- drivers/net/ethernet/mellanox/mlx5/core/mr.c | 3 ++- 2 files changed, 4 insertions(+), 6 deletions(-)
drivers/infiniband/hw/mlx5/mr.c +2 -5
@@ -87,7 +87,6 @@ static void reg_mr_callback(int status, struct mlx5_async_work *context)struct mlx5_mr_cache *cache = &dev->cache;int c =order2idx (dev, mr->order);struct mlx5_cache_ent *ent = &cache->ent[c]; -u8 key;unsigned long flags;spin_lock_irqsave (&ent->lock, flags);
@@ -102,10 +101,8 @@ static void reg_mr_callback(int status, struct mlx5_async_work *context) } mr->mmkey.type =MLX5_MKEY_MR ; -spin_lock_irqsave (&dev->mdev->priv.mkey_lock, flags); - key = dev->mdev->priv.mkey_key++; -spin_unlock_irqrestore (&dev->mdev->priv.mkey_lock, flags); - mr->mmkey.key =mlx5_idx_to_mkey (MLX5_GET (create_mkey_out, mr->out, mkey_index)) | key; + mr->mmkey.key |=mlx5_idx_to_mkey ( +MLX5_GET (create_mkey_out, mr->out, mkey_index)); cache->last_add = jiffies;
drivers/net/ethernet/mellanox/mlx5/core/mr.c +2 -1
@@ -56,6 +56,7 @@ int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,MLX5_SET (create_mkey_in, in, opcode,MLX5_CMD_OP_CREATE_MKEY );MLX5_SET (mkc, mkc, mkey_7_0, key); + mkey->key = key;if (callback) return mlx5_cmd_exec_cb(async_ctx, in, inlen, out, outlen,
@@ -68,7 +69,7 @@ int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev, mkey_index =MLX5_GET (create_mkey_out, lout, mkey_index); mkey->iova =MLX5_GET64 (mkc, mkc, start_addr); mkey->size =MLX5_GET64 (mkc, mkc, len); - mkey->key =mlx5_idx_to_mkey (mkey_index) | key; + mkey->key |=mlx5_idx_to_mkey (mkey_index); mkey->pd =MLX5_GET (mkc, mkc, pd); mlx5_core_dbg(dev, "out 0x%x, key 0x%x, mkey 0x%x\n",
Signature · 1 line
2.24.1
PATCH 2/12 {IB,net}/mlx5: Assign mkey variant in mlx5_ib only
In reply to a message not in the archiveFrom: Saeed Mahameed <[email protected]>
mkey variant is not required for mlx5_core use, move the mkey variant counter to mlx5_ib.
--- drivers/infiniband/hw/mlx5/main.c | 1 + drivers/infiniband/hw/mlx5/mlx5_ib.h | 5 ++ drivers/infiniband/hw/mlx5/mr.c | 58 +++++++++++++++---- .../net/ethernet/mellanox/mlx5/core/main.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/mr.c | 8 +-- include/linux/mlx5/driver.h | 4 -- 6 files changed, 55 insertions(+), 22 deletions(-)
drivers/infiniband/hw/mlx5/main.c +1 -0
@@ -6393,6 +6393,7 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)spin_lock_init (&dev->reset_flow_resource_lock);xa_init (&dev->odp_mkeys);xa_init (&dev->sig_mrs); +spin_lock_init (&dev->mkey_lock);spin_lock_init (&dev->dm.lock); dev->dm.dev = mdev;
drivers/infiniband/hw/mlx5/mlx5_ib.h +5 -0
@@ -999,6 +999,11 @@ struct mlx5_ib_dev {/* sync used page count stats */ struct mlx5_ib_resources devr; + +/* protect mkey key part */ +spinlock_t mkey_lock; +u8 mkey_key; +struct mlx5_mr_cache cache;struct timer_list delay_timer;/* Prevents soft lock on massive reg MRs */
drivers/infiniband/hw/mlx5/mr.c +48 -10
120 lines not on this page · raw message
drivers/net/ethernet/mellanox/mlx5/core/main.c +0 -1
@@ -1282,7 +1282,6 @@ static int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx)mutex_init (&priv->alloc_mutex);mutex_init (&priv->pgdir_mutex);INIT_LIST_HEAD (&priv->pgdir_list); -spin_lock_init (&priv->mkey_lock); priv->dbg_root =debugfs_create_dir (dev_name (dev->device), mlx5_debugfs_root);
drivers/net/ethernet/mellanox/mlx5/core/mr.c +1 -7
@@ -49,14 +49,7 @@ int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,int err;u8 key; -spin_lock_irq (&dev->priv.mkey_lock); - key = dev->priv.mkey_key++; -spin_unlock_irq (&dev->priv.mkey_lock); - mkc =MLX5_ADDR_OF (create_mkey_in, in, memory_key_mkey_entry); -MLX5_SET (create_mkey_in, in, opcode,MLX5_CMD_OP_CREATE_MKEY ); -MLX5_SET (mkc, mkc, mkey_7_0, key); - mkey->key = key;if (callback) return mlx5_cmd_exec_cb(async_ctx, in, inlen, out, outlen,
@@ -66,6 +59,7 @@ int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,if (err)return err; + mkc =MLX5_ADDR_OF (create_mkey_in, in, memory_key_mkey_entry); mkey_index =MLX5_GET (create_mkey_out, lout, mkey_index); mkey->iova =MLX5_GET64 (mkc, mkc, start_addr); mkey->size =MLX5_GET64 (mkc, mkc, len);
include/linux/mlx5/driver.h +0 -4
@@ -575,10 +575,6 @@ struct mlx5_priv {/* end: alloc staff */ struct dentry *dbg_root; -/* protect mkey key part */ -spinlock_t mkey_lock; -u8 mkey_key; -struct list_head dev_list;struct list_head ctx_list;spinlock_t ctx_lock;
Signature · 1 line
2.24.1
PATCH 3/12 IB/mlx5: Replace spinlock protected write with atomic var
In reply to a message not in the archiveFrom: Saeed Mahameed <[email protected]>
mkey variant calculation was spinlock protected to make it atomic, replace that with one atomic variable.
--- drivers/infiniband/hw/mlx5/main.c | 2 +- drivers/infiniband/hw/mlx5/mlx5_ib.h | 5 +---- drivers/infiniband/hw/mlx5/mr.c | 6 +----- 3 files changed, 3 insertions(+), 10 deletions(-)
drivers/infiniband/hw/mlx5/main.c +1 -1
@@ -6393,7 +6393,7 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)spin_lock_init (&dev->reset_flow_resource_lock);xa_init (&dev->odp_mkeys);xa_init (&dev->sig_mrs); -spin_lock_init (&dev->mkey_lock); +atomic_set (&dev->mkey_var,0 );spin_lock_init (&dev->dm.lock); dev->dm.dev = mdev;
drivers/infiniband/hw/mlx5/mlx5_ib.h +1 -4
@@ -1000,10 +1000,7 @@ struct mlx5_ib_dev {*/ struct mlx5_ib_resources devr; -/* protect mkey key part */ -spinlock_t mkey_lock; -u8 mkey_key; - +atomic_t mkey_var;struct mlx5_mr_cache cache;struct timer_list delay_timer;/* Prevents soft lock on massive reg MRs */
drivers/infiniband/hw/mlx5/mr.c +1 -5
@@ -54,12 +54,8 @@ static void assign_mkey_variant(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey, u32 *in) { + u8 key = atomic_inc_return(&dev->mkey_var);void *mkc; -u8 key; - -spin_lock_irq (&dev->mkey_lock); - key = dev->mkey_key++; -spin_unlock_irq (&dev->mkey_lock); mkc =MLX5_ADDR_OF (create_mkey_in, in, memory_key_mkey_entry);MLX5_SET (mkc, mkc, mkey_7_0, key);
Signature · 1 line
2.24.1
PATCH 5/12 RDMA/mlx5: Rename the tracking variables for the MR cache
In reply to a message not in the archiveFrom: Jason Gunthorpe <[email protected]>
The old names do not clearly indicate the intent.
--- drivers/infiniband/hw/mlx5/mlx5_ib.h | 19 +++++++--- drivers/infiniband/hw/mlx5/mr.c | 54 ++++++++++++++-------------- 2 files changed, 42 insertions(+), 31 deletions(-)
drivers/infiniband/hw/mlx5/mlx5_ib.h +15 -4
@@ -704,15 +704,26 @@ struct mlx5_cache_ent {u32 access_mode;u32 page; -u32 size; -u32 cur; +/* +* - available_mrs is the length of list head, ie the number of MRs +* available for immediate allocation. +* - total_mrs is available_mrs plus all in use MRs that could be +* returned to the cache. +* - limit is the low water mark for available_mrs, 2* limit is the +* upper water mark. +* - pending is the number of MRs currently being created +*/ +u32 total_mrs; +u32 available_mrs; +u32 limit; +u32 pending; + +/* Statistics */ u32 miss; -u32 limit;struct mlx5_ib_dev *dev;struct work_struct work;struct delayed_work dwork; -int pending;struct completion compl; };
drivers/infiniband/hw/mlx5/mr.c +27 -27
157 lines not on this page · raw message
Signature · 1 line
2.24.1
PATCH 6/12 RDMA/mlx5: Simplify how the MR cache bucket is located
In reply to a message not in the archiveFrom: Jason Gunthorpe <[email protected]>
There are many bad APIs here that are accepting a cache bucket index instead of a bucket pointer. Many of the callers already have a bucket pointer, so this results in a lot of confusing uses of order2idx().
Pass the struct mlx5_cache_ent into add_keys(), remove_keys(), and alloc_cached_mr().
Once the MR is in the cache, store the cache bucket pointer directly in the MR, replacing the 'bool allocated_from cache'.
In the end there is only one place that needs to form index from order, alloc_mr_from_cache(). Increase the safety of this function by disallowing it from accessing cache entries in the ODP special area.
--- drivers/infiniband/hw/mlx5/mlx5_ib.h | 7 +- drivers/infiniband/hw/mlx5/mr.c | 160 +++++++++++---------------- drivers/infiniband/hw/mlx5/odp.c | 2 +- 3 files changed, 71 insertions(+), 98 deletions(-)
drivers/infiniband/hw/mlx5/mlx5_ib.h +4 -3
@@ -622,8 +622,8 @@ struct mlx5_ib_mr {struct ib_umem *umem;struct mlx5_shared_mr_info *smr_info;struct list_head list; -int order; -bool allocated_from_cache; +unsigned int order; +struct mlx5_cache_ent *cache_ent;int npages;struct mlx5_ib_dev *dev;u32 out[MLX5_ST_SZ_DW (create_mkey_out)];
@@ -1281,7 +1281,8 @@ int mlx5_ib_get_cqe_size(struct ib_cq *ibcq);int mlx5_mr_cache_init (struct mlx5_ib_dev *dev);int mlx5_mr_cache_cleanup (struct mlx5_ib_dev *dev); -struct mlx5_ib_mr *mlx5_mr_cache_alloc (struct mlx5_ib_dev *dev,int entry); +struct mlx5_ib_mr *mlx5_mr_cache_alloc (struct mlx5_ib_dev *dev, +unsigned int entry);void mlx5_mr_cache_free (struct mlx5_ib_dev *dev,struct mlx5_ib_mr *mr);int mlx5_mr_cache_invalidate (struct mlx5_ib_mr *mr);
drivers/infiniband/hw/mlx5/mr.c +66 -94
394 lines not on this page · raw message
drivers/infiniband/hw/mlx5/odp.c +1 -1
@@ -197,7 +197,7 @@ static void dma_fence_odp_mr(struct mlx5_ib_mr *mr) odp->private =NULL ;mutex_unlock (&odp->umem_mutex); -if (!mr->allocated_from_cache) { +if (!mr->cache_ent) {mlx5_core_destroy_mkey (mr->dev->mdev, &mr->mmkey);WARN_ON (mr->descs); }
Signature · 1 line
2.24.1
PATCH 7/12 RDMA/mlx5: Always remove MRs from the cache before destroying them
In reply to a message not in the archiveFrom: Jason Gunthorpe <[email protected]>
The cache bucket tracks the total number of MRs that exists, both inside and outside of the cache. Removing a MR from the cache (by setting cache_ent to NULL) without updating total_mrs will cause the tracking to leak and be inflated.
Further fix the rereg_mr path to always destroy the MR. reg_create will always overwrite all the MR data in mlx5_ib_mr, so the MR must be completely destroyed, in all cases, before this function can be called. Detach the MR from the cache and unconditionally destroy it to avoid leaking HW mkeys.
--- drivers/infiniband/hw/mlx5/mr.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-)
drivers/infiniband/hw/mlx5/mr.c +13 -6
@@ -479,6 +479,16 @@ static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_cache_ent *req_ent)return mr; } +static void detach_mr_from_cache (struct mlx5_ib_mr *mr) +{ +struct mlx5_cache_ent *ent = mr->cache_ent; + + mr->cache_ent =NULL ; +spin_lock_irq (&ent->lock); + ent->total_mrs--; +spin_unlock_irq (&ent->lock); +} +void mlx5_mr_cache_free (struct mlx5_ib_dev *dev,struct mlx5_ib_mr *mr) {struct mlx5_cache_ent *ent = mr->cache_ent;
@@ -488,7 +498,7 @@ void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)return ;if (mlx5_mr_cache_invalidate (mr)) { - mr->cache_ent =NULL ; +detach_mr_from_cache (mr);destroy_mkey (dev, mr);if (ent->available_mrs < ent->limit)queue_work (dev->cache.wq, &ent->work);
@@ -1445,9 +1455,8 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,* UMR can't be used - MKey needs to be replaced. */ if (mr->cache_ent) - err =mlx5_mr_cache_invalidate (mr); -else - err =destroy_mkey (dev, mr); +detach_mr_from_cache (mr); + err =destroy_mkey (dev, mr);if (err) goto err;
@@ -1459,8 +1468,6 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, mr =to_mmr (ib_mr); goto err; } - - mr->cache_ent =NULL ; }else {/* * Send a UMR WQE
Signature · 1 line
2.24.1
PATCH 8/12 RDMA/mlx5: Fix MR cache size and limit debugfs
In reply to a message not in the archiveFrom: Jason Gunthorpe <[email protected]>
The size_write function is supposed to adjust the total_mr's to match the user's request, but lacks locking and safety checking.
total_mrs can only be adjusted by at most available_mrs. mrs already assigned to users cannot be revoked. Ensure that the user provides a target value within the range of available_mrs and within the high/low water mark.
limit_write has confusing and wrong sanity checking, and doesn't have the ability to deallocate on limit reduction.
Since both functions use the same algorithm to adjust the available_mrs, consolidate it into one function and write it correctly. Fix the locking and by holding the spinlock for all accesses to ent->X.
Always fail if the user provides a malformed string.
--- drivers/infiniband/hw/mlx5/mr.c | 152 ++++++++++++++++++-------------- 1 file changed, 88 insertions(+), 64 deletions(-)
drivers/infiniband/hw/mlx5/mr.c +88 -64
@@ -140,7 +140,7 @@ static void create_mkey_callback(int status, struct mlx5_async_work *context)complete (&ent->compl); } -static int add_keys (struct mlx5_cache_ent *ent,int num) +static int add_keys (struct mlx5_cache_ent *ent,unsigned int num) {int inlen =MLX5_ST_SZ_BYTES (create_mkey_in);struct mlx5_ib_mr *mr;
@@ -200,30 +200,54 @@ static int add_keys(struct mlx5_cache_ent *ent, int num)return err; } -static void remove_keys (struct mlx5_cache_ent *ent,int num) +static void remove_cache_mr (struct mlx5_cache_ent *ent) { -struct mlx5_ib_mr *tmp_mr;struct mlx5_ib_mr *mr; -LIST_HEAD (del_list); -int i; -for (i =0 ; i < num; i++) { -spin_lock_irq (&ent->lock); -if (list_empty (&ent->head)) { -spin_unlock_irq (&ent->lock); -break ; - } - mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); -list_move (&mr->list, &del_list); - ent->available_mrs--; - ent->total_mrs--; +spin_lock_irq (&ent->lock); +if (list_empty (&ent->head)) {spin_unlock_irq (&ent->lock); -mlx5_core_destroy_mkey (ent->dev->mdev, &mr->mmkey); +return ; } + mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); +list_del (&mr->list); + ent->available_mrs--; + ent->total_mrs--; +spin_unlock_irq (&ent->lock); +mlx5_core_destroy_mkey (ent->dev->mdev, &mr->mmkey); +kfree (mr); +} -list_for_each_entry_safe (mr, tmp_mr, &del_list, list) { -list_del (&mr->list); -kfree (mr); +static int resize_available_mrs (struct mlx5_cache_ent *ent,unsigned int target, +bool limit_fill) +{ +int err; + +lockdep_assert_held (&ent->lock); + +while (true) { +if (limit_fill) + target = ent->limit *2 ; +if (target == ent->available_mrs + ent->pending) +return 0 ; +if (target > ent->available_mrs + ent->pending) { +u32 todo = target - (ent->available_mrs + ent->pending); + +spin_unlock_irq (&ent->lock); + err =add_keys (ent, todo); +if (err == -EAGAIN ) +usleep_range (3000 ,5000 ); +spin_lock_irq (&ent->lock); +if (err) { +if (err != -EAGAIN ) +return err; + }else +return 0 ; + }else { +spin_unlock_irq (&ent->lock); +remove_cache_mr (ent); +spin_lock_irq (&ent->lock); + } } }
@@ -231,33 +255,38 @@ static ssize_t size_write(struct file *filp, const char __user *buf, size_t count, loff_t *pos) { struct mlx5_cache_ent *ent = filp->private_data; -char lbuf[20 ] = {0 }; -u32 var; +u32 target;int err; - count =min (count,sizeof (lbuf) -1 ); -if (copy_from_user (lbuf, buf, count)) -return -EFAULT ; - -if (sscanf (lbuf,"%u" , &var) !=1 ) -return -EINVAL ; - -if (var < ent->limit) -return -EINVAL ; - -if (var > ent->total_mrs) { -do { - err =add_keys (ent, var - ent->total_mrs); -if (err && err != -EAGAIN ) -return err; + err =kstrtou32_from_user (buf, count,0 , &target); +if (err) +return err; -usleep_range (3000 ,5000 ); - }while (err); - }else if (var < ent->total_mrs) { -remove_keys (ent, ent->total_mrs - var); +/* +* Target is the new value of total_mrs the user requests, however we +* cannot free MRs that are in use. Compute the target value for +* available_mrs. +*/ +spin_lock_irq (&ent->lock); +if (target < ent->total_mrs - ent->available_mrs) { + err = -EINVAL ; + goto err_unlock; + } + target = target - (ent->total_mrs - ent->available_mrs); +if (target < ent->limit || target > ent->limit*2 ) { + err = -EINVAL ; + goto err_unlock; } + err =resize_available_mrs (ent, target, false); +if (err) + goto err_unlock; +spin_unlock_irq (&ent->lock);return count; + +err_unlock: +spin_unlock_irq (&ent->lock); +return err; } static ssize_t size_read(struct file *filp, char __user *buf, size_t count,
@@ -285,28 +314,23 @@ static ssize_t limit_write(struct file *filp, const char __user *buf, size_t count, loff_t *pos) { struct mlx5_cache_ent *ent = filp->private_data; -char lbuf[20 ] = {0 };u32 var;int err; - count =min (count,sizeof (lbuf) -1 ); -if (copy_from_user (lbuf, buf, count)) -return -EFAULT ; - -if (sscanf (lbuf,"%u" , &var) !=1 ) -return -EINVAL ; - -if (var > ent->total_mrs) -return -EINVAL ; + err =kstrtou32_from_user (buf, count,0 , &var); +if (err) +return err; +/* +* Upon set we immediately fill the cache to high water mark implied by +* the limit. +*/ +spin_lock_irq (&ent->lock); ent->limit = var; - -if (ent->available_mrs < ent->limit) { - err =add_keys (ent,2 * ent->limit - ent->available_mrs); -if (err) -return err; - } - + err =resize_available_mrs (ent,0 , true); +spin_unlock_irq (&ent->lock); +if (err) +return err;return count; }
@@ -371,20 +395,20 @@ static void __cache_work_func(struct mlx5_cache_ent *ent) } }else if (ent->available_mrs >2 * ent->limit) {/* -* The remove_keys() logic is performed as garbage collection -* task. Such task is intended to be run when no other active -* processes are running. +* The remove_cache_mr() logic is performed as garbage +* collection task. Such task is intended to be run when no +* other active processes are running. * * The need_resched() will return TRUE if there are user tasks * to be activated in near future. * -* In such case, we don't execute remove_keys() and postpone -* the garbage collection work to try to run in next cycle, -* in order to free CPU resources to other tasks. +* In such case, we don't execute remove_cache_mr() and postpone +* the garbage collection work to try to run in next cycle, in +* order to free CPU resources to other tasks. */ if (!need_resched () && !someone_adding (cache) &&time_after (jiffies, cache->last_add +300 *HZ )) { -remove_keys (ent,1 ); +remove_cache_mr (ent);if (ent->available_mrs > ent->limit)queue_work (cache->wq, &ent->work); }else {
Signature · 1 line
2.24.1
PATCH 4/12 {IB,net}/mlx5: Move asynchronous mkey creation to mlx5_ib
In reply to a message not in the archiveFrom: Michael Guralnik <[email protected]>
As mlx5_ib is the only user of the mlx5_core_create_mkey_cb, move the logic inside mlx5_ib and cleanup the code in mlx5_core.
--- drivers/infiniband/hw/mlx5/mr.c | 6 +++--- drivers/net/ethernet/mellanox/mlx5/core/mr.c | 22 +++----------------- include/linux/mlx5/driver.h | 6 ------ 3 files changed, 6 insertions(+), 28 deletions(-)
drivers/infiniband/hw/mlx5/mr.c +3 -3
@@ -77,10 +77,10 @@ mlx5_ib_create_mkey_cb(struct mlx5_ib_dev *dev, u32 *in, int inlen, u32 *out, int outlen, struct mlx5_async_work *context) { +MLX5_SET (create_mkey_in ,in ,opcode ,MLX5_CMD_OP_CREATE_MKEY );assign_mkey_variant (dev, mkey, in); -return mlx5_core_create_mkey_cb (dev->mdev, mkey, async_ctx, - in, inlen, out, outlen, - create_mkey_callback, context); +return mlx5_cmd_exec_cb (async_ctx, in, inlen, out, outlen, + create_mkey_callback, context); }static void clean_mr (struct mlx5_ib_dev *dev,struct mlx5_ib_mr *mr);
drivers/net/ethernet/mellanox/mlx5/core/mr.c +3 -19
@@ -36,12 +36,9 @@#include <linux/mlx5/cmd.h> #include "mlx5_core.h" -int mlx5_core_create_mkey_cb (struct mlx5_core_dev *dev, -struct mlx5_core_mkey *mkey, -struct mlx5_async_ctx *async_ctx,u32 *in, -int inlen,u32 *out,int outlen, -mlx5_async_cbk_t callback, -struct mlx5_async_work *context) +int mlx5_core_create_mkey (struct mlx5_core_dev *dev, +struct mlx5_core_mkey *mkey, +u32 *in,int inlen) {u32 lout[MLX5_ST_SZ_DW (create_mkey_out)] = {0 };u32 mkey_index;
@@ -51,10 +48,6 @@ int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,MLX5_SET (create_mkey_in, in, opcode,MLX5_CMD_OP_CREATE_MKEY ); -if (callback) -return mlx5_cmd_exec_cb (async_ctx, in, inlen, out, outlen, - callback, context); - err =mlx5_cmd_exec (dev, in, inlen, lout,sizeof (lout));if (err)return err;
@@ -70,15 +63,6 @@ int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev, mkey_index, key, mkey->key);return 0 ; } -EXPORT_SYMBOL (mlx5_core_create_mkey_cb); - -int mlx5_core_create_mkey (struct mlx5_core_dev *dev, -struct mlx5_core_mkey *mkey, -u32 *in,int inlen) -{ -return mlx5_core_create_mkey_cb (dev, mkey,NULL , in, inlen, -NULL ,0 ,NULL ,NULL ); -}EXPORT_SYMBOL (mlx5_core_create_mkey); int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev,
include/linux/mlx5/driver.h +0 -6
@@ -943,12 +943,6 @@ struct mlx5_cmd_mailbox *mlx5_alloc_cmd_mailbox_chain(struct mlx5_core_dev *dev, gfp_t flags, int npages);void mlx5_free_cmd_mailbox_chain (struct mlx5_core_dev *dev,struct mlx5_cmd_mailbox *head); -int mlx5_core_create_mkey_cb (struct mlx5_core_dev *dev, -struct mlx5_core_mkey *mkey, -struct mlx5_async_ctx *async_ctx,u32 *in, -int inlen,u32 *out,int outlen, -mlx5_async_cbk_t callback, -struct mlx5_async_work *context);int mlx5_core_create_mkey (struct mlx5_core_dev *dev,struct mlx5_core_mkey *mkey,u32 *in,int inlen);
Signature · 1 line
2.24.1
PATCH 10/12 RDMA/mlx5: Fix locking in MR cache work queue
In reply to a message not in the archiveFrom: Jason Gunthorpe <[email protected]>
All of the members of mlx5_cache_ent must be accessed while holding the spinlock, add the missing spinlock in the __cache_work_func().
Using cache->stopped and flush_workqueue() is an inherently racy way to shutdown self-scheduling work on a queue. Replace it with ent->disabled under lock, and always check disabled before queuing any new work. Use cancel_work_sync() to shutdown the queue.
Use READ_ONCE/WRITE_ONCE for dev->last_add to manage concurrency as coherency is less important here.
Split fill_delay from the bitfield. C bitfield updates are not atomic and this is just a mess. Use READ_ONCE/WRITE_ONCE, but this could also use test_bit()/set_bit().
--- drivers/infiniband/hw/mlx5/mlx5_ib.h | 5 +- drivers/infiniband/hw/mlx5/mr.c | 121 +++++++++++++++++---------- 2 files changed, 80 insertions(+), 46 deletions(-)
drivers/infiniband/hw/mlx5/mlx5_ib.h +3 -2
@@ -704,6 +704,8 @@ struct mlx5_cache_ent {u32 access_mode;u32 page; + u8 disabled:1; +/* * - available_mrs is the length of list head, ie the number of MRs * available for immediate allocation.
@@ -730,7 +732,6 @@ struct mlx5_cache_ent {struct mlx5_mr_cache {struct workqueue_struct *wq;struct mlx5_cache_ent ent[MAX_MR_CACHE_ENTRIES ]; -int stopped;struct dentry *root;unsigned long last_add; };
@@ -1002,10 +1003,10 @@ struct mlx5_ib_dev {*/ struct mutex cap_mask_mutex; u8 ib_active:1; - u8 fill_delay:1; u8 is_rep:1; u8 lag_active:1; u8 wc_support:1; +u8 fill_delay;struct umr_common umrc;/* sync used page count stats */
drivers/infiniband/hw/mlx5/mr.c +77 -44
233 lines not on this page · raw message
Signature · 1 line
2.24.1
PATCH 11/12 RDMA/mlx5: Revise how the hysteresis scheme works for cache filling
In reply to a message not in the archiveFrom: Jason Gunthorpe <[email protected]>
Currently if the work queue is running then it is in 'hysteresis' mode and will fill until the cache reaches the high water mark. This implicit state is very tricky and doesn't interact with pending very well.
Instead of self re-scheduling the work queue after the add_keys() has started to create the new MR, have the queue scheduled from reg_mr_callback() only after the requested MR has been added.
This avoids the bad design of an in-rush of queue'd work doing back to back add_keys() until EAGAIN then sleeping. The add_keys() will be paced one at a time as they complete, slowly filling up the cache.
Also, fix pending to be only manipulated under lock.
--- drivers/infiniband/hw/mlx5/mlx5_ib.h | 1 + drivers/infiniband/hw/mlx5/mr.c | 41 ++++++++++++++++++---------- 2 files changed, 27 insertions(+), 15 deletions(-)
drivers/infiniband/hw/mlx5/mlx5_ib.h +1 -0
@@ -705,6 +705,7 @@ struct mlx5_cache_ent {u32 page; u8 disabled:1; + u8 fill_to_high_water:1;/* * - available_mrs is the length of list head, ie the number of MRs
drivers/infiniband/hw/mlx5/mr.c +26 -15
77 lines not on this page · raw message
Signature · 1 line
2.24.1
PATCH 12/12 RDMA/mlx5: Allow MRs to be created in the cache synchronously
In reply to a message not in the archiveFrom: Jason Gunthorpe <[email protected]>
If the cache is completely out of MRs, and we are running in cache mode, then directly, and synchronously, create an MR that is compatible with the cache bucket using a sleeping mailbox command. This ensures that the thread that is waiting for the MR absolutely will get one.
When a MR allocated in this way becomes freed then it is compatible with the cache bucket and will be recycled back into it.
Deletes the very buggy ent->compl scheme to create a synchronous MR allocation.
--- drivers/infiniband/hw/mlx5/mlx5_ib.h | 1 - drivers/infiniband/hw/mlx5/mr.c | 147 ++++++++++++++++----------- 2 files changed, 87 insertions(+), 61 deletions(-)
drivers/infiniband/hw/mlx5/mlx5_ib.h +0 -1
@@ -727,7 +727,6 @@ struct mlx5_cache_ent {struct mlx5_ib_dev *dev;struct work_struct work;struct delayed_work dwork; -struct completion compl; };struct mlx5_mr_cache {
drivers/infiniband/hw/mlx5/mr.c +87 -60
213 lines not on this page · raw message
Signature · 1 line
2.24.1
PATCH 9/12 RDMA/mlx5: Lock access to ent->available_mrs/limit when doing queue_work
In reply to a message not in the archiveFrom: Jason Gunthorpe <[email protected]>
Accesses to these members needs to be locked. There is no reason not to hold a spinlock while calling queue_work(), so move the tests into a helper and always call it under lock.
The helper should be called when available_mrs is adjusted.
--- drivers/infiniband/hw/mlx5/mr.c | 40 ++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 15 deletions(-)
drivers/infiniband/hw/mlx5/mr.c +25 -15
@@ -134,6 +134,10 @@ static void create_mkey_callback(int status, struct mlx5_async_work *context)list_add_tail (&mr->list, &ent->head); ent->available_mrs++; ent->total_mrs++; +/* +* Creating is always done in response to some demand, so do not call +* queue_adjust_cache_locked(). +*/ spin_unlock_irqrestore (&ent->lock, flags);if (!completion_done (&ent->compl))
@@ -367,6 +371,20 @@ static int someone_adding(struct mlx5_mr_cache *cache)return 0 ; } +/* +* Check if the bucket is outside the high/low water mark and schedule an async +* update. The cache refill has hysteresis, once the low water mark is hit it is +* refilled up to the high mark. +*/ +static void queue_adjust_cache_locked (struct mlx5_cache_ent *ent) +{ +lockdep_assert_held (&ent->lock); + +if (ent->available_mrs < ent->limit || + ent->available_mrs >2 * ent->limit) +queue_work (ent->dev->cache.wq, &ent->work); +} +static void __cache_work_func (struct mlx5_cache_ent *ent) {struct mlx5_ib_dev *dev = ent->dev;
@@ -462,9 +480,8 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, list);list_del (&mr->list); ent->available_mrs--; +queue_adjust_cache_locked (ent);spin_unlock_irq (&ent->lock); -if (ent->available_mrs < ent->limit) -queue_work (cache->wq, &ent->work);return mr; } }
@@ -487,14 +504,12 @@ static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_cache_ent *req_ent) list);list_del (&mr->list); ent->available_mrs--; +queue_adjust_cache_locked (ent);spin_unlock_irq (&ent->lock); -if (ent->available_mrs < ent->limit) -queue_work (dev->cache.wq, &ent->work);break ; } +queue_adjust_cache_locked (ent);spin_unlock_irq (&ent->lock); - -queue_work (dev->cache.wq, &ent->work); }if (!mr)
@@ -516,7 +531,6 @@ static void detach_mr_from_cache(struct mlx5_ib_mr *mr)void mlx5_mr_cache_free (struct mlx5_ib_dev *dev,struct mlx5_ib_mr *mr) {struct mlx5_cache_ent *ent = mr->cache_ent; -int shrink =0 ;if (!ent)return ;
@@ -524,20 +538,14 @@ void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)if (mlx5_mr_cache_invalidate (mr)) {detach_mr_from_cache (mr);destroy_mkey (dev, mr); -if (ent->available_mrs < ent->limit) -queue_work (dev->cache.wq, &ent->work);return ; }spin_lock_irq (&ent->lock);list_add_tail (&mr->list, &ent->head); ent->available_mrs++; -if (ent->available_mrs >2 * ent->limit) - shrink =1 ; +queue_adjust_cache_locked (ent);spin_unlock_irq (&ent->lock); - -if (shrink) -queue_work (dev->cache.wq, &ent->work); }static void clean_keys (struct mlx5_ib_dev *dev,int c)
@@ -653,7 +661,9 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev) ent->limit = dev->mdev->profile->mr_cache[i].limit; else ent->limit = 0; -queue_work (cache->wq, &ent->work); +spin_lock_irq (&ent->lock); +queue_adjust_cache_locked (ent); +spin_unlock_irq (&ent->lock); }mlx5_mr_cache_debugfs_init (dev);
Signature · 1 line
2.24.1
Re: [PATCH rdma-next v1 00/12] MR cache fixes and refactoring
In reply to a message not in the archive+ RDMA
Quoted 47 lines · Leon Romanovsky
From: Leon Romanovsky <[email protected]>
Changelog:
* v1: Added Saeed's patches.
* v0: https://lore.kernel.org/linux-rdma/[email protected]--------------------------------------------------------------------
Hi,This series fixes various corner cases in the
mlx5_ibMR cache implementation, see specific commit messages for more information.Thanks
Jason Gunthorpe (8):
RDMA/mlx5: Rename the tracking variables for the MR cache RDMA/mlx5: Simplify how the MR cache bucket is located RDMA/mlx5: Always remove MRs from the cache before destroying them RDMA/mlx5: Fix MR cache size and limit debugfs RDMA/mlx5: Lock access to ent->available_mrs/limit when doing
queue_work
RDMA/mlx5: Fix locking in MR cache work queue
RDMA/mlx5: Revise how the hysteresis scheme works for cache filling RDMA/mlx5: Allow MRs to be created in the cache synchronouslyMichael Guralnik (1):
{IB,net}/mlx5: Move asynchronous mkey creation tomlx5_ibSaeed Mahameed (3):
{IB,net}/mlx5: Setup mkey variant before mr create command invocation {IB,net}/mlx5: Assign mkey variant inmlx5_ibonly IB/mlx5: Replace spinlock protected write with atomic vardrivers/infiniband/hw/mlx5/main.c | 1 + drivers/infiniband/hw/mlx5/mlx5_ib.h | 35 +- drivers/infiniband/hw/mlx5/mr.c | 659 ++++++++++-------- drivers/infiniband/hw/mlx5/odp.c | 2 +- .../net/ethernet/mellanox/mlx5/core/main.c | 1 - drivers/net/ethernet/mellanox/mlx5/core/mr.c | 31 +- include/linux/mlx5/driver.h | 10 - 7 files changed, 419 insertions(+), 320 deletions(-)--
2.24.1
Re: [PATCH rdma-next v1 00/12] MR cache fixes and refactoring
Quoted 36 lines · Leon Romanovsky
+ RDMA
Quoted 33 lines · Leon Romanovsky
From: Leon Romanovsky <[email protected]>
Changelog:
* v1: Added Saeed's patches.
* v0: https://lore.kernel.org/linux-rdma/[email protected]Hi,
This series fixes various corner cases in the
mlx5_ibMR cache implementation, see specific commit messages for more information.Thanks
Jason Gunthorpe (8):
RDMA/mlx5: Rename the tracking variables for the MR cache RDMA/mlx5: Simplify how the MR cache bucket is located RDMA/mlx5: Always remove MRs from the cache before destroying them RDMA/mlx5: Fix MR cache size and limit debugfs RDMA/mlx5: Lock access to ent->available_mrs/limit when doing
queue_work
RDMA/mlx5: Fix locking in MR cache work queue
RDMA/mlx5: Revise how the hysteresis scheme works for cache filling RDMA/mlx5: Allow MRs to be created in the cache synchronouslyMichael Guralnik (1):
{IB,net}/mlx5: Move asynchronous mkey creation tomlx5_ibSaeed Mahameed (3):
{IB,net}/mlx5: Setup mkey variant before mr create command invocation {IB,net}/mlx5: Assign mkey variant inmlx5_ibonly IB/mlx5: Replace spinlock protected write with atomic var
These seem fine, can you update the shared branch please
Thanks,
Jason
Re: [PATCH rdma-next v1 00/12] MR cache fixes and refactoring
Quoted 33 lines · Jason Gunthorpe
Quoted 32 lines · Leon Romanovsky
+ RDMA
Quoted 29 lines · Leon Romanovsky
From: Leon Romanovsky <[email protected]>
Changelog:
* v1: Added Saeed's patches.
* v0: https://lore.kernel.org/linux-rdma/[email protected]Hi,
This series fixes various corner cases in the
mlx5_ibMR cache implementation, see specific commit messages for more information.Thanks
Jason Gunthorpe (8):
RDMA/mlx5: Rename the tracking variables for the MR cache RDMA/mlx5: Simplify how the MR cache bucket is located RDMA/mlx5: Always remove MRs from the cache before destroying them RDMA/mlx5: Fix MR cache size and limit debugfs RDMA/mlx5: Lock access to ent->available_mrs/limit when doing
queue_work
RDMA/mlx5: Fix locking in MR cache work queue
RDMA/mlx5: Revise how the hysteresis scheme works for cache filling RDMA/mlx5: Allow MRs to be created in the cache synchronouslyMichael Guralnik (1):
{IB,net}/mlx5: Move asynchronous mkey creation tomlx5_ib
Quoted 6 lines
Quoted 4 lines
Quoted 4 lines
Saeed Mahameed (3):
{IB,net}/mlx5: Setup mkey variant before mr create command invocation {IB,net}/mlx5: Assign mkey variant inmlx5_ibonly IB/mlx5: Replace spinlock protected write with atomic varThese seem fine, can you update the shared branch please
Thanks, applied to mlx5-nexta3cfdd392811 {IB,net}/mlx5: Move asynchronous mkey creation to mlx5_ib fc6a9f86f08a {IB,net}/mlx5: Assign mkey variant in mlx5_ib only 54c62e13ad76 {IB,net}/mlx5: Setup mkey variant before mr create command invocation
Quoted 3 lines
Thanks,
Jason
Jason Gunthorpe
AppliedRe: [PATCH rdma-next v1 00/12] MR cache fixes and refactoring
Quoted 39 lines · Leon Romanovsky
Quoted 38 lines · Jason Gunthorpe
Quoted 36 lines · Leon Romanovsky
+ RDMA
Quoted 33 lines · Leon Romanovsky
From: Leon Romanovsky <[email protected]>
Changelog:
* v1: Added Saeed's patches.
* v0: https://lore.kernel.org/linux-rdma/[email protected]Hi,
This series fixes various corner cases in the
mlx5_ibMR cache implementation, see specific commit messages for more information.Thanks
Jason Gunthorpe (8):
RDMA/mlx5: Rename the tracking variables for the MR cache RDMA/mlx5: Simplify how the MR cache bucket is located RDMA/mlx5: Always remove MRs from the cache before destroying them RDMA/mlx5: Fix MR cache size and limit debugfs RDMA/mlx5: Lock access to ent->available_mrs/limit when doing
queue_work
RDMA/mlx5: Fix locking in MR cache work queue
RDMA/mlx5: Revise how the hysteresis scheme works for cache filling RDMA/mlx5: Allow MRs to be created in the cache synchronouslyMichael Guralnik (1):
{IB,net}/mlx5: Move asynchronous mkey creation tomlx5_ibSaeed Mahameed (3):
{IB,net}/mlx5: Setup mkey variant before mr create command invocation {IB,net}/mlx5: Assign mkey variant inmlx5_ibonly IB/mlx5: Replace spinlock protected write with atomic var
Quoted 6 lines
Quoted 1 line
These seem fine, can you update the shared branch please
Thanks, applied to mlx5-next
a3cfdd392811{IB,net}/mlx5: Move asynchronous mkey creation tomlx5_ibfc6a9f86f08a{IB,net}/mlx5: Assign mkey variant inmlx5_ibonly54c62e13ad76{IB,net}/mlx5: Setup mkey variant before mr create command invocation
Applied to for-next
Thanks,
Jason