Use ib_umem_find_best_pgsz() when creating MRs
Subsystems (2)
cover Use ib_umem_find_best_pgsz() when creating MRs
From: Leon Romanovsky <[email protected]>
From Jason:
The new common code does a better job finding large page sizes. Use it in mlx5 for MRs.
This requires moving the MTT population for mailboxes and UMR over to rdma_for_each_dma_block().
Thanks
Jason Gunthorpe (5):
RDMA/mlx5: Change mlx5_ib_populate_pas() to use rdma_for_each_block() RDMA/mlx5: Move xlt_emergency_page_mutex into mr.c RDMA/mlx5: Split the WR setup out of mlx5_ib_update_xlt() RDMA/mlx5: Split mlx5_ib_update_xlt() into ODP and non-ODP cases RDMA/mlx5: Use ib_umem_find_best_pgsz() for mkc's
drivers/infiniband/core/umem.c | 9 + drivers/infiniband/hw/mlx5/cq.c | 6 +- drivers/infiniband/hw/mlx5/devx.c | 4 +- drivers/infiniband/hw/mlx5/main.c | 26 +- drivers/infiniband/hw/mlx5/mem.c | 73 +----- drivers/infiniband/hw/mlx5/mlx5_ib.h | 37 ++- drivers/infiniband/hw/mlx5/mr.c | 364 ++++++++++++++++++--------- drivers/infiniband/hw/mlx5/qp.c | 6 +- drivers/infiniband/hw/mlx5/srq.c | 2 +- 9 files changed, 312 insertions(+), 215 deletions(-)
--
2.26.2
PATCH 1/5 RDMA/mlx5: Change mlx5_ib_populate_pas() to use rdma_for_each_block()
From: Jason Gunthorpe <[email protected]>
This routine converts the umem SGL into a list of fixed pages for DMA, which is exactly what rdma_umem_for_each_dma_block() is for, use the common code directly.
--- drivers/infiniband/hw/mlx5/cq.c | 6 +++--- drivers/infiniband/hw/mlx5/devx.c | 4 ++-- drivers/infiniband/hw/mlx5/mem.c | 19 ++++++++++++++----- drivers/infiniband/hw/mlx5/mlx5_ib.h | 4 ++-- drivers/infiniband/hw/mlx5/mr.c | 7 +++++-- drivers/infiniband/hw/mlx5/qp.c | 6 +++--- drivers/infiniband/hw/mlx5/srq.c | 2 +- 7 files changed, 30 insertions(+), 18 deletions(-)
drivers/infiniband/hw/mlx5/cq.c +3 -3
@@ -762,7 +762,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, } pas = (__be64 *)MLX5_ADDR_OF (create_cq_in, *cqb, pas); -mlx5_ib_populate_pas (dev, cq->buf.umem, page_shift, pas,0 ); +mlx5_ib_populate_pas (cq->buf.umem,1UL << page_shift, pas,0 ); cqc =MLX5_ADDR_OF (create_cq_in, *cqb, cq_context); MLX5_SET(cqc, cqc, log_page_size,
@@ -1305,8 +1305,8 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) pas = (__be64 *)MLX5_ADDR_OF (modify_cq_in, in, pas);if (udata) -mlx5_ib_populate_pas (dev, cq->resize_umem, page_shift, - pas,0 ); +mlx5_ib_populate_pas (cq->resize_umem,1UL << page_shift, pas, +0 );else mlx5_fill_page_frag_array (&cq->resize_buf->frag_buf, pas);
drivers/infiniband/hw/mlx5/devx.c +2 -2
@@ -2115,9 +2115,9 @@ static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev,MLX5_SET (umem, umem, log_page_size, obj->page_shift -MLX5_ADAPTER_PAGE_SHIFT );MLX5_SET (umem, umem, page_offset, obj->page_offset); -mlx5_ib_populate_pas (dev, obj->umem, obj->page_shift, mtt, +mlx5_ib_populate_pas (obj->umem,1UL << obj->page_shift, mtt, (obj->umem->writable ?MLX5_IB_MTT_WRITE :0 ) | -MLX5_IB_MTT_READ ); +MLX5_IB_MTT_READ ); } static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
drivers/infiniband/hw/mlx5/mem.c +14 -5
@@ -155,13 +155,22 @@ void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, } } -void mlx5_ib_populate_pas (struct mlx5_ib_dev *dev,struct ib_umem *umem, -int page_shift,__be64 *pas,int access_flags) +/* +* Fill in a physical address list. ib_umem_num_dma_blocks() entries will be +* filled in the pas array. +*/ +void mlx5_ib_populate_pas (struct ib_umem *umem,size_t page_size,__be64 *pas, +u64 access_flags) { -return __mlx5_ib_populate_pas (dev, umem, page_shift,0 , -ib_umem_num_dma_blocks (umem,PAGE_SIZE ), - pas, access_flags); +struct ib_block_iter biter; + + rdma_umem_for_each_dma_block (umem, &biter, page_size) { + *pas =cpu_to_be64 (rdma_block_iter_dma_address (&biter) | + access_flags); + pas++; + } } +int mlx5_ib_get_buf_offset (u64 addr,int page_shift,u32 *offset) {u64 page_size;
drivers/infiniband/hw/mlx5/mlx5_ib.h +2 -2
@@ -1235,8 +1235,8 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,void __mlx5_ib_populate_pas (struct mlx5_ib_dev *dev,struct ib_umem *umem,int page_shift,size_t offset,size_t num_pages,__be64 *pas,int access_flags); -void mlx5_ib_populate_pas (struct mlx5_ib_dev *dev,struct ib_umem *umem, -int page_shift,__be64 *pas,int access_flags); +void mlx5_ib_populate_pas (struct ib_umem *umem,size_t page_size,__be64 *pas, +u64 access_flags);void mlx5_ib_copy_pas (u64 *old,u64 *new,int step,int num);int mlx5_ib_get_cqe_size (struct ib_cq *ibcq);int mlx5_mr_cache_init (struct mlx5_ib_dev *dev);
drivers/infiniband/hw/mlx5/mr.c +5 -2
@@ -1167,7 +1167,10 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, inlen =MLX5_ST_SZ_BYTES (create_mkey_in);if (populate) - inlen +=sizeof (*pas) *roundup (ib_umem_num_pages (umem),2 ); + inlen += +sizeof (*pas) * +roundup (ib_umem_num_dma_blocks (umem,1UL << page_shift), +2 ); in =kvzalloc (inlen,GFP_KERNEL );if (!in) { err = -ENOMEM ;
@@ -1179,7 +1182,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, err = -EINVAL ; goto err_2; } -mlx5_ib_populate_pas (dev, umem, page_shift, pas, +mlx5_ib_populate_pas (umem,1ULL << page_shift, pas, pg_cap ?MLX5_IB_MTT_PRESENT :0 ); }
drivers/infiniband/hw/mlx5/qp.c +3 -3
@@ -971,7 +971,7 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,MLX5_SET (create_qp_in, *in, uid, uid); pas = (__be64 *)MLX5_ADDR_OF (create_qp_in, *in, pas);if (ubuffer->umem) -mlx5_ib_populate_pas (dev, ubuffer->umem, page_shift, pas,0 ); +mlx5_ib_populate_pas (ubuffer->umem,1UL << page_shift, pas,0 ); qpc =MLX5_ADDR_OF (create_qp_in, *in, qpc);
@@ -1251,7 +1251,7 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,MLX5_SET (wq, wq, page_offset, offset); pas = (__be64 *)MLX5_ADDR_OF (wq, wq, pas); -mlx5_ib_populate_pas (dev, sq->ubuffer.umem, page_shift, pas,0 ); +mlx5_ib_populate_pas (sq->ubuffer.umem,1UL << page_shift, pas,0 ); err =mlx5_core_create_sq_tracked (dev, in, inlen, &sq->base.mqp);
@@ -4881,7 +4881,7 @@ static int create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,MLX5_SET (rqc, rqc, delay_drop_en,1 ); } rq_pas0 = (__be64 *)MLX5_ADDR_OF (wq, wq, pas); -mlx5_ib_populate_pas (dev, rwq->umem, rwq->page_shift, rq_pas0,0 ); +mlx5_ib_populate_pas (rwq->umem,1UL << rwq->page_shift, rq_pas0,0 ); err =mlx5_core_create_rq_tracked (dev, in, inlen, &rwq->core_qp);if (!err && init_attr->create_flags &IB_WQ_FLAGS_DELAY_DROP ) { err =set_delay_drop (dev);
drivers/infiniband/hw/mlx5/srq.c +1 -1
@@ -100,7 +100,7 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, goto err_umem; } -mlx5_ib_populate_pas (dev, srq->umem, page_shift, in->pas,0 ); +mlx5_ib_populate_pas (srq->umem,1UL << page_shift, in->pas,0 ); err =mlx5_ib_db_map_user (ucontext, udata, ucmd.db_addr, &srq->db);if (err) {
Signature · 1 line
2.26.2
PATCH 2/5 RDMA/mlx5: Move xlt_emergency_page_mutex into mr.c
In reply to Leon RomanovskyFrom: Jason Gunthorpe <[email protected]>
This is the only user, so remove the wrappers.
--- drivers/infiniband/hw/mlx5/main.c | 20 -------------------- drivers/infiniband/hw/mlx5/mlx5_ib.h | 3 +-- drivers/infiniband/hw/mlx5/mr.c | 18 ++++++++++++++++++ 3 files changed, 19 insertions(+), 22 deletions(-)
drivers/infiniband/hw/mlx5/main.c +0 -20
@@ -75,12 +75,6 @@ static LIST_HEAD(mlx5_ib_dev_list);*/ static DEFINE_MUTEX (mlx5_ib_multiport_mutex); -/* We can't use an array for xlt_emergency_page because dma_map_single -* doesn't work on kernel modules memory -*/ -static unsigned long xlt_emergency_page; -static struct mutex xlt_emergency_page_mutex; -struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi (struct mlx5_ib_multiport_info *mpi) {struct mlx5_ib_dev *dev;
@@ -4877,17 +4871,6 @@ static struct mlx5_interface mlx5_ib_interface = { .protocol = MLX5_INTERFACE_PROTOCOL_IB, }; -unsigned long mlx5_ib_get_xlt_emergency_page (void ) -{ -mutex_lock (&xlt_emergency_page_mutex); -return xlt_emergency_page; -} - -void mlx5_ib_put_xlt_emergency_page (void ) -{ -mutex_unlock (&xlt_emergency_page_mutex); -} -static int __init mlx5_ib_init (void ) {int err;
@@ -4896,8 +4879,6 @@ static int __init mlx5_ib_init(void)if (!xlt_emergency_page)return -ENOMEM ; -mutex_init (&xlt_emergency_page_mutex); - mlx5_ib_event_wq =alloc_ordered_workqueue ("mlx5_ib_event_wq" ,0 );if (!mlx5_ib_event_wq) {free_page (xlt_emergency_page);
@@ -4915,7 +4896,6 @@ static void __exit mlx5_ib_cleanup(void) {mlx5_unregister_interface (&mlx5_ib_interface);destroy_workqueue (mlx5_ib_event_wq); -mutex_destroy (&xlt_emergency_page_mutex);free_page (xlt_emergency_page); }
drivers/infiniband/hw/mlx5/mlx5_ib.h +1 -2
@@ -1454,8 +1454,7 @@ static inline int get_num_static_uars(struct mlx5_ib_dev *dev,return get_uars_per_sys_page (dev, bfregi->lib_uar_4k) * bfregi->num_static_sys_pages; } -unsigned long mlx5_ib_get_xlt_emergency_page (void ); -void mlx5_ib_put_xlt_emergency_page (void ); +extern unsigned long xlt_emergency_page; int bfregn_to_uar_index(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, u32 bfregn,
drivers/infiniband/hw/mlx5/mr.c +18 -0
@@ -41,6 +41,13 @@#include <rdma/ib_verbs.h> #include "mlx5_ib.h" +/* +* We can't use an array for xlt_emergency_page because dma_map_single doesn't +* work on kernel modules memory +*/ +unsigned long xlt_emergency_page; +static DEFINE_MUTEX (xlt_emergency_page_mutex); +enum {MAX_PENDING_REG_MR =8 , };
@@ -992,6 +999,17 @@ static struct mlx5_ib_mr *alloc_mr_from_cache(struct ib_pd *pd, MLX5_UMR_MTT_ALIGNMENT)#define MLX5_SPARE_UMR_CHUNK 0x10000 +static unsigned long mlx5_ib_get_xlt_emergency_page (void ) +{ +mutex_lock (&xlt_emergency_page_mutex); +return xlt_emergency_page; +} + +static void mlx5_ib_put_xlt_emergency_page (void ) +{ +mutex_unlock (&xlt_emergency_page_mutex); +} +int mlx5_ib_update_xlt (struct mlx5_ib_mr *mr,u64 idx,int npages,int page_shift,int flags) {
Signature · 1 line
2.26.2
PATCH 3/5 RDMA/mlx5: Split the WR setup out of mlx5_ib_update_xlt()
In reply to Leon RomanovskyFrom: Jason Gunthorpe <[email protected]>
The memory allocation is quite complicated, and makes this function hard to understand. Refactor things so that a function call sets up the WR, SG, DMA mapping and buffer, further splitting that into buffer and DMA/wr.
This also slightly changes the buffer allocation logic to try an order 0 page allocation (with OOM warnings on) before going to the emergency page.
--- drivers/infiniband/hw/mlx5/main.c | 6 +- drivers/infiniband/hw/mlx5/mlx5_ib.h | 2 +- drivers/infiniband/hw/mlx5/mr.c | 189 ++++++++++++++++++--------- 3 files changed, 128 insertions(+), 69 deletions(-)
drivers/infiniband/hw/mlx5/main.c +3 -3
@@ -4875,13 +4875,13 @@ static int __init mlx5_ib_init(void) {int err; - xlt_emergency_page =__get_free_page (GFP_KERNEL ); + xlt_emergency_page = (void *)__get_free_page (GFP_KERNEL );if (!xlt_emergency_page)return -ENOMEM ; mlx5_ib_event_wq =alloc_ordered_workqueue ("mlx5_ib_event_wq" ,0 );if (!mlx5_ib_event_wq) { -free_page (xlt_emergency_page); +free_page ((unsigned long )xlt_emergency_page);return -ENOMEM ; }
@@ -4896,7 +4896,7 @@ static void __exit mlx5_ib_cleanup(void) {mlx5_unregister_interface (&mlx5_ib_interface);destroy_workqueue (mlx5_ib_event_wq); -free_page (xlt_emergency_page); +free_page ((unsigned long )xlt_emergency_page); }module_init (mlx5_ib_init);
drivers/infiniband/hw/mlx5/mlx5_ib.h +1 -1
@@ -1454,7 +1454,7 @@ static inline int get_num_static_uars(struct mlx5_ib_dev *dev,return get_uars_per_sys_page (dev, bfregi->lib_uar_4k) * bfregi->num_static_sys_pages; } -extern unsigned long xlt_emergency_page; +extern void *xlt_emergency_page; int bfregn_to_uar_index(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, u32 bfregn,
drivers/infiniband/hw/mlx5/mr.c +124 -65
254 lines not on this page · raw message
Signature · 1 line
2.26.2
PATCH 5/5 RDMA/mlx5: Use ib_umem_find_best_pgsz() for mkc's
In reply to Leon RomanovskyFrom: Jason Gunthorpe <[email protected]>
Now that all the PAS arrays or UMR XLT's for mkcs are filled using rdma_for_each_block() we can use the common ib_umem_find_best_pgsz() algorithm.
--- drivers/infiniband/core/umem.c | 9 +++++++ drivers/infiniband/hw/mlx5/mlx5_ib.h | 27 ++++++++++++++++++++ drivers/infiniband/hw/mlx5/mr.c | 37 +++++++++++++++------------- 3 files changed, 56 insertions(+), 17 deletions(-)
drivers/infiniband/core/umem.c +9 -0
@@ -84,6 +84,15 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,dma_addr_t mask;int i; +if (umem->is_odp) { +unsigned int page_size =BIT (to_ib_umem_odp (umem)->page_shift); + +/* ODP must always be self consistent. */ +if (!(pgsz_bitmap & page_size)) +return 0 ; +return page_size; + } +/* rdma_for_each_block() has a bug if the page size is smaller than the * page size used to build the umem. For now prevent smaller page sizes * from being returned.
drivers/infiniband/hw/mlx5/mlx5_ib.h +27 -0
@@ -42,6 +42,33 @@#define MLX5_MKEY_PAGE_SHIFT_MASK __mlx5_mask(mkc, log_page_size) +static __always_inline unsigned long +__mlx5_log_page_size_to_bitmap (unsigned int log_pgsz_bits, +unsigned int pgsz_shift) +{ +unsigned int largest_pg_shift = + min_t(unsigned long, (1ULL << log_pgsz_bits) - 1 + pgsz_shift, +BITS_PER_LONG -1 ); + +/* +* Despite a command allowing it, the device does not support lower than +* 4k page size. +*/ + pgsz_shift = max_t(unsigned int, MLX5_ADAPTER_PAGE_SHIFT, pgsz_shift); +return GENMASK (largest_pg_shift, pgsz_shift); +} + +/* +* For mkc users, instead of a page_offset the command has a start_iova which +* specifies both the page_offset and the on-the-wire IOVA +*/ +#define mlx5_umem_find_best_pgsz (umem, typ, log_pgsz_fld, pgsz_shift, iova) \ + ib_umem_find_best_pgsz(umem, \ + __mlx5_log_page_size_to_bitmap( \ + __mlx5_bit_sz(typ, log_pgsz_fld), \ + pgsz_shift), \ + iova) +enum {MLX5_IB_MMAP_OFFSET_START =9 ,MLX5_IB_MMAP_OFFSET_END =255 ,
drivers/infiniband/hw/mlx5/mr.c +20 -17
@@ -964,11 +964,13 @@ static struct mlx5_ib_mr *alloc_mr_from_cache(struct ib_pd *pd,struct mlx5_ib_dev *dev =to_mdev (pd->device);struct mlx5_cache_ent *ent;struct mlx5_ib_mr *mr; -int page_shift; +unsigned int page_size; -mlx5_ib_cont_pages (umem, iova,MLX5_MKEY_PAGE_SHIFT_MASK , &page_shift); - ent =mr_cache_ent_from_order (dev,order_base_2 (ib_umem_num_dma_blocks ( - umem,1UL << page_shift))); + page_size =mlx5_umem_find_best_pgsz (umem, mkc, log_page_size,0 , iova); +if (WARN_ON (!page_size)) +return ERR_PTR (-EINVAL ); + ent =mr_cache_ent_from_order ( + dev,order_base_2 (ib_umem_num_dma_blocks (umem, page_size)));if (!ent)return ERR_PTR (-E2BIG );
@@ -990,7 +992,7 @@ static struct mlx5_ib_mr *alloc_mr_from_cache(struct ib_pd *pd, mr->mmkey.iova = iova; mr->mmkey.size = umem->length; mr->mmkey.pd =to_mpd (pd)->pdn; - mr->page_shift = page_shift; + mr->page_shift =order_base_2 (page_size);return mr; }
@@ -1280,8 +1282,8 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, int access_flags, bool populate) {struct mlx5_ib_dev *dev =to_mdev (pd->device); +unsigned int page_size;struct mlx5_ib_mr *mr; -int page_shift;__be64 *pas;void *mkc;int inlen;
@@ -1289,22 +1291,23 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,int err;bool pg_cap = !!(MLX5_CAP_GEN (dev->mdev, pg)); + page_size = +mlx5_umem_find_best_pgsz (umem, mkc, log_page_size,0 , iova); +if (WARN_ON (!page_size)) +return ERR_PTR (-EINVAL ); + mr = ibmr ?to_mmr (ibmr) :kzalloc (sizeof (*mr),GFP_KERNEL );if (!mr)return ERR_PTR (-ENOMEM ); -mlx5_ib_cont_pages (umem, iova,MLX5_MKEY_PAGE_SHIFT_MASK , &page_shift); - - mr->page_shift = page_shift; mr->ibmr.pd = pd; mr->access_flags = access_flags; + mr->page_shift =order_base_2 (page_size); inlen =MLX5_ST_SZ_BYTES (create_mkey_in);if (populate) - inlen += -sizeof (*pas) * -roundup (ib_umem_num_dma_blocks (umem,1UL << page_shift), -2 ); + inlen +=sizeof (*pas) * +roundup (ib_umem_num_dma_blocks (umem, page_size),2 ); in =kvzalloc (inlen,GFP_KERNEL );if (!in) { err = -ENOMEM ;
@@ -1316,7 +1319,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, err = -EINVAL ; goto err_2; } -mlx5_ib_populate_pas (umem,1ULL << page_shift, pas, +mlx5_ib_populate_pas (umem,1UL << mr->page_shift, pas, pg_cap ?MLX5_IB_MTT_PRESENT :0 ); }
@@ -1334,11 +1337,11 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,MLX5_SET64 (mkc, mkc, len, umem->length);MLX5_SET (mkc, mkc, bsf_octword_size,0 );MLX5_SET (mkc, mkc, translations_octword_size, -get_octo_len (iova, umem->length, page_shift)); -MLX5_SET (mkc, mkc, log_page_size, page_shift); +get_octo_len (iova, umem->length, mr->page_shift)); +MLX5_SET (mkc, mkc, log_page_size, mr->page_shift);if (populate) {MLX5_SET (create_mkey_in, in, translations_octword_actual_size, -get_octo_len (iova, umem->length, page_shift)); +get_octo_len (iova, umem->length, mr->page_shift)); } err =mlx5_ib_create_mkey (dev, &mr->mmkey, in, inlen);
Signature · 1 line
2.26.2
PATCH 4/5 RDMA/mlx5: Split mlx5_ib_update_xlt() into ODP and non-ODP cases
In reply to Leon RomanovskyFrom: Jason Gunthorpe <[email protected]>
Mixing these together is just a mess, make a dedicated version, mlx5_ib_update_mr_pas(), which directly loads the whole MTT for a non-ODP MR.
The split out version can trivially use a simple loop with rdma_for_each_block() which allows using the core code to compute the MR pages and avoids seeking in the SGL list after each chunk as the __mlx5_ib_populate_pas() call required.
Significantly speeds loading large MTTs.
--- drivers/infiniband/hw/mlx5/mem.c | 64 ------------- drivers/infiniband/hw/mlx5/mlx5_ib.h | 3 - drivers/infiniband/hw/mlx5/mr.c | 137 +++++++++++++++++++-------- 3 files changed, 97 insertions(+), 107 deletions(-)
drivers/infiniband/hw/mlx5/mem.c +0 -64
@@ -91,70 +91,6 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, *shift =PAGE_SHIFT + m; } -/* -* Populate the given array with bus addresses from the umem. -* -* dev - mlx5_ib device -* umem - umem to use to fill the pages -* page_shift - determines the page size used in the resulting array -* offset - offset into the umem to start from, -* only implemented for ODP umems -* num_pages - total number of pages to fill -* pas - bus addresses array to fill -* access_flags - access flags to set on all present pages. -use enum mlx5_ib_mtt_access_flags for this. -*/ -void __mlx5_ib_populate_pas (struct mlx5_ib_dev *dev,struct ib_umem *umem, -int page_shift,size_t offset,size_t num_pages, -__be64 *pas,int access_flags) -{ -int shift = page_shift -PAGE_SHIFT ; -int mask = (1 << shift) -1 ; -int i, k, idx; -u64 cur =0 ; -u64 base; -int len; -struct scatterlist *sg; -int entry; - - i =0 ; -for_each_sg (umem->sg_head.sgl, sg, umem->nmap, entry) { - len =sg_dma_len (sg) >>PAGE_SHIFT ; - base =sg_dma_address (sg); - -/* Skip elements below offset */ -if (i + len < offset << shift) { - i += len; -continue ; - } - -/* Skip pages below offset */ -if (i < offset << shift) { - k = (offset << shift) - i; - i = offset << shift; - }else { - k =0 ; - } - -for (; k < len; k++) { -if (!(i & mask)) { - cur = base + (k <<PAGE_SHIFT ); - cur |= access_flags; - idx = (i >> shift) - offset; - - pas[idx] =cpu_to_be64 (cur); -mlx5_ib_dbg (dev,"pas[%d] 0x%llx\n" , - i >> shift,be64_to_cpu (pas[idx])); - } - i++; - -/* Stop after num_pages reached */ -if (i >> shift >= offset + num_pages) -return ; - } - } -} -/* * Fill in a physical address list. ib_umem_num_dma_blocks() entries will be * filled in the pas array.
drivers/infiniband/hw/mlx5/mlx5_ib.h +0 -3
@@ -1232,9 +1232,6 @@ int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,void mlx5_ib_cont_pages (struct ib_umem *umem,u64 addr,unsigned long max_page_shift,int *shift); -void __mlx5_ib_populate_pas (struct mlx5_ib_dev *dev,struct ib_umem *umem, -int page_shift,size_t offset,size_t num_pages, -__be64 *pas,int access_flags);void mlx5_ib_populate_pas (struct ib_umem *umem,size_t page_size,__be64 *pas,u64 access_flags);void mlx5_ib_copy_pas (u64 *old,u64 *new,int step,int num);
drivers/infiniband/hw/mlx5/mr.c +97 -40
185 lines not on this page · raw message
Signature · 1 line
2.26.2
Jason Gunthorpe
maintainerRe: [PATCH rdma-next 4/5] RDMA/mlx5: Split mlx5_ib_update_xlt() into ODP and non-ODP cases
Quoted 6 lines · Leon Romanovsky
drivers/infiniband/hw/mlx5/mr.c +0
-2@@ -1483,10 +1540,14 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,*/ int update_xlt_flags =MLX5_IB_UPD_XLT_ENABLE ; - err = mlx5_ib_update_xlt( - mr, 0,
drivers/infiniband/hw/mlx5/mr.c struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, · line 1550
Quoted 10 lines
drivers/infiniband/hw/mlx5/mr.c +8
-2-ib_umem_num_dma_blocks (umem,1UL << mr->page_shift), - mr->page_shift, update_xlt_flags); +if (is_odp_mr (mr)) + err =mlx5_ib_update_xlt ( + mr,0 , +ib_umem_num_dma_blocks (umem, +1UL << mr->page_shift), + mr->page_shift, update_xlt_flags); +else + err =mlx5_ib_update_mr_pas (mr, update_xlt_flags);
This rebase looks a bit weird, this whole block is !ODP already, so why is there an 'if is_odp_mr()' ?
Should just be this:
if (xlt_with_umr && !(access_flags & IB_ACCESS_ON_DEMAND)) {
/*
* If the MR was created with reg_create then it will be
* configured properly but left disabled. It is safe to go ahead
* and configure it again via UMR while enabling it.
*/
err = mlx5_ib_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE);
if (err) {
dereg_mr(dev, mr);
return ERR_PTR(err);
}
}Jason
Quoted 10 lines · Leon Romanovsky
From: Leon Romanovsky <[email protected]>
Quoted 1 line
From Jason:
The new common code does a better job finding large page sizes. Use it in mlx5 for MRs.
This requires moving the MTT population for mailboxes and UMR over to
rdma_for_each_dma_block().
Quoted 8 lines
Thanks
Jason Gunthorpe (5):
RDMA/mlx5: Changemlx5_ib_populate_pas()to userdma_for_each_block()RDMA/mlx5: Movexlt_emergency_page_mutexintomr.cRDMA/mlx5: Split the WR setup out ofmlx5_ib_update_xlt()RDMA/mlx5: Splitmlx5_ib_update_xlt()into ODP and non-ODP cases RDMA/mlx5: Useib_umem_find_best_pgsz()for mkc's
Applied to for-next, with the updated hunk
Thanks,
Jason