Use ib_umem_find_best_pgsz() for all umems
Subsystems (2)
cover Use ib_umem_find_best_pgsz() for all umems
From: Leon Romanovsky <[email protected]>
Changelog:
v1:
* Added patch for raw QP
* Fixed commit messages
v0: https://lore.kernel.org/lkml/[email protected]
-------------------------
From Jason:
Move the remaining cases working with umems to use versions of ib_umem_find_best_pgsz() tailored to the calculations the devices requires.
Unlike a MR there is no IOVA, instead a page offset from the starting page is possible, with various restrictions.
Compute the best page size to meet the page_offset restrictions.
Thanks
Jason Gunthorpe (7):
RDMA/mlx5: Use ib_umem_find_best_pgoff() for SRQ RDMA/mlx5: Use mlx5_umem_find_best_quantized_pgoff() for WQ RDMA/mlx5: Directly compute the PAS list for raw QP RQ's RDMA/mlx5: Use mlx5_umem_find_best_quantized_pgoff() for QP RDMA/mlx5: mlx5_umem_find_best_quantized_pgoff() for CQ RDMA/mlx5: Use ib_umem_find_best_pgsz() for devx RDMA/mlx5: Lower setting the umem's PAS for SRQ
drivers/infiniband/hw/mlx5/cq.c | 48 +++++--- drivers/infiniband/hw/mlx5/devx.c | 66 ++++++----- drivers/infiniband/hw/mlx5/mem.c | 115 +++++++------------ drivers/infiniband/hw/mlx5/mlx5_ib.h | 47 +++++++- drivers/infiniband/hw/mlx5/qp.c | 165 ++++++++++++--------------- drivers/infiniband/hw/mlx5/srq.c | 27 +---- drivers/infiniband/hw/mlx5/srq.h | 1 + drivers/infiniband/hw/mlx5/srq_cmd.c | 80 ++++++++++++- include/rdma/ib_umem.h | 42 +++++++ 9 files changed, 343 insertions(+), 248 deletions(-)
--
2.28.0
PATCH 1/7 RDMA/mlx5: Use ib_umem_find_best_pgoff() for SRQ
From: Jason Gunthorpe <[email protected]>
SRQ uses a quantized and scaled page_offset, which is another variation of ib_umem_find_best_pgsz(). Add mlx5_umem_find_best_quantized_pgoff() to perform this calculation for each mailbox. A macro shows how the calculation is directly connected to the mailbox format.
This new routine replaces the limited mlx5_ib_cont_pages() and mlx5_ib_get_buf_offset() pairing which would reject valid configurations rather than adjust the page_size to make it work.
In turn this is much more aggressive about choosing large page sizes for these objects and when THP is enabled it will now often find a single page solution.
--- drivers/infiniband/hw/mlx5/mem.c | 45 ++++++++++++++++++++++++++++ drivers/infiniband/hw/mlx5/mlx5_ib.h | 31 +++++++++++++++++++ drivers/infiniband/hw/mlx5/srq.c | 20 ++++++------- include/rdma/ib_umem.h | 42 ++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 10 deletions(-)
drivers/infiniband/hw/mlx5/mem.c +45 -0
@@ -107,6 +107,51 @@ void mlx5_ib_populate_pas(struct ib_umem *umem, size_t page_size, __be64 *pas, } } +/* +* Compute the page shift and page_offset for mailboxes that use a quantized +* page_offset. The granulatity of the page offset scales according to page +* size. +*/ +unsigned long __mlx5_umem_find_best_quantized_pgoff ( +struct ib_umem *umem,unsigned long pgsz_bitmap, +unsigned int page_offset_bits,u64 pgoff_bitmask,unsigned int scale, +unsigned int *page_offset_quantized) +{ +const u64 page_offset_mask = (1 << page_offset_bits) -1 ; +unsigned long page_size; +u64 page_offset; + + page_size =ib_umem_find_best_pgoff (umem, pgsz_bitmap, pgoff_bitmask); +if (!page_size) +return 0 ; + +/* +* page size is the largest possible page size. +* +* Reduce the page_size, and thus the page_offset and quanta, until the +* page_offset fits into the mailbox field. Once page_size < scale this +* loop is guaranteed to terminate. +*/ + page_offset =ib_umem_dma_offset (umem, page_size); +while (page_offset & ~(u64)(page_offset_mask * (page_size / scale))) { + page_size /=2 ; + page_offset =ib_umem_dma_offset (umem, page_size); + } + +/* +* The address is not aligned, or otherwise cannot be represented by the +* page_offset. +*/ +if (!(pgsz_bitmap & page_size)) +return 0 ; + + *page_offset_quantized = + (unsigned long )page_offset / (page_size / scale); +if (WARN_ON (*page_offset_quantized > page_offset_mask)) +return 0 ; +return page_size; +} +int mlx5_ib_get_buf_offset (u64 addr,int page_shift,u32 *offset) {u64 page_size;
drivers/infiniband/hw/mlx5/mlx5_ib.h +31 -0
@@ -69,6 +69,37 @@ __mlx5_log_page_size_to_bitmap(unsigned int log_pgsz_bits, pgsz_shift), \ iova) +static __always_inline unsigned long +__mlx5_page_offset_to_bitmask (unsigned int page_offset_bits, +unsigned int offset_shift) +{ +unsigned int largest_offset_shift = + min_t(unsigned long, page_offset_bits - 1 + offset_shift, +BITS_PER_LONG -1 ); + +return GENMASK (largest_offset_shift, offset_shift); +} + +/* +* QP/CQ/WQ/etc type commands take a page offset that satisifies: +* page_offset_quantized * (page_size/scale) = page_offset +* Which restricts allowed page sizes to ones that satisify the above. +*/ +unsigned long __mlx5_umem_find_best_quantized_pgoff ( +struct ib_umem *umem,unsigned long pgsz_bitmap, +unsigned int page_offset_bits,u64 pgoff_bitmask,unsigned int scale, +unsigned int *page_offset_quantized); +#define mlx5_umem_find_best_quantized_pgoff (umem, typ, log_pgsz_fld, \ + pgsz_shift, page_offset_fld, \ + scale, page_offset_quantized) \ + __mlx5_umem_find_best_quantized_pgoff( \ + umem, \ + __mlx5_log_page_size_to_bitmap( \ + __mlx5_bit_sz(typ, log_pgsz_fld), pgsz_shift), \ + __mlx5_bit_sz(typ, page_offset_fld), \ + GENMASK(31, order_base_2(scale)), scale, \ + page_offset_quantized) +enum {MLX5_IB_MMAP_OFFSET_START =9 ,MLX5_IB_MMAP_OFFSET_END =255 ,
drivers/infiniband/hw/mlx5/srq.c +10 -10
@@ -51,8 +51,8 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, udata, struct mlx5_ib_ucontext, ibucontext);size_t ucmdlen;int err; -int page_shift; -u32 offset; +unsigned int page_offset_quantized; +unsigned int page_size;u32 uidx =MLX5_IB_DEFAULT_UIDX ; ucmdlen =min (udata->inlen,sizeof (ucmd));
@@ -85,22 +85,22 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,return err; } -mlx5_ib_cont_pages (srq->umem, ucmd.buf_addr,0 , &page_shift); - err =mlx5_ib_get_buf_offset (ucmd.buf_addr, page_shift, - &offset); -if (err) { + page_size =mlx5_umem_find_best_quantized_pgoff ( + srq->umem, srqc, log_page_size,MLX5_ADAPTER_PAGE_SHIFT , + page_offset,64 , &page_offset_quantized); +if (!page_size) {mlx5_ib_warn (dev,"bad offset\n" ); goto err_umem; } - in->pas =kvcalloc (ib_umem_num_dma_blocks (srq->umem,1UL << page_shift), + in->pas =kvcalloc (ib_umem_num_dma_blocks (srq->umem, page_size),sizeof (*in->pas),GFP_KERNEL );if (!in->pas) { err = -ENOMEM ; goto err_umem; } -mlx5_ib_populate_pas (srq->umem,1UL << page_shift, in->pas,0 ); +mlx5_ib_populate_pas (srq->umem, page_size, in->pas,0 ); err =mlx5_ib_db_map_user (ucontext, udata, ucmd.db_addr, &srq->db);if (err) {
@@ -108,8 +108,8 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, goto err_in; } - in->log_page_size = page_shift -MLX5_ADAPTER_PAGE_SHIFT ; - in->page_offset = offset; + in->log_page_size =order_base_2 (page_size) -MLX5_ADAPTER_PAGE_SHIFT ; + in->page_offset = page_offset_quantized; in->uid = (in->type !=IB_SRQT_XRC ) ?to_mpd (pd)->uid :0 ;if (MLX5_CAP_GEN (dev->mdev, cqe_version) ==MLX5_CQE_VERSION_V1 && in->type !=IB_SRQT_BASIC )
include/rdma/ib_umem.h +42 -0
60 lines not on this page · raw message
Signature · 1 line
2.28.0
PATCH 2/7 RDMA/mlx5: Use mlx5_umem_find_best_quantized_pgoff() for WQ
In reply to Leon RomanovskyFrom: Jason Gunthorpe <[email protected]>
This fixes a subtle bug, the WQ mailbox has only 5 bits to describe the page_offset, while mlx5_ib_get_buf_offset() is hard wired to only work with 6 bit page_offsets.
Thus it did not properly reject badly aligned buffers.
--- drivers/infiniband/hw/mlx5/qp.c | 53 +++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 22 deletions(-)
drivers/infiniband/hw/mlx5/qp.c +31 -22
@@ -833,7 +833,7 @@ static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd, {struct mlx5_ib_ucontext *ucontext =rdma_udata_to_drv_context ( udata, struct mlx5_ib_ucontext, ibucontext); -int page_shift =0 ; +unsigned long page_size =0 ;u32 offset =0 ;int err;
@@ -847,24 +847,25 @@ static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,return err; } -mlx5_ib_cont_pages (rwq->umem, ucmd->buf_addr,0 , &page_shift); - err =mlx5_ib_get_buf_offset (ucmd->buf_addr, page_shift, - &rwq->rq_page_offset); -if (err) { + page_size =mlx5_umem_find_best_quantized_pgoff ( + rwq->umem, wq, log_wq_pg_sz,MLX5_ADAPTER_PAGE_SHIFT , + page_offset,64 , &rwq->rq_page_offset); +if (!page_size) {mlx5_ib_warn (dev,"bad offset\n" ); + err = -EINVAL ; goto err_umem; } - rwq->rq_num_pas =ib_umem_num_dma_blocks (rwq->umem,1UL << page_shift); - rwq->page_shift = page_shift; - rwq->log_page_size = page_shift -MLX5_ADAPTER_PAGE_SHIFT ; + rwq->rq_num_pas =ib_umem_num_dma_blocks (rwq->umem, page_size); + rwq->page_shift =order_base_2 (page_size); + rwq->log_page_size = rwq->page_shift -MLX5_ADAPTER_PAGE_SHIFT ; rwq->wq_sig = !!(ucmd->flags &MLX5_WQ_FLAG_SIGNATURE );mlx5_ib_dbg ( dev, -"addr 0x%llx, size %zd, npages %zu, page_shift %d, ncont %d, offset %d\n" , +"addr 0x%llx, size %zd, npages %zu, page_size %ld, ncont %d, offset %d\n" , (unsigned long long )ucmd->buf_addr, rwq->buf_size, -ib_umem_num_pages (rwq->umem), page_shift, rwq->rq_num_pas, +ib_umem_num_pages (rwq->umem), page_size, rwq->rq_num_pas, offset); err =mlx5_ib_db_map_user (ucontext, udata, ucmd->db_addr, &rwq->db);
@@ -1209,17 +1210,24 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,void *wq;int inlen;int err; -int page_shift =0 ; -u32 offset =0 ; - - err =mlx5_ib_umem_get (dev, udata, ubuffer->buf_addr, ubuffer->buf_size, - &sq->ubuffer.umem, &page_shift, &offset); -if (err) -return err; +unsigned int page_offset_quantized; +unsigned long page_size; + + sq->ubuffer.umem =ib_umem_get (&dev->ib_dev, ubuffer->buf_addr, + ubuffer->buf_size,0 ); +if (IS_ERR (sq->ubuffer.umem)) +return PTR_ERR (sq->ubuffer.umem); + page_size =mlx5_umem_find_best_quantized_pgoff ( + ubuffer->umem, wq, log_wq_pg_sz,MLX5_ADAPTER_PAGE_SHIFT , + page_offset,64 , &page_offset_quantized); +if (!page_size) { + err = -EINVAL ; + goto err_umem; + } inlen =MLX5_ST_SZ_BYTES (create_sq_in) + -sizeof (u64) *ib_umem_num_dma_blocks (sq->ubuffer.umem, -1UL << page_shift); +sizeof (u64) * +ib_umem_num_dma_blocks (sq->ubuffer.umem, page_size); in =kvzalloc (inlen,GFP_KERNEL );if (!in) { err = -ENOMEM ;
@@ -1247,11 +1255,12 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,MLX5_SET64 (wq, wq, dbr_addr,MLX5_GET64 (qpc, qpc, dbr_addr));MLX5_SET (wq, wq, log_wq_stride,ilog2 (MLX5_SEND_WQE_BB ));MLX5_SET (wq, wq, log_wq_sz,MLX5_GET (qpc, qpc, log_sq_size)); -MLX5_SET (wq, wq, log_wq_pg_sz, page_shift -MLX5_ADAPTER_PAGE_SHIFT ); -MLX5_SET (wq, wq, page_offset, offset); +MLX5_SET (wq, wq, log_wq_pg_sz, +order_base_2 (page_size) -MLX5_ADAPTER_PAGE_SHIFT ); +MLX5_SET (wq, wq, page_offset, page_offset_quantized); pas = (__be64 *)MLX5_ADDR_OF (wq, wq, pas); -mlx5_ib_populate_pas (sq->ubuffer.umem,1UL << page_shift, pas,0 ); +mlx5_ib_populate_pas (sq->ubuffer.umem, page_size, pas,0 ); err =mlx5_core_create_sq_tracked (dev, in, inlen, &sq->base.mqp);
Signature · 1 line
2.28.0
PATCH 3/7 RDMA/mlx5: Directly compute the PAS list for raw QP RQ's
In reply to Leon RomanovskyFrom: Jason Gunthorpe <[email protected]>
The RQ WQ created when making a raw ethernet QP copies the PAS list from a dummy QPC command created earlier in the flow. The WQC and QPC PAS lists are not fully compatible as the page_offset is a different size.
Create the RQ WQ's PAS list directly and do not try to copy it from another command structure.
Like the prior patch, this also means that badly aligned buffers were not correctly rejected.
--- drivers/infiniband/hw/mlx5/qp.c | 41 +++++++++++++-------------------- 1 file changed, 16 insertions(+), 25 deletions(-)
drivers/infiniband/hw/mlx5/qp.c +16 -25
@@ -1286,40 +1286,31 @@ static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,ib_umem_release (sq->ubuffer.umem); } -static size_t get_rq_pas_size (void *qpc) -{ -u32 log_page_size =MLX5_GET (qpc, qpc, log_page_size) +12 ; -u32 log_rq_stride =MLX5_GET (qpc, qpc, log_rq_stride); -u32 log_rq_size =MLX5_GET (qpc, qpc, log_rq_size); -u32 page_offset =MLX5_GET (qpc, qpc, page_offset); -u32 po_quanta =1 << (log_page_size -6 ); -u32 rq_sz =1 << (log_rq_size +4 + log_rq_stride); -u32 page_size =1 << log_page_size; -u32 rq_sz_po = rq_sz + (page_offset * po_quanta); -u32 rq_num_pas = (rq_sz_po + page_size -1 ) / page_size; - -return rq_num_pas *sizeof (u64); -} -static int create_raw_packet_qp_rq (struct mlx5_ib_dev *dev,struct mlx5_ib_rq *rq,void *qpin, -size_t qpinlen,struct ib_pd *pd) +struct ib_pd *pd) {struct mlx5_ib_qp *mqp = rq->base.container_mibqp;__be64 *pas; -__be64 *qp_pas;void *in;void *rqc;void *wq;void *qpc =MLX5_ADDR_OF (create_qp_in, qpin, qpc); -size_t rq_pas_size =get_rq_pas_size (qpc); +struct ib_umem *umem = rq->base.ubuffer.umem; +unsigned int page_offset_quantized; +unsigned long page_size =0 ;size_t inlen;int err; -if (qpinlen < rq_pas_size +MLX5_BYTE_OFF (create_qp_in, pas)) + page_size =mlx5_umem_find_best_quantized_pgoff (umem, wq, log_wq_pg_sz, +MLX5_ADAPTER_PAGE_SHIFT , + page_offset,64 , + &page_offset_quantized); +if (!page_size)return -EINVAL ; - inlen =MLX5_ST_SZ_BYTES (create_rq_in) + rq_pas_size; + inlen =MLX5_ST_SZ_BYTES (create_rq_in) + +sizeof (u64) *ib_umem_num_dma_blocks (umem, page_size); in =kvzalloc (inlen,GFP_KERNEL );if (!in)return -ENOMEM ;
@@ -1341,16 +1332,16 @@ static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,MLX5_SET (wq, wq, wq_type,MLX5_WQ_TYPE_CYCLIC );if (rq->flags &MLX5_IB_RQ_PCI_WRITE_END_PADDING )MLX5_SET (wq, wq, end_padding_mode,MLX5_WQ_END_PAD_MODE_ALIGN ); -MLX5_SET (wq, wq, page_offset,MLX5_GET (qpc, qpc, page_offset)); +MLX5_SET (wq, wq, page_offset, page_offset_quantized);MLX5_SET (wq, wq, pd,MLX5_GET (qpc, qpc, pd));MLX5_SET64 (wq, wq, dbr_addr,MLX5_GET64 (qpc, qpc, dbr_addr));MLX5_SET (wq, wq, log_wq_stride,MLX5_GET (qpc, qpc, log_rq_stride) +4 ); -MLX5_SET (wq, wq, log_wq_pg_sz,MLX5_GET (qpc, qpc, log_page_size)); +MLX5_SET (wq, wq, log_wq_pg_sz, +order_base_2 (page_size) -MLX5_ADAPTER_PAGE_SHIFT );MLX5_SET (wq, wq, log_wq_sz,MLX5_GET (qpc, qpc, log_rq_size)); pas = (__be64 *)MLX5_ADDR_OF (wq, wq, pas); - qp_pas = (__be64 *)MLX5_ADDR_OF (create_qp_in, qpin, pas); -memcpy (pas, qp_pas, rq_pas_size); +mlx5_ib_populate_pas (umem, page_size, pas,0 ); err =mlx5_core_create_rq_tracked (dev, in, inlen, &rq->base.mqp);
@@ -1471,7 +1462,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, rq->flags |=MLX5_IB_RQ_CVLAN_STRIPPING ;if (qp->flags &IB_QP_CREATE_PCI_WRITE_END_PADDING ) rq->flags |=MLX5_IB_RQ_PCI_WRITE_END_PADDING ; - err =create_raw_packet_qp_rq (dev, rq, in, inlen, pd); + err =create_raw_packet_qp_rq (dev, rq, in, pd);if (err) goto err_destroy_sq;
Signature · 1 line
2.28.0
PATCH 4/7 RDMA/mlx5: Use mlx5_umem_find_best_quantized_pgoff() for QP
In reply to Leon RomanovskyFrom: Jason Gunthorpe <[email protected]>
Delete custom logic in the QP in favor of more general variant.
--- drivers/infiniband/hw/mlx5/mem.c | 21 -------- drivers/infiniband/hw/mlx5/mlx5_ib.h | 1 - drivers/infiniband/hw/mlx5/qp.c | 71 +++++++++------------------- 3 files changed, 23 insertions(+), 70 deletions(-)
drivers/infiniband/hw/mlx5/mem.c +0 -21
@@ -152,27 +152,6 @@ unsigned long __mlx5_umem_find_best_quantized_pgoff(return page_size; } -int mlx5_ib_get_buf_offset (u64 addr,int page_shift,u32 *offset) -{ -u64 page_size; -u64 page_mask; -u64 off_size; -u64 off_mask; -u64 buf_off; - - page_size = (u64 )1 << page_shift; - page_mask = page_size -1 ; - buf_off = addr & page_mask; - off_size = page_size >>6 ; - off_mask = off_size -1 ; - -if (buf_off & off_mask) -return -EINVAL ; - - *offset = buf_off >>ilog2 (off_size); -return 0 ; -} -#define WR_ID_BF 0xBF#define WR_ID_END 0xBAD#define TEST_WC_NUM_WQES 255
drivers/infiniband/hw/mlx5/mlx5_ib.h +0 -1
@@ -1267,7 +1267,6 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, size_t *out_mad_size, u16 *out_mad_pkey_index);int mlx5_ib_alloc_xrcd (struct ib_xrcd *xrcd,struct ib_udata *udata);int mlx5_ib_dealloc_xrcd (struct ib_xrcd *xrcd,struct ib_udata *udata); -int mlx5_ib_get_buf_offset (u64 addr,int page_shift,u32 *offset);int mlx5_query_ext_port_caps (struct mlx5_ib_dev *dev,u8 port);int mlx5_query_mad_ifc_smp_attr_node_info (struct ib_device *ibdev,struct ib_smp *out_mad);
drivers/infiniband/hw/mlx5/qp.c +23 -48
98 lines not on this page · raw message
Signature · 1 line
2.28.0
PATCH 5/7 RDMA/mlx5: mlx5_umem_find_best_quantized_pgoff() for CQ
In reply to Leon RomanovskyFrom: Jason Gunthorpe <[email protected]>
This fixes a bug where the page_offset was not being considered when building a CQ. The HW specification says it 'must be zero', so use a variant of mlx5_umem_find_best_quantized_pgoff() with a 0 pgoff_bitmask to force this result.
--- drivers/infiniband/hw/mlx5/cq.c | 48 ++++++++++++++++++++-------- drivers/infiniband/hw/mlx5/mlx5_ib.h | 10 ++++++ 2 files changed, 44 insertions(+), 14 deletions(-)
drivers/infiniband/hw/mlx5/cq.c +34 -14
@@ -707,8 +707,9 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, int *cqe_size, int *index, int *inlen) { struct mlx5_ib_create_cq ucmd = {}; +unsigned long page_size; +unsigned int page_offset_quantized;size_t ucmdlen; -int page_shift;__be64 *pas;int ncont;void *cqc;
@@ -741,17 +742,24 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,return err; } + page_size =mlx5_umem_find_best_cq_quantized_pgoff ( + cq->buf.umem, cqc, log_page_size,MLX5_ADAPTER_PAGE_SHIFT , + page_offset,64 , &page_offset_quantized); +if (!page_size) { + err = -EINVAL ; + goto err_umem; + } + err =mlx5_ib_db_map_user (context, udata, ucmd.db_addr, &cq->db);if (err) goto err_umem; -mlx5_ib_cont_pages (cq->buf.umem, ucmd.buf_addr,0 , &page_shift); - ncont =ib_umem_num_dma_blocks (cq->buf.umem,1UL << page_shift); + ncont =ib_umem_num_dma_blocks (cq->buf.umem, page_size);mlx5_ib_dbg ( dev, -"addr 0x%llx, size %u, npages %zu, page_shift %d, ncont %d\n" , +"addr 0x%llx, size %u, npages %zu, page_size %lu, ncont %d\n" , ucmd.buf_addr, entries * ucmd.cqe_size, -ib_umem_num_pages (cq->buf.umem), page_shift, ncont); +ib_umem_num_pages (cq->buf.umem), page_size, ncont); *inlen =MLX5_ST_SZ_BYTES (create_cq_in) +MLX5_FLD_SZ_BYTES (create_cq_in, pas[0 ]) * ncont;
@@ -762,11 +770,12 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata, } pas = (__be64 *)MLX5_ADDR_OF (create_cq_in, *cqb, pas); -mlx5_ib_populate_pas (cq->buf.umem,1UL << page_shift, pas,0 ); +mlx5_ib_populate_pas (cq->buf.umem, page_size, pas,0 ); cqc =MLX5_ADDR_OF (create_cq_in, *cqb, cq_context);MLX5_SET (cqc, cqc, log_page_size, - page_shift -MLX5_ADAPTER_PAGE_SHIFT ); +order_base_2 (page_size) -MLX5_ADAPTER_PAGE_SHIFT ); +MLX5_SET (cqc, cqc, page_offset, page_offset_quantized);if (ucmd.flags &MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX ) { *index = ucmd.uar_page_index;
@@ -1131,7 +1140,7 @@ int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)static int resize_user (struct mlx5_ib_dev *dev,struct mlx5_ib_cq *cq,int entries,struct ib_udata *udata, -int *page_shift,int *cqe_size) +int *cqe_size) {struct mlx5_ib_resize_cq ucmd;struct ib_umem *umem;
@@ -1156,8 +1165,6 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,return err; } -mlx5_ib_cont_pages (umem, ucmd.buf_addr,0 , page_shift); - cq->resize_umem = umem; *cqe_size = ucmd.cqe_size;
@@ -1250,7 +1257,8 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)int err;int npas;__be64 *pas; -int page_shift; +unsigned int page_offset_quantized =0 ; +unsigned int page_shift;int inlen;int cqe_size;unsigned long flags;
@@ -1277,11 +1285,22 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)mutex_lock (&cq->resize_mutex);if (udata) { - err =resize_user (dev, cq, entries, udata, &page_shift, - &cqe_size); +unsigned long page_size; + + err =resize_user (dev, cq, entries, udata, &cqe_size);if (err) goto ex; - npas =ib_umem_num_dma_blocks (cq->resize_umem,1UL << page_shift); + + page_size =mlx5_umem_find_best_cq_quantized_pgoff ( + cq->resize_umem, cqc, log_page_size, +MLX5_ADAPTER_PAGE_SHIFT , page_offset,64 , + &page_offset_quantized); +if (!page_size) { + err = -EINVAL ; + goto ex_resize; + } + npas =ib_umem_num_dma_blocks (cq->resize_umem, page_size); + page_shift =order_base_2 (page_size); }else {struct mlx5_frag_buf *frag_buf;
@@ -1320,6 +1339,7 @@ int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)MLX5_SET (cqc, cqc, log_page_size, page_shift -MLX5_ADAPTER_PAGE_SHIFT ); +MLX5_SET (cqc, cqc, page_offset, page_offset_quantized); MLX5_SET(cqc, cqc, cqe_sz, cqe_sz_to_mlx_sz(cqe_size, cq->private_flags &
drivers/infiniband/hw/mlx5/mlx5_ib.h +10 -0
@@ -100,6 +100,16 @@ unsigned long __mlx5_umem_find_best_quantized_pgoff( GENMASK(31, order_base_2(scale)), scale, \ page_offset_quantized) +#define mlx5_umem_find_best_cq_quantized_pgoff (umem, typ, log_pgsz_fld, \ + pgsz_shift, page_offset_fld, \ + scale, page_offset_quantized) \ + __mlx5_umem_find_best_quantized_pgoff( \ + umem, \ + __mlx5_log_page_size_to_bitmap( \ + __mlx5_bit_sz(typ, log_pgsz_fld), pgsz_shift), \ + __mlx5_bit_sz(typ, page_offset_fld), 0, scale, \ + page_offset_quantized) +enum {MLX5_IB_MMAP_OFFSET_START =9 ,MLX5_IB_MMAP_OFFSET_END =255 ,
Signature · 1 line
2.28.0
PATCH 6/7 RDMA/mlx5: Use ib_umem_find_best_pgsz() for devx
In reply to Leon RomanovskyFrom: Jason Gunthorpe <[email protected]>
Since devx uses the new rdma_for_each_block() to fill the PAS it can also use ib_umem_find_best_pgsz().
However, the umem constructionin devx is complicated, the umem must still respect all the HW limits such as page_offset_quantized and the IOVA alignment.
Since we don't know what the user intends to use the umem for we have to compute and combine all the HW limits together. In practice this means umems are unlikely to use larger page sizes.
There are users trying to mix umem's with mkeys so this makes them work reliably, at least for an identity IOVA, by ensuring the IOVA matches the selected page size.
Last user of mlx5_ib_get_buf_offset() so it can also be removed.
--- drivers/infiniband/hw/mlx5/devx.c | 66 +++++++++++++++------------- drivers/infiniband/hw/mlx5/mem.c | 55 ----------------------- drivers/infiniband/hw/mlx5/mlx5_ib.h | 5 --- 3 files changed, 35 insertions(+), 91 deletions(-)
drivers/infiniband/hw/mlx5/devx.c +35 -31
@@ -93,8 +93,6 @@ struct devx_async_event_file {struct devx_umem {struct mlx5_core_dev *mdev;struct ib_umem *umem; -u32 page_offset; -int page_shift;u32 dinlen;u32 dinbox[MLX5_ST_SZ_DW (general_obj_in_cmd_hdr)]; };
@@ -2057,7 +2055,6 @@ static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,size_t size;u32 access;int err; -u32 page_mask;if (uverbs_copy_from (&addr, attrs,MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR ) ||uverbs_copy_from (&size, attrs,MLX5_IB_ATTR_DEVX_UMEM_REG_LEN ))
@@ -2078,46 +2075,55 @@ static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext, obj->umem =ib_umem_get (&dev->ib_dev, addr, size, access);if (IS_ERR (obj->umem))return PTR_ERR (obj->umem); - -mlx5_ib_cont_pages (obj->umem, obj->umem->address, -MLX5_MKEY_PAGE_SHIFT_MASK , &obj->page_shift); - page_mask = (1 << obj->page_shift) -1 ; - obj->page_offset = obj->umem->address & page_mask; -return 0 ; } -static int devx_umem_reg_cmd_alloc (struct uverbs_attr_bundle *attrs, +static int devx_umem_reg_cmd_alloc (struct mlx5_ib_dev *dev, +struct uverbs_attr_bundle *attrs,struct devx_umem *obj,struct devx_umem_reg_cmd *cmd) { - cmd->inlen = -MLX5_ST_SZ_BYTES (create_umem_in) + - (MLX5_ST_SZ_BYTES (mtt) * -ib_umem_num_dma_blocks (obj->umem,1UL << obj->page_shift)); - cmd->in =uverbs_zalloc (attrs, cmd->inlen); -return PTR_ERR_OR_ZERO (cmd->in); -} - -static void devx_umem_reg_cmd_build (struct mlx5_ib_dev *dev, -struct devx_umem *obj, -struct devx_umem_reg_cmd *cmd) -{ -void *umem; +unsigned int page_size;__be64 *mtt; +void *umem; + +/* +* We don't know what the user intends to use this umem for, but the HW +* restrictions must be met. MR, doorbell records, QP, WQ and CQ all +* have different requirements. Since we have no idea how to sort this +* out, only support PAGE_SIZE with the expectation that userspace will +* provide the necessary alignments inside the known PAGE_SIZE and that +* FW will check everything. +*/ + page_size =ib_umem_find_best_pgoff ( + obj->umem,PAGE_SIZE , +__mlx5_page_offset_to_bitmask (__mlx5_bit_sz (umem, page_offset), +0 )); +if (!page_size) +return -EINVAL ; + + cmd->inlen =MLX5_ST_SZ_BYTES (create_umem_in) + + (MLX5_ST_SZ_BYTES (mtt) * +ib_umem_num_dma_blocks (obj->umem, page_size)); + cmd->in =uverbs_zalloc (attrs, cmd->inlen); +if (!cmd->in) +return PTR_ERR (cmd->in); umem =MLX5_ADDR_OF (create_umem_in, cmd->in, umem); mtt = (__be64 *)MLX5_ADDR_OF (umem, umem, mtt);MLX5_SET (create_umem_in, cmd->in, opcode,MLX5_CMD_OP_CREATE_UMEM );MLX5_SET64 (umem, umem, num_of_mtt, -ib_umem_num_dma_blocks (obj->umem,1UL << obj->page_shift)); -MLX5_SET (umem, umem, log_page_size, obj->page_shift - -MLX5_ADAPTER_PAGE_SHIFT ); -MLX5_SET (umem, umem, page_offset, obj->page_offset); -mlx5_ib_populate_pas (obj->umem,1UL << obj->page_shift, mtt, +ib_umem_num_dma_blocks (obj->umem, page_size)); +MLX5_SET (umem, umem, log_page_size, +order_base_2 (page_size) -MLX5_ADAPTER_PAGE_SHIFT ); +MLX5_SET (umem, umem, page_offset, +ib_umem_dma_offset (obj->umem, page_size)); + +mlx5_ib_populate_pas (obj->umem, page_size, mtt, (obj->umem->writable ?MLX5_IB_MTT_WRITE :0 ) |MLX5_IB_MTT_READ ); +return 0 ; } static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
@@ -2144,12 +2150,10 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(if (err) goto err_obj_free; - err =devx_umem_reg_cmd_alloc (attrs, obj, &cmd); + err =devx_umem_reg_cmd_alloc (dev, attrs, obj, &cmd);if (err) goto err_umem_release; -devx_umem_reg_cmd_build (dev, obj, &cmd); -MLX5_SET (create_umem_in, cmd.in, uid, c->devx_uid); err =mlx5_cmd_exec (dev->mdev, cmd.in, cmd.inlen, cmd.out,sizeof (cmd.out));
drivers/infiniband/hw/mlx5/mem.c +0 -55
61 lines not on this page · raw message
drivers/infiniband/hw/mlx5/mlx5_ib.h +0 -5
@@ -40,8 +40,6 @@#define MLX5_IB_DEFAULT_UIDX 0xffffff#define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index) -#define MLX5_MKEY_PAGE_SHIFT_MASK __mlx5_mask(mkc, log_page_size) -static __always_inline unsigned long __mlx5_log_page_size_to_bitmap (unsigned int log_pgsz_bits,unsigned int pgsz_shift)
@@ -1296,9 +1294,6 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props);int mlx5_ib_query_port (struct ib_device *ibdev,u8 port,struct ib_port_attr *props); -void mlx5_ib_cont_pages (struct ib_umem *umem,u64 addr, -unsigned long max_page_shift, -int *shift);void mlx5_ib_populate_pas (struct ib_umem *umem,size_t page_size,__be64 *pas,u64 access_flags);void mlx5_ib_copy_pas (u64 *old,u64 *new,int step,int num);
Signature · 1 line
2.28.0
PATCH 7/7 RDMA/mlx5: Lower setting the umem's PAS for SRQ
In reply to Leon RomanovskyFrom: Jason Gunthorpe <[email protected]>
Some of the SRQ types are created using a WQ, and the WQ requires a different parameter set to mlx5_umem_find_best_quantized_pgoff() as it has a 5 bit page_offset.
Add the umem to the mlx5_srq_attr and defer computing the PAS data until the code has figured out what kind of mailbox to use. Compute the PAS directly from the umem for each of the four unique mailbox types.
This also avoids allocating memory to store the user PAS, instead it is written directly to the mailbox as in most other cases.
--- drivers/infiniband/hw/mlx5/srq.c | 27 +--------- drivers/infiniband/hw/mlx5/srq.h | 1 + drivers/infiniband/hw/mlx5/srq_cmd.c | 80 ++++++++++++++++++++++++++-- 3 files changed, 79 insertions(+), 29 deletions(-)
drivers/infiniband/hw/mlx5/srq.c +2 -25
@@ -51,8 +51,6 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, udata, struct mlx5_ib_ucontext, ibucontext);size_t ucmdlen;int err; -unsigned int page_offset_quantized; -unsigned int page_size;u32 uidx =MLX5_IB_DEFAULT_UIDX ; ucmdlen =min (udata->inlen,sizeof (ucmd));
@@ -84,32 +82,14 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, err =PTR_ERR (srq->umem);return err; } - - page_size =mlx5_umem_find_best_quantized_pgoff ( - srq->umem, srqc, log_page_size,MLX5_ADAPTER_PAGE_SHIFT , - page_offset,64 , &page_offset_quantized); -if (!page_size) { -mlx5_ib_warn (dev,"bad offset\n" ); - goto err_umem; - } - - in->pas =kvcalloc (ib_umem_num_dma_blocks (srq->umem, page_size), -sizeof (*in->pas),GFP_KERNEL ); -if (!in->pas) { - err = -ENOMEM ; - goto err_umem; - } - -mlx5_ib_populate_pas (srq->umem, page_size, in->pas,0 ); + in->umem = srq->umem; err =mlx5_ib_db_map_user (ucontext, udata, ucmd.db_addr, &srq->db);if (err) {mlx5_ib_dbg (dev,"map doorbell failed\n" ); - goto err_in; + goto err_umem; } - in->log_page_size =order_base_2 (page_size) -MLX5_ADAPTER_PAGE_SHIFT ; - in->page_offset = page_offset_quantized; in->uid = (in->type !=IB_SRQT_XRC ) ?to_mpd (pd)->uid :0 ;if (MLX5_CAP_GEN (dev->mdev, cqe_version) ==MLX5_CQE_VERSION_V1 && in->type !=IB_SRQT_BASIC )
@@ -117,9 +97,6 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,return 0 ; -err_in: -kvfree (in->pas); - err_umem:ib_umem_release (srq->umem);
drivers/infiniband/hw/mlx5/srq.h +1 -0
@@ -28,6 +28,7 @@ struct mlx5_srq_attr {u32 user_index;u64 db_record;__be64 *pas; +struct ib_umem *umem;u32 tm_log_list_size;u32 tm_next_tag;u32 tm_hw_phase_cnt;
drivers/infiniband/hw/mlx5/srq_cmd.c +76 -4
146 lines not on this page · raw message
Signature · 1 line
2.28.0
Quoted 22 lines · Leon Romanovsky
From: Leon Romanovsky <[email protected]>
Changelog:
v1:
* Added patch for raw QP
* Fixed commit messages
v0: https://lore.kernel.org/lkml/[email protected]-------------------------
Quoted 1 line
From Jason:
Move the remaining cases working with umems to use versions of
ib_umem_find_best_pgsz()tailored to the calculations the devices requires.Unlike a MR there is no IOVA, instead a page offset from the starting page is possible, with various restrictions.
Compute the best page size to meet the
page_offsetrestrictions.Thanks
Quoted 8 lines
Jason Gunthorpe (7):
RDMA/mlx5: Useib_umem_find_best_pgoff()for SRQ RDMA/mlx5: Usemlx5_umem_find_best_quantized_pgoff()for WQ RDMA/mlx5: Directly compute the PAS list for raw QP RQ's RDMA/mlx5: Usemlx5_umem_find_best_quantized_pgoff()for QP RDMA/mlx5:mlx5_umem_find_best_quantized_pgoff()for CQ RDMA/mlx5: Useib_umem_find_best_pgsz()for devx RDMA/mlx5: Lower setting the umem's PAS for SRQ
Applied to for-next
Jason