SG fix together with update to RDMA umem
Subsystems (13)
v4 cover SG fix together with update to RDMA umem
From: Maor Gottlieb <[email protected]>
Changelog:
v4:
* Unify sg_free_table_entries with __sg_free_table
v3: https://lore.kernel.org/lkml/[email protected]/
* Rewrote to new API suggestion
* Split for more patches
v2: https://lore.kernel.org/lkml/[email protected]
* Changed implementation of first patch, based on our discussion with
* Christoph.
https://lore.kernel.org/lkml/[email protected]/
v1: https://lore.kernel.org/lkml/[email protected]/
* Fixed sg_page with a _dma_ API in the umem.c
v0: https://lore.kernel.org/lkml/[email protected]
Maor Gottlieb (3):
lib/scatterlist: Provide a dedicated function to support table append lib/scatterlist: Fix wrong update of orig_nents
RDMA: Use the sg_table directly and remove the opencoded version from
umem
drivers/gpu/drm/drm_prime.c | 13 +- drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 11 +- drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c | 14 +- drivers/infiniband/core/umem.c | 56 ++++--- drivers/infiniband/core/umem_dmabuf.c | 5 +- drivers/infiniband/hw/hns/hns_roce_db.c | 4 +- drivers/infiniband/hw/irdma/verbs.c | 2 +- drivers/infiniband/hw/mlx4/doorbell.c | 3 +- drivers/infiniband/hw/mlx4/mr.c | 4 +- drivers/infiniband/hw/mlx5/doorbell.c | 3 +- drivers/infiniband/hw/mlx5/mr.c | 3 +- drivers/infiniband/hw/qedr/verbs.c | 2 +- drivers/infiniband/sw/rdmavt/mr.c | 2 +- drivers/infiniband/sw/rxe/rxe_mr.c | 2 +- include/linux/scatterlist.h | 56 +++++-- include/rdma/ib_umem.h | 11 +- include/rdma/ib_verbs.h | 28 ++++ lib/scatterlist.c | 155 ++++++++++++-------- lib/sg_pool.c | 3 +- tools/testing/scatterlist/main.c | 38 +++-- 20 files changed, 258 insertions(+), 157 deletions(-)
Signature · 1 line
2.25.4
PATCH v4 1/3 lib/scatterlist: Provide a dedicated function to support table append
RDMA is the only in-kernel user that uses __sg_alloc_table_from_pages to append pages dynamically. In the next patch. That mode will be extended and that function will get more parameters. So separate it into a unique function to make such change more clear.
--- drivers/gpu/drm/drm_prime.c | 13 ++++--- drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 11 +++--- drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c | 14 +++----- drivers/infiniband/core/umem.c | 4 +-- include/linux/scatterlist.h | 39 ++++++++++++++++++--- lib/scatterlist.c | 36 ++++++++++--------- tools/testing/scatterlist/main.c | 25 +++++++++---- 7 files changed, 90 insertions(+), 52 deletions(-)
drivers/gpu/drm/drm_prime.c +6 -7
@@ -807,8 +807,8 @@ struct sg_table *drm_prime_pages_to_sg(struct drm_device *dev, struct page **pages, unsigned int nr_pages) {struct sg_table *sg; -struct scatterlist *sge;size_t max_segment =0 ; +int err; sg =kmalloc (sizeof (struct sg_table ),GFP_KERNEL );if (!sg)
@@ -818,13 +818,12 @@ struct sg_table *drm_prime_pages_to_sg(struct drm_device *dev, max_segment =dma_max_mapping_size (dev->dev);if (max_segment ==0 ) max_segment =UINT_MAX ; - sge =__sg_alloc_table_from_pages (sg, pages, nr_pages,0 , - nr_pages <<PAGE_SHIFT , - max_segment, -NULL ,0 ,GFP_KERNEL ); -if (IS_ERR (sge)) { + err =sg_alloc_table_from_pages_segment (sg, pages, nr_pages,0 , + nr_pages <<PAGE_SHIFT , + max_segment,GFP_KERNEL ); +if (err) {kfree (sg); - sg =ERR_CAST (sge); + sg =ERR_PTR (err); }return sg; }
drivers/gpu/drm/i915/gem/i915_gem_userptr.c +4 -7
@@ -133,7 +133,6 @@ static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)unsigned int max_segment =i915_sg_segment_size ();struct sg_table *st;unsigned int sg_page_sizes; -struct scatterlist *sg;struct page **pvec;int ret;
@@ -153,13 +152,11 @@ static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)spin_unlock (&i915->mm.notifier_lock); alloc_table: - sg =__sg_alloc_table_from_pages (st, pvec, num_pages,0 , - num_pages <<PAGE_SHIFT , max_segment, -NULL ,0 ,GFP_KERNEL ); -if (IS_ERR (sg)) { - ret =PTR_ERR (sg); + ret =sg_alloc_table_from_pages_segment (st, pvec, num_pages,0 , + num_pages <<PAGE_SHIFT , + max_segment,GFP_KERNEL ); +if (ret) goto err; - } ret =i915_gem_gtt_prepare_pages (obj, st);if (ret) {
drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c +5 -9
@@ -363,7 +363,6 @@ static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt)int ret =0 ;static size_t sgl_size;static size_t sgt_size; -struct scatterlist *sg;if (vmw_tt->mapped)return 0 ;
@@ -386,15 +385,12 @@ static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt)if (unlikely (ret !=0 ))return ret; - sg =__sg_alloc_table_from_pages (&vmw_tt->sgt, vsgt->pages, - vsgt->num_pages,0 , - (unsigned long ) vsgt->num_pages <<PAGE_SHIFT , -dma_get_max_seg_size (dev_priv->drm.dev), -NULL ,0 ,GFP_KERNEL ); -if (IS_ERR (sg)) { - ret =PTR_ERR (sg); + ret =sg_alloc_table_from_pages_segment ( + &vmw_tt->sgt, vsgt->pages, vsgt->num_pages,0 , + (unsigned long )vsgt->num_pages <<PAGE_SHIFT , +dma_get_max_seg_size (dev_priv->drm.dev),GFP_KERNEL ); +if (ret) goto out_sg_alloc_fail; - }if (vsgt->num_pages > vmw_tt->sgt.orig_nents) { uint64_t over_alloc =
drivers/infiniband/core/umem.c +2 -2
@@ -226,8 +226,8 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, cur_base += ret *PAGE_SIZE ; npages -= ret; - sg =__sg_alloc_table_from_pages (&umem->sg_head, page_list, ret, -0 , ret <<PAGE_SHIFT , + sg =sg_alloc_append_table_from_pages (&umem->sg_head, page_list, + ret,0 , ret <<PAGE_SHIFT ,ib_dma_max_seg_size (device), sg, npages,GFP_KERNEL ); umem->sg_nents = umem->sg_head.nents;
include/linux/scatterlist.h +35 -4
@@ -285,14 +285,45 @@ void sg_free_table(struct sg_table *);int __sg_alloc_table (struct sg_table *,unsigned int ,unsigned int ,struct scatterlist *,unsigned int ,gfp_t ,sg_alloc_fn *);int sg_alloc_table (struct sg_table *,unsigned int ,gfp_t ); -struct scatterlist *__sg_alloc_table_from_pages (struct sg_table *sgt, +struct scatterlist *sg_alloc_append_table_from_pages (struct sg_table *sgt,struct page **pages,unsigned int n_pages,unsigned int offset,unsigned long size,unsigned int max_segment,struct scatterlist *prv,unsigned int left_pages,gfp_t gfp_mask); -int sg_alloc_table_from_pages (struct sg_table *sgt,struct page **pages, -unsigned int n_pages,unsigned int offset, -unsigned long size,gfp_t gfp_mask); +int sg_alloc_table_from_pages_segment (struct sg_table *sgt,struct page **pages, +unsigned int n_pages,unsigned int offset, +unsigned long size, +unsigned int max_segment,gfp_t gfp_mask); + +/** +* sg_alloc_table_from_pages - Allocate and initialize an sg table from +* an array of pages +* @sgt: The sg table header to use +* @pages: Pointer to an array of page pointers +* @n_pages: Number of pages in the pages array +* @offset: Offset from start of the first page to the start of a buffer +* @size: Number of valid bytes in the buffer (after offset) +* @gfp_mask: GFP allocation mask +* +* Description: +* Allocate and initialize an sg table from a list of pages. Contiguous +* ranges of the pages are squashed into a single scatterlist node. A user +* may provide an offset at a start and a size of valid data in a buffer +* specified by the page array. The returned sg table is released by +* sg_free_table. +* +* Returns: +* 0 on success, negative error on failure +*/ +static inline int sg_alloc_table_from_pages (struct sg_table *sgt, +struct page **pages, +unsigned int n_pages, +unsigned int offset, +unsigned long size,gfp_t gfp_mask) +{ +return sg_alloc_table_from_pages_segment (sgt, pages, n_pages, offset, + size,UINT_MAX , gfp_mask); +}#ifdef CONFIG_SGL_ALLOC struct scatterlist *sgl_alloc_order(unsigned long long length,
lib/scatterlist.c +20 -16
@@ -397,7 +397,7 @@ static struct scatterlist *get_next_sg(struct sg_table *table, }/** -* __sg_alloc_table_from_pages - Allocate and initialize an sg table from +* sg_alloc_append_table_from_pages - Allocate and initialize an sg table from * an array of pages * @sgt: The sg table header to use * @pages: Pointer to an array of page pointers
@@ -425,7 +425,7 @@ static struct scatterlist *get_next_sg(struct sg_table *table,* If this function returns non-0 (eg failure), the caller must call * sg_free_table() to cleanup any leftover allocations. */ -struct scatterlist *__sg_alloc_table_from_pages(struct sg_table *sgt, +struct scatterlist *sg_alloc_append_table_from_pages(struct sg_table *sgt, struct page **pages, unsigned int n_pages, unsigned int offset, unsigned long size, unsigned int max_segment, struct scatterlist *prv, unsigned int left_pages,
@@ -520,36 +520,40 @@ struct scatterlist *__sg_alloc_table_from_pages(struct sg_table *sgt,sg_mark_end (s);return s; } -EXPORT_SYMBOL (__sg_alloc_table_from_pages); +EXPORT_SYMBOL (sg_alloc_append_table_from_pages);/** -* sg_alloc_table_from_pages - Allocate and initialize an sg table from -* an array of pages +* sg_alloc_table_from_pages_segment - Allocate and initialize an sg table from +* an array of pages and given maximum +* segment. * @sgt: The sg table header to use * @pages: Pointer to an array of page pointers * @n_pages: Number of pages in the pages array * @offset: Offset from start of the first page to the start of a buffer * @size: Number of valid bytes in the buffer (after offset) +* @max_segment: Maximum size of a scatterlist element in bytes * @gfp_mask: GFP allocation mask * * Description: * Allocate and initialize an sg table from a list of pages. Contiguous -* ranges of the pages are squashed into a single scatterlist node. A user -* may provide an offset at a start and a size of valid data in a buffer -* specified by the page array. The returned sg table is released by -* sg_free_table. +* ranges of the pages are squashed into a single scatterlist node up to the +* maximum size specified in @max_segment. A user may provide an offset at a +* start and a size of valid data in a buffer specified by the page array. * -* Returns: +* The returned sg table is released by sg_free_table. +* +* Returns: * 0 on success, negative error on failure */ -int sg_alloc_table_from_pages (struct sg_table *sgt,struct page **pages, -unsigned int n_pages,unsigned int offset, -unsigned long size,gfp_t gfp_mask) +int sg_alloc_table_from_pages_segment (struct sg_table *sgt,struct page **pages, +unsigned int n_pages,unsigned int offset, +unsigned long size,unsigned int max_segment, +gfp_t gfp_mask) { -return PTR_ERR_OR_ZERO (__sg_alloc_table_from_pages (sgt, pages, n_pages, - offset, size,UINT_MAX ,NULL ,0 , gfp_mask)); +return PTR_ERR_OR_ZERO (sg_alloc_append_table_from_pages (sgt, pages, + n_pages, offset, size, max_segment,NULL ,0 , gfp_mask)); } -EXPORT_SYMBOL (sg_alloc_table_from_pages); +EXPORT_SYMBOL (sg_alloc_table_from_pages_segment);#ifdef CONFIG_SGL_ALLOC
tools/testing/scatterlist/main.c +18 -7
@@ -87,28 +87,39 @@ int main(void)int left_pages = test->pfn_app ? test->num_pages :0 ;struct page *pages[MAX_PAGES ];struct sg_table st; -struct scatterlist *sg; +struct scatterlist *sg =NULL ; +int ret;set_pages (pages, test->pfn, test->num_pages); - sg =__sg_alloc_table_from_pages (&st, pages, test->num_pages,0 , - test->size, test->max_seg,NULL , left_pages,GFP_KERNEL ); -assert (PTR_ERR_OR_ZERO (sg) == test->alloc_ret); +if (test->pfn_app) { + sg =sg_alloc_append_table_from_pages ( + &st, pages, test->num_pages,0 , test->size, + test->max_seg,NULL , left_pages,GFP_KERNEL ); +assert (PTR_ERR_OR_ZERO (sg) == test->alloc_ret); + }else { + ret =sg_alloc_table_from_pages_segment ( + &st, pages, test->num_pages,0 , test->size, + test->max_seg,GFP_KERNEL ); +assert (ret == test->alloc_ret); + }if (test->alloc_ret)continue ;if (test->pfn_app) {set_pages (pages, test->pfn_app, test->num_pages); - sg =__sg_alloc_table_from_pages (&st, pages, test->num_pages,0 , - test->size, test->max_seg, sg,0 ,GFP_KERNEL ); + sg =sg_alloc_append_table_from_pages ( + &st, pages, test->num_pages,0 , test->size, + test->max_seg, sg,0 ,GFP_KERNEL );assert (PTR_ERR_OR_ZERO (sg) == test->alloc_ret); }VALIDATE (st.nents == test->expected_segments, &st, test);if (!test->pfn_app) -VALIDATE (st.orig_nents == test->expected_segments, &st, test); +VALIDATE (st.orig_nents == test->expected_segments, &st, + test);sg_free_table (&st); }
Signature · 1 line
2.25.4
PATCH v4 2/3 lib/scatterlist: Fix wrong update of orig_nents
In reply to Maor Gottlieborig_nents should represent the number of entries with pages, but __sg_alloc_table_from_pages sets orig_nents as the number of total entries in the table. This is wrong when the API is used for dynamic allocation where not all the table entries are mapped with pages. It wasn't observed until now, since RDMA umem who uses this API in the dynamic form doesn't use orig_nents implicit or explicit by the scatterlist APIs.
Fix it by changing the append API to track the SG append table state and have an API to free the append table according to the total number of entries in the table.
Now all APIs set orig_nents as number of enries with pages.
--- drivers/infiniband/core/umem.c | 34 +++++---- include/linux/scatterlist.h | 19 +++-- include/rdma/ib_umem.h | 1 + lib/scatterlist.c | 127 +++++++++++++++++++------------ lib/sg_pool.c | 3 +- tools/testing/scatterlist/main.c | 45 ++++++----- 6 files changed, 137 insertions(+), 92 deletions(-)
drivers/infiniband/core/umem.c +19 -15
@@ -59,7 +59,7 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dunpin_user_page_range_dirty_lock (sg_page (sg),DIV_ROUND_UP (sg->length,PAGE_SIZE ), make_dirty); -sg_free_table (&umem->sg_head); +sg_free_append_table (&umem->sgt_append); }/**
@@ -155,8 +155,7 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr,unsigned long dma_attr =0 ;struct mm_struct *mm;unsigned long npages; -int ret; -struct scatterlist *sg =NULL ; +int pinned, ret;unsigned int gup_flags =FOLL_WRITE ;/*
@@ -216,28 +215,33 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr,while (npages) {cond_resched (); - ret =pin_user_pages_fast (cur_base, + pinned =pin_user_pages_fast (cur_base, min_t(unsigned long, npages,PAGE_SIZE /sizeof (struct page *)), gup_flags |FOLL_LONGTERM , page_list); -if (ret <0 ) +if (pinned <0 ) { + ret = pinned; goto umem_release; + } - cur_base += ret *PAGE_SIZE ; - npages -= ret; - sg =sg_alloc_append_table_from_pages (&umem->sg_head, page_list, - ret,0 , ret <<PAGE_SHIFT , -ib_dma_max_seg_size (device), sg, npages, -GFP_KERNEL ); - umem->sg_nents = umem->sg_head.nents; -if (IS_ERR (sg)) { -unpin_user_pages_dirty_lock (page_list, ret,0 ); - ret =PTR_ERR (sg); + cur_base += pinned *PAGE_SIZE ; + npages -= pinned; + ret =sg_alloc_append_table_from_pages ( + &umem->sgt_append, page_list, pinned,0 , + pinned <<PAGE_SHIFT ,ib_dma_max_seg_size (device), + npages,GFP_KERNEL ); + umem->sg_nents = umem->sgt_append.sgt.nents; +if (ret) { +memcpy (&umem->sg_head.sgl, &umem->sgt_append.sgt, +sizeof (umem->sgt_append.sgt)); +unpin_user_pages_dirty_lock (page_list, pinned,0 ); goto umem_release; } } +memcpy (&umem->sg_head.sgl, &umem->sgt_append.sgt, +sizeof (umem->sgt_append.sgt));if (access &IB_ACCESS_RELAXED_ORDERING ) dma_attr |=DMA_ATTR_WEAK_ORDERING ;
include/linux/scatterlist.h +13 -6
@@ -39,6 +39,12 @@ struct sg_table {unsigned int orig_nents;/* original size of list */ }; +struct sg_append_table { +struct sg_table sgt;/* The scatter list table */ +struct scatterlist *prv;/* last populated sge in the table */ +unsigned int total_nents;/* Total entries in the table */ +}; +/* * Notes on SG table design. *
@@ -280,16 +286,17 @@ typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t);typedef void (sg_free_fn )(struct scatterlist *,unsigned int );void __sg_free_table (struct sg_table *,unsigned int ,unsigned int , -sg_free_fn *); +sg_free_fn *,unsigned int );void sg_free_table (struct sg_table *); +void sg_free_append_table (struct sg_append_table *sgt);int __sg_alloc_table (struct sg_table *,unsigned int ,unsigned int ,struct scatterlist *,unsigned int ,gfp_t ,sg_alloc_fn *);int sg_alloc_table (struct sg_table *,unsigned int ,gfp_t ); -struct scatterlist *sg_alloc_append_table_from_pages (struct sg_table *sgt, -struct page **pages,unsigned int n_pages,unsigned int offset, -unsigned long size,unsigned int max_segment, -struct scatterlist *prv,unsigned int left_pages, -gfp_t gfp_mask); +int sg_alloc_append_table_from_pages (struct sg_append_table *sgt, +struct page **pages,unsigned int n_pages, +unsigned int offset,unsigned long size, +unsigned int max_segment, +unsigned int left_pages,gfp_t gfp_mask); int sg_alloc_table_from_pages_segment(struct sg_table *sgt, struct page **pages, unsigned int n_pages, unsigned int offset, unsigned long size,
include/rdma/ib_umem.h +1 -0
@@ -26,6 +26,7 @@ struct ib_umem { u32 is_odp : 1; u32 is_dmabuf : 1;struct work_struct work; +struct sg_append_table sgt_append;struct sg_table sg_head;int nmap;unsigned int sg_nents;
lib/scatterlist.c +78 -49
245 lines not on this page · raw message
lib/sg_pool.c +2 -1
@@ -90,7 +90,8 @@ void sg_free_table_chained(struct sg_table *table,if (nents_first_chunk ==1 ) nents_first_chunk =0 ; -__sg_free_table (table,SG_CHUNK_SIZE , nents_first_chunk, sg_pool_free); +__sg_free_table (table,SG_CHUNK_SIZE , nents_first_chunk, sg_pool_free, + table->orig_nents); }EXPORT_SYMBOL_GPL (sg_free_table_chained);
tools/testing/scatterlist/main.c +24 -21
67 lines not on this page · raw message
Signature · 1 line
2.25.4
Jason Gunthorpe
maintainerRe: [PATCH rdma-next v4 2/3] lib/scatterlist: Fix wrong update of orig_nents
lib/scatterlist.c struct scatterlist *sg_alloc_append_table_from_pages(struct sg_table *sgt, · line 536
Quoted 8 lines · Maor Gottlieb
lib/scatterlist.c +3
-1@@ -514,11 +531,13 @@ struct scatterlist *sg_alloc_append_table_from_pages(struct sg_table *sgt, offset =0 ; cur_page = j; } - sgt->nents += added_nents; + sgt_append->sgt.nents += added_nents; + sgt_append->sgt.orig_nents = sgt_append->sgt.nents; + sgt_append->prv = s;
Why is nents being touched here?
Shouldn't it just be
sgt_append->sgt.orig_nents += added_nents;
sgt_append->prv = s;?
Let me know I can fix it
Jason
Re: [PATCH rdma-next v4 2/3] lib/scatterlist: Fix wrong update of orig_nents
Quoted 11 lines · Jason Gunthorpe
lib/scatterlist.c struct scatterlist *sg_alloc_append_table_from_pages(struct sg_table *sgt, · line 536
Quoted 8 lines · Maor Gottlieb
lib/scatterlist.c +3
-1@@ -514,11 +531,13 @@ struct scatterlist *sg_alloc_append_table_from_pages(struct sg_table *sgt, offset =0 ; cur_page = j; } - sgt->nents += added_nents; + sgt_append->sgt.nents += added_nents; + sgt_append->sgt.orig_nents = sgt_append->sgt.nents; + sgt_append->prv = s;Why is nents being touched here?
Quoted 6 lines
Shouldn't it just be
sgt_append->sgt.orig_nents += added_nents; sgt_append->prv = s;?
In general yes, but all the sg_alloc_* functions update both orig_nents and nents (to same value). May be drivers that pass nents instead of orig_nents to dma_map_sg* (like umem did before this patch set). So I prefer to set it here as well, also not to break sg_alloc_table_from_pages_segment which use the same path. This kind of change could be done very carefully next cycle for all sg_alloc_*.
Quoted 4 lines
Let me know I can fix it
Jason
Jason Gunthorpe
maintainerRe: [PATCH rdma-next v4 2/3] lib/scatterlist: Fix wrong update of orig_nents
Quoted 20 lines · Maor Gottlieb
Quoted 11 lines · Jason Gunthorpe
lib/scatterlist.c struct scatterlist *sg_alloc_append_table_from_pages(struct sg_table *sgt, · line 536
Quoted 8 lines · Maor Gottlieb
lib/scatterlist.c +3
-1@@ -514,11 +531,13 @@ struct scatterlist *sg_alloc_append_table_from_pages(struct sg_table *sgt, offset =0 ; cur_page = j; } - sgt->nents += added_nents; + sgt_append->sgt.nents += added_nents; + sgt_append->sgt.orig_nents = sgt_append->sgt.nents; + sgt_append->prv = s;Why is nents being touched here?
Quoted 6 lines
Shouldn't it just be
sgt_append->sgt.orig_nents += added_nents; sgt_append->prv = s;?
Quoted 7 lines
In general yes, but all the sg_alloc_* functions update both
orig_nentsand nents (to same value). May be drivers that pass nents instead oforig_nentsto dma_map_sg* (like umem did before this patch set). So I prefer to set it here as well, also not to breaksg_alloc_table_from_pages_segmentwhich use the same path. This kind of change could be done very carefully next cycle for all sg_alloc_*.
Ok, interesting
Jason
PATCH v4 3/3 RDMA: Use the sg_table directly and remove the opencoded version from umem
In reply to Maor GottliebThis allows using the normal sg_table APIs and makes all the code cleaner. Remove sgt, nents and nmapd from ib_umem.
--- drivers/infiniband/core/umem.c | 32 +++++++++---------------- drivers/infiniband/core/umem_dmabuf.c | 5 ++-- drivers/infiniband/hw/hns/hns_roce_db.c | 4 ++-- drivers/infiniband/hw/irdma/verbs.c | 2 +- drivers/infiniband/hw/mlx4/doorbell.c | 3 ++- drivers/infiniband/hw/mlx4/mr.c | 4 ++-- drivers/infiniband/hw/mlx5/doorbell.c | 3 ++- drivers/infiniband/hw/mlx5/mr.c | 3 ++- drivers/infiniband/hw/qedr/verbs.c | 2 +- drivers/infiniband/sw/rdmavt/mr.c | 2 +- drivers/infiniband/sw/rxe/rxe_mr.c | 2 +- include/rdma/ib_umem.h | 10 ++++---- include/rdma/ib_verbs.h | 28 ++++++++++++++++++++++ 13 files changed, 59 insertions(+), 41 deletions(-)
drivers/infiniband/core/umem.c +11 -21
@@ -51,11 +51,11 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dstruct scatterlist *sg;unsigned int i; -if (umem->nmap >0 ) -ib_dma_unmap_sg (dev, umem->sg_head.sgl, umem->sg_nents, -DMA_BIDIRECTIONAL ); +if (dirty) +ib_dma_unmap_sgtable_attrs (dev, &umem->sgt_append.sgt, +DMA_BIDIRECTIONAL ,0 ); -for_each_sg (umem->sg_head.sgl, sg, umem->sg_nents, i) +for_each_sgtable_sg (&umem->sgt_append.sgt, sg, i)unpin_user_page_range_dirty_lock (sg_page (sg),DIV_ROUND_UP (sg->length,PAGE_SIZE ), make_dirty);
@@ -111,7 +111,7 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,/* offset into first SGL */ pgoff = umem->address & ~PAGE_MASK ; -for_each_sg (umem->sg_head.sgl, sg, umem->nmap, i) { +for_each_sgtable_dma_sg (&umem->sgt_append.sgt, sg, i) {/* Walk SGL and reduce max page size if VA/PA bits differ * for any address. */
@@ -121,7 +121,7 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,* the maximum possible page size as the low bits of the iova * must be zero when starting the next chunk. */ -if (i != (umem->nmap -1 )) +if (i != (umem->sgt_append.sgt.nents -1 )) mask |= va; pgoff =0 ; }
@@ -231,30 +231,19 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, &umem->sgt_append, page_list, pinned,0 , pinned <<PAGE_SHIFT ,ib_dma_max_seg_size (device), npages, GFP_KERNEL); - umem->sg_nents = umem->sgt_append.sgt.nents;if (ret) { -memcpy (&umem->sg_head.sgl, &umem->sgt_append.sgt, -sizeof (umem->sgt_append.sgt));unpin_user_pages_dirty_lock (page_list, pinned,0 ); goto umem_release; } } -memcpy (&umem->sg_head.sgl, &umem->sgt_append.sgt, -sizeof (umem->sgt_append.sgt));if (access &IB_ACCESS_RELAXED_ORDERING ) dma_attr |=DMA_ATTR_WEAK_ORDERING ; - umem->nmap = -ib_dma_map_sg_attrs (device, umem->sg_head.sgl, umem->sg_nents, -DMA_BIDIRECTIONAL , dma_attr); - -if (!umem->nmap) { - ret = -ENOMEM ; + ret =ib_dma_map_sgtable_attrs (device, &umem->sgt_append.sgt, +DMA_BIDIRECTIONAL , dma_attr); +if (ret) goto umem_release; - } - - ret =0 ; goto out; umem_release:
@@ -314,7 +303,8 @@ int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,return -EINVAL ; } - ret =sg_pcopy_to_buffer (umem->sg_head.sgl, umem->sg_nents, dst, length, + ret =sg_pcopy_to_buffer (umem->sgt_append.sgt.sgl, + umem->sgt_append.sgt.orig_nents, dst, length, offset +ib_umem_offset (umem));if (ret <0 )
drivers/infiniband/core/umem_dmabuf.c +2 -3
@@ -55,9 +55,8 @@ int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf) cur +=sg_dma_len (sg); } - umem_dmabuf->umem.sg_head.sgl = umem_dmabuf->first_sg; - umem_dmabuf->umem.sg_head.nents = nmap; - umem_dmabuf->umem.nmap = nmap; + umem_dmabuf->umem.sgt_append.sgt.sgl = umem_dmabuf->first_sg; + umem_dmabuf->umem.sgt_append.sgt.nents = nmap; umem_dmabuf->sgt = sgt; wait_fence:
drivers/infiniband/hw/hns/hns_roce_db.c +2 -2
@@ -42,8 +42,8 @@ int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt, found: offset = virt - page_addr; - db->dma =sg_dma_address (page->umem->sg_head.sgl) + offset; - db->virt_addr =sg_virt (page->umem->sg_head.sgl) + offset; + db->dma =sg_dma_address (page->umem->sgt_append.sgt.sgl) + offset; + db->virt_addr =sg_virt (page->umem->sgt_append.sgt.sgl) + offset; db->u.user_page = page;refcount_inc (&page->refcount);
drivers/infiniband/hw/irdma/verbs.c +1 -1
@@ -2231,7 +2231,7 @@ static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, pinfo = (level ==PBLE_LEVEL_1 ) ?NULL : palloc->level2.leaf;if (iwmr->type ==IRDMA_MEMREG_TYPE_QP ) - iwpbl->qp_mr.sq_page =sg_page (region->sg_head.sgl); + iwpbl->qp_mr.sq_page =sg_page (region->sgt_append.sgt.sgl);rdma_umem_for_each_dma_block (region, &biter, iwmr->page_size) { *pbl =rdma_block_iter_dma_address (&biter);
drivers/infiniband/hw/mlx4/doorbell.c +2 -1
@@ -75,7 +75,8 @@ int mlx4_ib_db_map_user(struct ib_udata *udata, unsigned long virt,list_add (&page->list, &context->db_page_list); found: - db->dma =sg_dma_address (page->umem->sg_head.sgl) + (virt & ~PAGE_MASK ); + db->dma =sg_dma_address (page->umem->sgt_append.sgt.sgl) + + (virt & ~PAGE_MASK ); db->u.user_page = page; ++page->refcnt;
drivers/infiniband/hw/mlx4/mr.c +2 -2
@@ -200,7 +200,7 @@ int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt, mtt_shift = mtt->page_shift; mtt_size =1ULL << mtt_shift; -for_each_sg (umem->sg_head.sgl, sg, umem->nmap, i) { +for_each_sgtable_dma_sg (&umem->sgt_append.sgt, sg, i) {if (cur_start_addr + len ==sg_dma_address (sg)) {/* still the same block */ len +=sg_dma_len (sg);
@@ -273,7 +273,7 @@ int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va, *num_of_mtts =ib_umem_num_dma_blocks (umem,PAGE_SIZE ); -for_each_sg (umem->sg_head.sgl, sg, umem->nmap, i) { +for_each_sgtable_dma_sg (&umem->sgt_append.sgt, sg, i) {/* * Initialization - save the first chunk start as the * current_block_start - block means contiguous pages.
drivers/infiniband/hw/mlx5/doorbell.c +2 -1
@@ -78,7 +78,8 @@ int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,list_add (&page->list, &context->db_page_list); found: - db->dma =sg_dma_address (page->umem->sg_head.sgl) + (virt & ~PAGE_MASK ); + db->dma =sg_dma_address (page->umem->sgt_append.sgt.sgl) + + (virt & ~PAGE_MASK ); db->u.user_page = page; ++page->refcnt;
drivers/infiniband/hw/mlx5/mr.c +2 -1
@@ -1226,7 +1226,8 @@ int mlx5_ib_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags) orig_sg_length = sg.length; cur_mtt = mtt; -rdma_for_each_block (mr->umem->sg_head.sgl, &biter, mr->umem->nmap, +rdma_for_each_block (mr->umem->sgt_append.sgt.sgl, &biter, + mr->umem->sgt_append.sgt.nents, BIT(mr->page_shift)) {if (cur_mtt == (void *)mtt + sg.length) { dma_sync_single_for_device(ddev, sg.addr, sg.length,
drivers/infiniband/hw/qedr/verbs.c +1 -1
@@ -1481,7 +1481,7 @@ static int qedr_init_srq_user_params(struct ib_udata *udata,return PTR_ERR (srq->prod_umem); } - sg = srq->prod_umem->sg_head.sgl; + sg = srq->prod_umem->sgt_append.sgt.sgl; srq->hw_srq.phy_prod_pair_addr =sg_dma_address (sg);return 0 ;
drivers/infiniband/sw/rdmavt/mr.c +1 -1
@@ -410,7 +410,7 @@ struct ib_mr *rvt_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, mr->mr.page_shift =PAGE_SHIFT ; m =0 ; n =0 ; - for_each_sg_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { + for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) {void *vaddr; vaddr =page_address (sg_page_iter_page (&sg_iter));
drivers/infiniband/sw/rxe/rxe_mr.c +1 -1
@@ -141,7 +141,7 @@ int rxe_mr_init_user(struct rxe_pd *pd, u64 start, u64 length, u64 iova,if (length >0 ) { buf = map[0 ]->buf; -for_each_sg_page (umem->sg_head.sgl, &sg_iter, umem->nmap,0 ) { + for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) {if (num_buf >=RXE_BUF_PER_MAP ) { map++; buf = map[0 ]->buf;
include/rdma/ib_umem.h +4 -6
@@ -27,9 +27,6 @@ struct ib_umem { u32 is_dmabuf : 1;struct work_struct work;struct sg_append_table sgt_append; -struct sg_table sg_head; -int nmap; -unsigned int sg_nents; };struct ib_umem_dmabuf {
@@ -57,7 +54,7 @@ static inline int ib_umem_offset(struct ib_umem *umem)static inline unsigned long ib_umem_dma_offset (struct ib_umem *umem,unsigned long pgsz) { -return (sg_dma_address (umem->sg_head.sgl) +ib_umem_offset (umem)) & +return (sg_dma_address (umem->sgt_append.sgt.sgl) +ib_umem_offset (umem)) & (pgsz -1 ); }
@@ -78,7 +75,8 @@ static inline void __rdma_umem_block_iter_start(struct ib_block_iter *biter,struct ib_umem *umem, unsigned long pgsz) { - __rdma_block_iter_start(biter, umem->sg_head.sgl, umem->nmap, pgsz); + __rdma_block_iter_start(biter, umem->sgt_append.sgt.sgl, + umem->sgt_append.sgt.nents, pgsz); }/**
@@ -129,7 +127,7 @@ static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, unsigned long pgsz_bitmap, u64 pgoff_bitmask) { -struct scatterlist *sg = umem->sg_head.sgl; +struct scatterlist *sg = umem->sgt_append.sgt.sgl;dma_addr_t dma_addr; dma_addr =sg_dma_address (sg) + (umem->address & ~PAGE_MASK );
include/rdma/ib_verbs.h +28 -0
@@ -4085,6 +4085,34 @@ static inline void ib_dma_unmap_sg_attrs(struct ib_device *dev, dma_attrs); } +/** +* ib_dma_map_sgtable_attrs - Map a scatter/gather table to DMA addresses +* @dev: The device for which the DMA addresses are to be created +* @sg: The sg_table object describing the buffer +* @direction: The direction of the DMA +* @attrs: Optional DMA attributes for the map operation +*/ +static inline int ib_dma_map_sgtable_attrs (struct ib_device *dev, +struct sg_table *sgt, +enum dma_data_direction direction, +unsigned long dma_attrs) +{ +if (ib_uses_virt_dma (dev)) { +ib_dma_virt_map_sg (dev, sgt->sgl, sgt->orig_nents); +return 0 ; + } +return dma_map_sgtable (dev->dma_device, sgt, direction, dma_attrs); +} + +static inline void ib_dma_unmap_sgtable_attrs (struct ib_device *dev, +struct sg_table *sgt, +enum dma_data_direction direction, +unsigned long dma_attrs) +{ +if (!ib_uses_virt_dma (dev)) +dma_unmap_sgtable (dev->dma_device, sgt, direction, dma_attrs); +} +/** * ib_dma_map_sg - Map a scatter/gather list to DMA addresses * @dev: The device for which the DMA addresses are to be created
Signature · 1 line
2.25.4
Patchwork
CI[Intel-gfx] ✗ Fi.CI.BUILD: failure for SG fix together with update to RDMA umem (rev4)
In reply to Maor Gottlieb== Series Details ==
== Series Details ==
Series: SG fix together with update to RDMA umem (rev4)
URL : https://patchwork.freedesktop.org/series/92682/
State : failure
== Summary ==
CALL scripts/checksyscalls.sh
CALL scripts/atomic/check-atomics.sh
DESCEND objtool
CHK include/generated/compile.h
CC [M] drivers/gpu/drm/i915/gem/i915_gem_ttm.odrivers/gpu/drm/i915/gem/i915_gem_ttm.c: In function ‘i915_ttm_tt_get_st’:drivers/gpu/drm/i915/gem/i915_gem_ttm.c:395:7: error: implicit declaration of function ‘__sg_alloc_table_from_pages’; did you mean ‘sg_alloc_table_from_pages’? [-Werror=implicit-function-declaration]
sg = __sg_alloc_table_from_pages
^~~~~~~~~~~~~~~~~~~~~~~~~~~
sg_alloc_table_from_pages
drivers/gpu/drm/i915/gem/i915_gem_ttm.c:395:5: error: assignment to ‘struct scatterlist *’ from ‘int’ makes pointer from integer without a cast [-Werror=int-conversion]
sg = __sg_alloc_table_from_pages
^cc1: all warnings being treated as errorsscripts/Makefile.build:271: recipe for target 'drivers/gpu/drm/i915/gem/i915_gem_ttm.o' failed make[4]: *** [drivers/gpu/drm/i915/gem/i915_gem_ttm.o] Error 1 scripts/Makefile.build:514: recipe for target 'drivers/gpu/drm/i915' failed make[3]: *** [drivers/gpu/drm/i915] Error 2scripts/Makefile.build:514: recipe for target 'drivers/gpu/drm' failed
make[2]: *** [drivers/gpu/drm] Error 2scripts/Makefile.build:514: recipe for target 'drivers/gpu' failed
make[1]: *** [drivers/gpu] Error 2
Makefile:1851: recipe for target 'drivers' failed
make: *** [drivers] Error 2
Leon Romanovsky
maintainerQuoted 35 lines · Maor Gottlieb
From: Maor Gottlieb <[email protected]>
Changelog:
v4:
* Unifysg_free_table_entrieswith__sg_free_table
v3: https://lore.kernel.org/lkml/[email protected]/
* Rewrote to new API suggestion
* Split for more patches
v2: https://lore.kernel.org/lkml/[email protected]
* Changed implementation of first patch, based on our discussion with
* Christoph.
https://lore.kernel.org/lkml/[email protected]/
v1: https://lore.kernel.org/lkml/[email protected]/
* Fixedsg_pagewith a _dma_ API in theumem.c
v0: https://lore.kernel.org/lkml/[email protected]Maor Gottlieb (3):
lib/scatterlist: Provide a dedicated function to support table append lib/scatterlist: Fix wrong update oforig_nents
RDMA: Use thesg_tabledirectly and remove the opencoded version from
umemdrivers/gpu/drm/drm_prime.c | 13 +- drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 11 +- drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c | 14 +- drivers/infiniband/core/umem.c | 56 ++++--- drivers/infiniband/core/umem_dmabuf.c | 5 +- drivers/infiniband/hw/hns/hns_roce_db.c | 4 +- drivers/infiniband/hw/irdma/verbs.c | 2 +- drivers/infiniband/hw/mlx4/doorbell.c | 3 +- drivers/infiniband/hw/mlx4/mr.c | 4 +- drivers/infiniband/hw/mlx5/doorbell.c | 3 +- drivers/infiniband/hw/mlx5/mr.c | 3 +- drivers/infiniband/hw/qedr/verbs.c | 2 +- drivers/infiniband/sw/rdmavt/mr.c | 2 +-
Quoted 8 lines
drivers/infiniband/sw/rxe/rxe_mr.c | 2 +- include/linux/scatterlist.h | 56 +++++-- include/rdma/ib_umem.h | 11 +- include/rdma/ib_verbs.h | 28 ++++ lib/scatterlist.c | 155 ++++++++++++-------- lib/sg_pool.c | 3 +- tools/testing/scatterlist/main.c | 38 +++-- 20 files changed, 258 insertions(+), 157 deletions(-)
Jason,
Did you add these patches to the -next? I can't find them.
Thanks
Quoted 4 lines
Signature · 2 lines
-- 2.25.4
Jason Gunthorpe
maintainerQuoted 45 lines · Leon Romanovsky
Quoted 35 lines · Maor Gottlieb
From: Maor Gottlieb <[email protected]>
Changelog:
v4:
* Unifysg_free_table_entrieswith__sg_free_table
v3: https://lore.kernel.org/lkml/[email protected]/
* Rewrote to new API suggestion
* Split for more patches
v2: https://lore.kernel.org/lkml/[email protected]
* Changed implementation of first patch, based on our discussion with
* Christoph.
https://lore.kernel.org/lkml/[email protected]/
v1: https://lore.kernel.org/lkml/[email protected]/
* Fixedsg_pagewith a _dma_ API in theumem.c
v0: https://lore.kernel.org/lkml/[email protected]Maor Gottlieb (3):
lib/scatterlist: Provide a dedicated function to support table append lib/scatterlist: Fix wrong update oforig_nents
RDMA: Use thesg_tabledirectly and remove the opencoded version from
umemdrivers/gpu/drm/drm_prime.c | 13 +- drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 11 +- drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c | 14 +- drivers/infiniband/core/umem.c | 56 ++++--- drivers/infiniband/core/umem_dmabuf.c | 5 +- drivers/infiniband/hw/hns/hns_roce_db.c | 4 +- drivers/infiniband/hw/irdma/verbs.c | 2 +- drivers/infiniband/hw/mlx4/doorbell.c | 3 +- drivers/infiniband/hw/mlx4/mr.c | 4 +- drivers/infiniband/hw/mlx5/doorbell.c | 3 +- drivers/infiniband/hw/mlx5/mr.c | 3 +- drivers/infiniband/hw/qedr/verbs.c | 2 +- drivers/infiniband/sw/rdmavt/mr.c | 2 +-Quoted 8 lines
drivers/infiniband/sw/rxe/rxe_mr.c | 2 +- include/linux/scatterlist.h | 56 +++++-- include/rdma/ib_umem.h | 11 +- include/rdma/ib_verbs.h | 28 ++++ lib/scatterlist.c | 155 ++++++++++++-------- lib/sg_pool.c | 3 +- tools/testing/scatterlist/main.c | 38 +++-- 20 files changed, 258 insertions(+), 157 deletions(-)
Quoted 3 lines
Jason,
Did you add these patches to the -next? I can't find them.
They sat in linux-next for awhile outside the rdma-next tree
All synced up now
Jason
Leon Romanovsky
maintainerQuoted 46 lines · Jason Gunthorpe
Quoted 45 lines · Leon Romanovsky
Quoted 43 lines · Maor Gottlieb
From: Maor Gottlieb <[email protected]>
Changelog:
v4:
* Unifysg_free_table_entrieswith__sg_free_table
v3: https://lore.kernel.org/lkml/[email protected]/
* Rewrote to new API suggestion
* Split for more patches
v2: https://lore.kernel.org/lkml/[email protected]
* Changed implementation of first patch, based on our discussion with
* Christoph.
https://lore.kernel.org/lkml/[email protected]/
v1: https://lore.kernel.org/lkml/[email protected]/
* Fixedsg_pagewith a _dma_ API in theumem.c
v0: https://lore.kernel.org/lkml/[email protected]Maor Gottlieb (3):
lib/scatterlist: Provide a dedicated function to support table append lib/scatterlist: Fix wrong update oforig_nents
RDMA: Use thesg_tabledirectly and remove the opencoded version from
umemdrivers/gpu/drm/drm_prime.c | 13 +- drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 11 +- drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c | 14 +- drivers/infiniband/core/umem.c | 56 ++++--- drivers/infiniband/core/umem_dmabuf.c | 5 +- drivers/infiniband/hw/hns/hns_roce_db.c | 4 +- drivers/infiniband/hw/irdma/verbs.c | 2 +- drivers/infiniband/hw/mlx4/doorbell.c | 3 +- drivers/infiniband/hw/mlx4/mr.c | 4 +- drivers/infiniband/hw/mlx5/doorbell.c | 3 +- drivers/infiniband/hw/mlx5/mr.c | 3 +- drivers/infiniband/hw/qedr/verbs.c | 2 +- drivers/infiniband/sw/rdmavt/mr.c | 2 +- drivers/infiniband/sw/rxe/rxe_mr.c | 2 +- include/linux/scatterlist.h | 56 +++++-- include/rdma/ib_umem.h | 11 +- include/rdma/ib_verbs.h | 28 ++++ lib/scatterlist.c | 155 ++++++++++++-------- lib/sg_pool.c | 3 +- tools/testing/scatterlist/main.c | 38 +++-- 20 files changed, 258 insertions(+), 157 deletions(-)
Quoted 7 lines
Quoted 3 lines
Jason,
Did you add these patches to the -next? I can't find them.
They sat in linux-next for awhile outside the rdma-next tree
All synced up now
Thanks
Quoted 2 lines
Jason