Move away from remap_pfn_range()
Subsystems (3)
Jens Axboe
maintainerv3 cover Move away from remap_pfn_range()
Hi,
This series switches both the ring, sqes, and kbuf side away from using remap_pfn_range().
Patch 1 is just a prep patch, and patches 2-3 add ring support, and patch 4 just unifies some identical code.
Patches 5-7 cleanup some kbuf side code, and patch 8 prepares buffer lists to be reference counted, and then patch 9 can finally switch kbuf to also use the nicer vm_insert_pages(). Patch 10 is a cleanup, and patch 11 moves the alloc/pin/map etc code into a separate file.
With this, no more remap_pfn_range(), and no more manual cleanup of having used it.
Changes since v2:
- Simplify references on compound pages (Johannes)
- Fix hunk of one patch being wrong (Jeff)
- Fix typo for !CONFIG_MMU (me)
include/linux/io_uring_types.h | 4 - io_uring/Makefile | 3 +- io_uring/io_uring.c | 246 +----------------------- io_uring/io_uring.h | 5 - io_uring/kbuf.c | 291 ++++++++-------------------- io_uring/kbuf.h | 11 +- io_uring/memmap.c | 333 +++++++++++++++++++++++++++++++++ io_uring/memmap.h | 25 +++ io_uring/rsrc.c | 37 +--- mm/nommu.c | 7 + 10 files changed, 467 insertions(+), 495 deletions(-)
Signature · 1 line
Jens Axboe
Jens Axboe
maintainerPATCH v3 1/11 mm: add nommu variant of vm_insert_pages()
An identical one exists for vm_insert_page(), add one for vm_insert_pages() to avoid needing to check for CONFIG_MMU in code using it.
--- mm/nommu.c | 7 +++++++ 1 file changed, 7 insertions(+)
mm/nommu.c +7 -0
@@ -355,6 +355,13 @@ int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, }EXPORT_SYMBOL (vm_insert_page); +int vm_insert_pages (struct vm_area_struct *vma,unsigned long addr, +struct page **pages,unsigned long *num) +{ +return -EINVAL ; +} +EXPORT_SYMBOL (vm_insert_pages); +int vm_map_pages (struct vm_area_struct *vma,struct page **pages,unsigned long num) {
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH v3 2/11 io_uring: get rid of remap_pfn_range() for mapping rings/sqes
In reply to Jens AxboeRather than use remap_pfn_range() for this and manually free later, switch to using vm_insert_pages() and have it Just Work.
If possible, allocate a single compound page that covers the range that is needed. If that works, then we can just use page_address() on that page. If we fail to get a compound page, allocate single pages and use vmap() to map them into the kernel virtual address space.
This just covers the rings/sqes, the other remaining user of the mmap remap_pfn_range() user will be converted separately. Once that is done, we can kill the old alloc/free code.
--- io_uring/io_uring.c | 136 +++++++++++++++++++++++++++++++++++++++++--- io_uring/io_uring.h | 2 + 2 files changed, 130 insertions(+), 8 deletions(-)
io_uring/io_uring.c +128 -8
@@ -2594,6 +2594,33 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,return READ_ONCE (rings->cq.head) ==READ_ONCE (rings->cq.tail) ? ret :0 ; } +static void io_pages_unmap (void *ptr,struct page ***pages, +unsigned short *npages) +{ +bool do_vunmap = false; + +if (*npages) { +struct page **to_free = *pages; +int i; + +/* +* Only did vmap for the non-compound multiple page case. +* For the compound page, we just need to put the head. +*/ +if (PageCompound (to_free[0 ])) + *npages =1 ; +else if (*npages >1 ) + do_vunmap = true; +for (i =0 ; i < *npages; i++) +put_page (to_free[i]); + } +if (do_vunmap) +vunmap (ptr); +kvfree (*pages); + *pages =NULL ; + *npages =0 ; +} +void io_mem_free (void *ptr) {if (!ptr)
@@ -2694,8 +2721,8 @@ static void *io_sqes_map(struct io_ring_ctx *ctx, unsigned long uaddr,static void io_rings_free (struct io_ring_ctx *ctx) {if (!(ctx->flags &IORING_SETUP_NO_MMAP )) { -io_mem_free (ctx->rings); -io_mem_free (ctx->sq_sqes); +io_pages_unmap (ctx->rings, &ctx->ring_pages, &ctx->n_ring_pages); +io_pages_unmap (ctx->sq_sqes, &ctx->sqe_pages, &ctx->n_sqe_pages); }else {io_pages_free (&ctx->ring_pages, ctx->n_ring_pages); ctx->n_ring_pages =0 ;
@@ -2707,6 +2734,80 @@ static void io_rings_free(struct io_ring_ctx *ctx) ctx->sq_sqes =NULL ; } +static void *io_mem_alloc_compound (struct page **pages,int nr_pages, +size_t size,gfp_t gfp) +{ +struct page *page; +int i, order; + + order =get_order (size); +if (order >MAX_PAGE_ORDER ) +return NULL ; +else if (order) + gfp |= __GFP_COMP; + + page =alloc_pages (gfp, order); +if (!page) +return NULL ; + +for (i =0 ; i < nr_pages; i++) + pages[i] = page + i; + +return page_address (page); +} + +static void *io_mem_alloc_single (struct page **pages,int nr_pages,size_t size, +gfp_t gfp) +{ +void *ret; +int i; + +for (i =0 ; i < nr_pages; i++) { + pages[i] =alloc_page (gfp); +if (!pages[i]) + goto err; + } + + ret =vmap (pages, nr_pages,VM_MAP ,PAGE_KERNEL ); +if (ret) +return ret; +err: +while (i--) +put_page (pages[i]); +return ERR_PTR (-ENOMEM ); +} + +static void *io_pages_map (struct page ***out_pages,unsigned short *npages, +size_t size) +{ +gfp_t gfp =GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN; +struct page **pages; +int nr_pages; +void *ret; + + nr_pages = (size +PAGE_SIZE -1 ) >>PAGE_SHIFT ; + pages =kvmalloc_array (nr_pages,sizeof (struct page *), gfp); +if (!pages) +return ERR_PTR (-ENOMEM ); + + ret =io_mem_alloc_compound (pages, nr_pages, size, gfp); +if (ret) + goto done; + + ret =io_mem_alloc_single (pages, nr_pages, size, gfp); +if (ret) { +done: + *out_pages = pages; + *npages = nr_pages; +return ret; + } + +kvfree (pages); + *out_pages =NULL ; + *npages =0 ; +return ERR_PTR (-ENOMEM ); +} +void *io_mem_alloc (size_t size) {gfp_t gfp =GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP;
@@ -3294,14 +3395,12 @@ static void *io_uring_validate_mmap_request(struct file *file,/* Don't allow mmap if the ring was setup without it */ if (ctx->flags &IORING_SETUP_NO_MMAP )return ERR_PTR (-EINVAL ); - ptr = ctx->rings; -break ; +return ctx->rings;case IORING_OFF_SQES :/* Don't allow mmap if the ring was setup without it */ if (ctx->flags &IORING_SETUP_NO_MMAP )return ERR_PTR (-EINVAL ); - ptr = ctx->sq_sqes; -break ; +return ctx->sq_sqes;case IORING_OFF_PBUF_RING : {unsigned int bgid;
@@ -3324,11 +3423,22 @@ static void *io_uring_validate_mmap_request(struct file *file,return ptr; } +int io_uring_mmap_pages (struct io_ring_ctx *ctx,struct vm_area_struct *vma, +struct page **pages,int npages) +{ +unsigned long nr_pages = npages; + +vm_flags_set (vma,VM_DONTEXPAND ); +return vm_insert_pages (vma, vma->vm_start, pages, &nr_pages); +} +#ifdef CONFIG_MMU static __cold int io_uring_mmap (struct file *file,struct vm_area_struct *vma) { +struct io_ring_ctx *ctx = file->private_data;size_t sz = vma->vm_end - vma->vm_start; +long offset = vma->vm_pgoff <<PAGE_SHIFT ;unsigned long pfn;void *ptr;
@@ -3336,6 +3446,16 @@ static __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma)if (IS_ERR (ptr))return PTR_ERR (ptr); +switch (offset &IORING_OFF_MMAP_MASK ) { +case IORING_OFF_SQ_RING : +case IORING_OFF_CQ_RING : +return io_uring_mmap_pages (ctx, vma, ctx->ring_pages, + ctx->n_ring_pages); +case IORING_OFF_SQES : +return io_uring_mmap_pages (ctx, vma, ctx->sqe_pages, + ctx->n_sqe_pages); + } + pfn =virt_to_phys (ptr) >>PAGE_SHIFT ;return remap_pfn_range (vma, vma->vm_start, pfn, sz, vma->vm_page_prot); }
@@ -3625,7 +3745,7 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,return -EOVERFLOW ;if (!(ctx->flags &IORING_SETUP_NO_MMAP )) - rings =io_mem_alloc (size); + rings =io_pages_map (&ctx->ring_pages, &ctx->n_ring_pages, size);else rings =io_rings_map (ctx, p->cq_off.user_addr, size);
@@ -3650,7 +3770,7 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, }if (!(ctx->flags &IORING_SETUP_NO_MMAP )) - ptr =io_mem_alloc (size); + ptr =io_pages_map (&ctx->sqe_pages, &ctx->n_sqe_pages, size);else ptr =io_sqes_map (ctx, p->sq_off.user_addr, size);
io_uring/io_uring.h +2 -0
@@ -70,6 +70,8 @@ bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags);void __io_commit_cqring_flush (struct io_ring_ctx *ctx);struct page **io_pin_pages (unsigned long ubuf,unsigned long len,int *npages); +int io_uring_mmap_pages (struct io_ring_ctx *ctx,struct vm_area_struct *vma, +struct page **pages,int npages);struct file *io_file_get_normal (struct io_kiocb *req,int fd); struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH v3 3/11 io_uring: use vmap() for ring mapping
In reply to Jens AxboeThis is the last holdout which does odd page checking, convert it to vmap just like what is done for the non-mmap path.
--- io_uring/io_uring.c | 38 +++++++++----------------------------- 1 file changed, 9 insertions(+), 29 deletions(-)
io_uring/io_uring.c +9 -29
@@ -63,7 +63,6 @@#include <linux/sched/mm.h> #include <linux/uaccess.h> #include <linux/nospec.h> -#include <linux/highmem.h> #include <linux/fsnotify.h> #include <linux/fadvise.h> #include <linux/task_work.h>
@@ -2649,7 +2648,7 @@ static void *__io_uaddr_map(struct page ***pages, unsigned short *npages,struct page **page_array;unsigned int nr_pages;void *page_addr; -int ret, i, pinned; +int ret, pinned; *npages =0 ;
@@ -2671,34 +2670,13 @@ static void *__io_uaddr_map(struct page ***pages, unsigned short *npages, goto free_pages; } - page_addr =page_address (page_array[0 ]); -for (i =0 ; i < nr_pages; i++) { - ret = -EINVAL ; - -/* -* Can't support mapping user allocated ring memory on 32-bit -* archs where it could potentially reside in highmem. Just -* fail those with -EINVAL, just like we did on kernels that -* didn't support this feature. -*/ -if (PageHighMem (page_array[i])) - goto free_pages; - -/* -* No support for discontig pages for now, should either be a -* single normal page, or a huge page. Later on we can add -* support for remapping discontig pages, for now we will -* just fail them with EINVAL. -*/ -if (page_address (page_array[i]) != page_addr) - goto free_pages; - page_addr +=PAGE_SIZE ; + page_addr =vmap (page_array, nr_pages,VM_MAP ,PAGE_KERNEL ); +if (page_addr) { + *pages = page_array; + *npages = nr_pages; +return page_addr; } - - *pages = page_array; - *npages = nr_pages; -return page_to_virt (page_array[0 ]); - + ret = -ENOMEM ; free_pages:io_pages_free (&page_array, pinned >0 ? pinned :0 );return ERR_PTR (ret);
@@ -2728,6 +2706,8 @@ static void io_rings_free(struct io_ring_ctx *ctx) ctx->n_ring_pages =0 ;io_pages_free (&ctx->sqe_pages, ctx->n_sqe_pages); ctx->n_sqe_pages =0 ; +vunmap (ctx->rings); +vunmap (ctx->sq_sqes); } ctx->rings =NULL ;
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH v3 4/11 io_uring: unify io_pin_pages()
In reply to Jens AxboeMove it into io_uring.c where it belongs, and use it in there as well rather than have two implementations of this.
--- io_uring/io_uring.c | 61 +++++++++++++++++++++++++++++++-------------- io_uring/rsrc.c | 36 -------------------------- 2 files changed, 42 insertions(+), 55 deletions(-)
io_uring/io_uring.c +42 -19
@@ -2642,33 +2642,57 @@ static void io_pages_free(struct page ***pages, int npages) *pages =NULL ; } +struct page **io_pin_pages (unsigned long uaddr,unsigned long len,int *npages) +{ +unsigned long start, end, nr_pages; +struct page **pages; +int ret; + + end = (uaddr + len +PAGE_SIZE -1 ) >>PAGE_SHIFT ; + start = uaddr >>PAGE_SHIFT ; + nr_pages = end - start; +if (WARN_ON_ONCE (!nr_pages)) +return ERR_PTR (-EINVAL ); + + pages =kvmalloc_array (nr_pages,sizeof (struct page *),GFP_KERNEL ); +if (!pages) +return ERR_PTR (-ENOMEM ); + + ret =pin_user_pages_fast (uaddr, nr_pages,FOLL_WRITE |FOLL_LONGTERM , + pages); +/* success, mapped all pages */ +if (ret == nr_pages) { + *npages = nr_pages; +return pages; + } + +/* partial map, or didn't map anything */ +if (ret >=0 ) { +/* if we did partial map, release any pages we did get */ +if (ret) +unpin_user_pages (pages, ret); + ret = -EFAULT ; + } +kvfree (pages); +return ERR_PTR (ret); +} +static void *__io_uaddr_map (struct page ***pages,unsigned short *npages,unsigned long uaddr,size_t size) {struct page **page_array;unsigned int nr_pages;void *page_addr; -int ret, pinned; *npages =0 ;if (uaddr & (PAGE_SIZE -1 ) || !size)return ERR_PTR (-EINVAL ); - nr_pages = (size +PAGE_SIZE -1 ) >>PAGE_SHIFT ; -if (nr_pages >USHRT_MAX ) -return ERR_PTR (-EINVAL ); - page_array =kvmalloc_array (nr_pages,sizeof (struct page *),GFP_KERNEL ); -if (!page_array) -return ERR_PTR (-ENOMEM ); - - - pinned =pin_user_pages_fast (uaddr, nr_pages,FOLL_WRITE |FOLL_LONGTERM , - page_array); -if (pinned != nr_pages) { - ret = (pinned <0 ) ? pinned : -EFAULT ; - goto free_pages; - } + nr_pages =0 ; + page_array =io_pin_pages (uaddr, size, &nr_pages); +if (IS_ERR (page_array)) +return page_array; page_addr =vmap (page_array, nr_pages,VM_MAP ,PAGE_KERNEL );if (page_addr) {
@@ -2676,10 +2700,9 @@ static void *__io_uaddr_map(struct page ***pages, unsigned short *npages, *npages = nr_pages;return page_addr; } - ret = -ENOMEM ; -free_pages: -io_pages_free (&page_array, pinned >0 ? pinned :0 ); -return ERR_PTR (ret); + +io_pages_free (&page_array, nr_pages); +return ERR_PTR (-ENOMEM ); } static void *io_rings_map(struct io_ring_ctx *ctx, unsigned long uaddr,
io_uring/rsrc.c +0 -36
@@ -870,42 +870,6 @@ static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,return ret; } -struct page **io_pin_pages (unsigned long ubuf,unsigned long len,int *npages) -{ -unsigned long start, end, nr_pages; -struct page **pages =NULL ; -int ret; - - end = (ubuf + len +PAGE_SIZE -1 ) >>PAGE_SHIFT ; - start = ubuf >>PAGE_SHIFT ; - nr_pages = end - start; -WARN_ON (!nr_pages); - - pages =kvmalloc_array (nr_pages,sizeof (struct page *),GFP_KERNEL ); -if (!pages) -return ERR_PTR (-ENOMEM ); - -mmap_read_lock (current->mm); - ret =pin_user_pages (ubuf, nr_pages,FOLL_WRITE |FOLL_LONGTERM , pages); -mmap_read_unlock (current->mm); - -/* success, mapped all pages */ -if (ret == nr_pages) { - *npages = nr_pages; -return pages; - } - -/* partial map, or didn't map anything */ -if (ret >=0 ) { -/* if we did partial map, release any pages we did get */ -if (ret) -unpin_user_pages (pages, ret); - ret = -EFAULT ; - } -kvfree (pages); -return ERR_PTR (ret); -} -static int io_sqe_buffer_register (struct io_ring_ctx *ctx,struct iovec *iov,struct io_mapped_ubuf **pimu,struct page **last_hpage)
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH v3 5/11 io_uring/kbuf: get rid of lower BGID lists
In reply to Jens AxboeJust rely on the xarray for any kind of bgid. This simplifies things, and it really doesn't bring us much, if anything.
--- include/linux/io_uring_types.h | 1 - io_uring/io_uring.c | 2 - io_uring/kbuf.c | 70 ++++------------------------------ 3 files changed, 8 insertions(+), 65 deletions(-)
include/linux/io_uring_types.h +0 -1
@@ -295,7 +295,6 @@ struct io_ring_ctx {struct io_submit_state submit_state; -struct io_buffer_list *io_bl;struct xarray io_bl_xa;struct io_hash_table cancel_table_locked;
io_uring/io_uring.c +0 -2
@@ -353,7 +353,6 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)io_futex_cache_free (ctx);kfree (ctx->cancel_table.hbs);kfree (ctx->cancel_table_locked.hbs); -kfree (ctx->io_bl);xa_destroy (&ctx->io_bl_xa);kfree (ctx);return NULL ;
@@ -2928,7 +2927,6 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)io_napi_free (ctx);kfree (ctx->cancel_table.hbs);kfree (ctx->cancel_table_locked.hbs); -kfree (ctx->io_bl);xa_destroy (&ctx->io_bl_xa);kfree (ctx); }
io_uring/kbuf.c +8 -62
138 lines not on this page · raw message
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH v3 6/11 io_uring/kbuf: get rid of bl->is_ready
In reply to Jens AxboeNow that xarray is being exclusively used for the buffer_list lookup, this check is no longer needed. Get rid of it and the is_ready member.
--- io_uring/kbuf.c | 8 -------- io_uring/kbuf.h | 2 -- 2 files changed, 10 deletions(-)
io_uring/kbuf.c +0 -8
@@ -61,7 +61,6 @@ static int io_buffer_add_list(struct io_ring_ctx *ctx,* always under the ->uring_lock, but the RCU lookup from mmap does. */ bl->bgid = bgid; -smp_store_release (&bl->is_ready,1 );return xa_err (xa_store (&ctx->io_bl_xa, bgid, bl,GFP_KERNEL )); }
@@ -721,13 +720,6 @@ void *io_pbuf_get_address(struct io_ring_ctx *ctx, unsigned long bgid)if (!bl || !bl->is_mmap)return NULL ; -/* -* Ensure the list is fully setup. Only strictly needed for RCU lookup -* via mmap, and in that case only for the array indexed groups. For -* the xarray lookups, it's either visible and ready, or not at all. -*/ -if (!smp_load_acquire (&bl->is_ready)) -return NULL ;return bl->buf_ring; }
io_uring/kbuf.h +0 -2
@@ -29,8 +29,6 @@ struct io_buffer_list {__u8 is_buf_ring;/* ring mapped provided buffers, but mmap'ed by application */ __u8 is_mmap; -/* bl is visible from an RCU point of view for lookup */ -__u8 is_ready; };struct io_buffer {
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH v3 7/11 io_uring/kbuf: vmap pinned buffer ring
In reply to Jens AxboeThis avoids needing to care about HIGHMEM, and it makes the buffer indexing easier as both ring provided buffer methods are now virtually mapped in a contigious fashion.
--- io_uring/kbuf.c | 39 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 24 deletions(-)
io_uring/kbuf.c +15 -24
@@ -7,6 +7,7 @@#include <linux/slab.h> #include <linux/namei.h> #include <linux/poll.h> +#include <linux/vmalloc.h> #include <linux/io_uring.h> #include <uapi/linux/io_uring.h>
@@ -145,15 +146,7 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len, req->flags |=REQ_F_BL_EMPTY ; head &= bl->mask; -/* mmaped buffers are always contig */ -if (bl->is_mmap || head <IO_BUFFER_LIST_BUF_PER_PAGE ) { - buf = &br->bufs[head]; - }else { -int off = head & (IO_BUFFER_LIST_BUF_PER_PAGE -1 ); -int index = head /IO_BUFFER_LIST_BUF_PER_PAGE ; - buf =page_address (bl->buf_pages[index]); - buf += off; - } + buf = &br->bufs[head];if (*len ==0 || *len > buf->len) *len = buf->len; req->flags |=REQ_F_BUFFER_RING ;
@@ -240,6 +233,7 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx,for (j =0 ; j < bl->buf_nr_pages; j++)unpin_user_page (bl->buf_pages[j]);kvfree (bl->buf_pages); +vunmap (bl->buf_ring); bl->buf_pages =NULL ; bl->buf_nr_pages =0 ; }
@@ -490,9 +484,9 @@ int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)static int io_pin_pbuf_ring (struct io_uring_buf_reg *reg,struct io_buffer_list *bl) { -struct io_uring_buf_ring *br; +struct io_uring_buf_ring *br =NULL ; +int nr_pages, ret, i;struct page **pages; -int i, nr_pages; pages = io_pin_pages(reg->ring_addr, flex_array_size(br, bufs, reg->ring_entries),
@@ -500,18 +494,12 @@ static int io_pin_pbuf_ring(struct io_uring_buf_reg *reg,if (IS_ERR (pages))return PTR_ERR (pages); -/* -* Apparently some 32-bit boxes (ARM) will return highmem pages, -* which then need to be mapped. We could support that, but it'd -* complicate the code and slowdown the common cases quite a bit. -* So just error out, returning -EINVAL just like we did on kernels -* that didn't support mapped buffer rings. -*/ -for (i =0 ; i < nr_pages; i++) -if (PageHighMem (pages[i])) - goto error_unpin; + br =vmap (pages, nr_pages,VM_MAP ,PAGE_KERNEL ); +if (!br) { + ret = -ENOMEM ; + goto error_unpin; + } - br =page_address (pages[0 ]);#ifdef SHM_COLOUR /* * On platforms that have specific aliasing requirements, SHM_COLOUR
@@ -522,8 +510,10 @@ static int io_pin_pbuf_ring(struct io_uring_buf_reg *reg,* should use IOU_PBUF_RING_MMAP instead, and liburing will handle * this transparently. */ -if ((reg->ring_addr | (unsigned long ) br) & (SHM_COLOUR -1 )) +if ((reg->ring_addr | (unsigned long ) br) & (SHM_COLOUR -1 )) { + ret = -EINVAL ; goto error_unpin; + }#endif bl->buf_pages = pages; bl->buf_nr_pages = nr_pages;
@@ -535,7 +525,8 @@ static int io_pin_pbuf_ring(struct io_uring_buf_reg *reg,for (i =0 ; i < nr_pages; i++)unpin_user_page (pages[i]);kvfree (pages); -return -EINVAL ; +vunmap (br); +return ret; }/*
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH v3 8/11 io_uring/kbuf: protect io_buffer_list teardown with a reference
In reply to Jens AxboeNo functional changes in this patch, just in preparation for being able to keep the buffer list alive outside of the ctx->uring_lock.
--- io_uring/kbuf.c | 15 +++++++++++---- io_uring/kbuf.h | 2 ++ 2 files changed, 13 insertions(+), 4 deletions(-)
io_uring/kbuf.c +11 -4
@@ -62,6 +62,7 @@ static int io_buffer_add_list(struct io_ring_ctx *ctx,* always under the ->uring_lock, but the RCU lookup from mmap does. */ bl->bgid = bgid; +atomic_set (&bl->refs,1 );return xa_err (xa_store (&ctx->io_bl_xa, bgid, bl,GFP_KERNEL )); }
@@ -259,6 +260,14 @@ static int __io_remove_buffers(struct io_ring_ctx *ctx,return i; } +static void io_put_bl (struct io_ring_ctx *ctx,struct io_buffer_list *bl) +{ +if (atomic_dec_and_test (&bl->refs)) { +__io_remove_buffers (ctx, bl,-1U ); +kfree_rcu (bl, rcu); + } +} +void io_destroy_buffers (struct io_ring_ctx *ctx) {struct io_buffer_list *bl;
@@ -268,8 +277,7 @@ void io_destroy_buffers(struct io_ring_ctx *ctx)xa_for_each (&ctx->io_bl_xa, index, bl) {xa_erase (&ctx->io_bl_xa, bl->bgid); -__io_remove_buffers (ctx, bl,-1U ); -kfree_rcu (bl, rcu); +io_put_bl (ctx, bl); }/*
@@ -671,9 +679,8 @@ int io_unregister_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg)if (!bl->is_buf_ring)return -EINVAL ; -__io_remove_buffers (ctx, bl,-1U );xa_erase (&ctx->io_bl_xa, bl->bgid); -kfree_rcu (bl, rcu); +io_put_bl (ctx, bl);return 0 ; }
io_uring/kbuf.h +2 -0
@@ -25,6 +25,8 @@ struct io_buffer_list {__u16 head;__u16 mask; +atomic_t refs; +/* ring mapped provided buffers */ __u8 is_buf_ring;/* ring mapped provided buffers, but mmap'ed by application */
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH v3 9/11 io_uring/kbuf: use vm_insert_pages() for mmap'ed pbuf ring
In reply to Jens AxboeRather than use remap_pfn_range() for this and manually free later, switch to using vm_insert_page() and have it Just Work.
This requires a bit of effort on the mmap lookup side, as the ctx uring_lock isn't held, which otherwise protects buffer_lists from being torn down, and it's not safe to grab from mmap context that would introduce an ABBA deadlock between the mmap lock and the ctx uring_lock. Instead, lookup the buffer_list under RCU, as the the list is RCU freed already. Use the existing reference count to determine whether it's possible to safely grab a reference to it (eg if it's not zero already), and drop that reference when done with the mapping. If the mmap reference is the last one, the buffer_list and the associated memory can go away, since the vma insertion has references to the inserted pages at that point.
--- include/linux/io_uring_types.h | 3 - io_uring/io_uring.c | 69 +++++-------- io_uring/io_uring.h | 6 +- io_uring/kbuf.c | 171 +++++++++++---------------------- io_uring/kbuf.h | 7 +- 5 files changed, 85 insertions(+), 171 deletions(-)
include/linux/io_uring_types.h +0 -3
@@ -372,9 +372,6 @@ struct io_ring_ctx {struct list_head io_buffers_cache; -/* deferred free list, protected by ->uring_lock */ -struct hlist_head io_buf_list; -/* Keep this last, we don't need it for the fast path */ struct wait_queue_head poll_wq;struct io_restriction restrictions;
io_uring/io_uring.c +22 -47
147 lines not on this page · raw message
io_uring/io_uring.h +4 -2
@@ -109,8 +109,10 @@ bool __io_alloc_req_refill(struct io_ring_ctx *ctx);bool io_match_task_safe (struct io_kiocb *head,struct task_struct *task,bool cancel_all); -void *io_mem_alloc (size_t size); -void io_mem_free (void *ptr); +void *io_pages_map (struct page ***out_pages,unsigned short *npages, +size_t size); +void io_pages_unmap (void *ptr,struct page ***pages,unsigned short *npages, +bool put_pages);enum {IO_EVENTFD_OP_SIGNAL_BIT ,
io_uring/kbuf.c +55 -116
227 lines not on this page · raw message
io_uring/kbuf.h +4 -3
@@ -55,13 +55,14 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg);int io_unregister_pbuf_ring (struct io_ring_ctx *ctx,void __user *arg);int io_register_pbuf_status (struct io_ring_ctx *ctx,void __user *arg); -void io_kbuf_mmap_list_free (struct io_ring_ctx *ctx); -void __io_put_kbuf (struct io_kiocb *req,unsigned issue_flags);bool io_kbuf_recycle_legacy (struct io_kiocb *req,unsigned issue_flags); -void *io_pbuf_get_address (struct io_ring_ctx *ctx,unsigned long bgid); +void io_put_bl (struct io_ring_ctx *ctx,struct io_buffer_list *bl); +struct io_buffer_list *io_pbuf_get_bl (struct io_ring_ctx *ctx, +unsigned long bgid); +int io_pbuf_mmap (struct file *file,struct vm_area_struct *vma);static inline bool io_kbuf_recycle_ring (struct io_kiocb *req) {
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH v3 10/11 io_uring: use unpin_user_pages() where appropriate
In reply to Jens AxboeThere are a few cases of open-rolled loops around unpin_user_page(), use the generic helper instead.
--- io_uring/io_uring.c | 4 +--- io_uring/kbuf.c | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-)
io_uring/io_uring.c +1 -3
@@ -2621,13 +2621,11 @@ void io_pages_unmap(void *ptr, struct page ***pages, unsigned short *npages,static void io_pages_free (struct page ***pages,int npages) {struct page **page_array = *pages; -int i;if (!page_array)return ; -for (i =0 ; i < npages; i++) -unpin_user_page (page_array[i]); +unpin_user_pages (page_array, npages);kvfree (page_array); *pages =NULL ; }
io_uring/kbuf.c +2 -3
@@ -457,8 +457,8 @@ static int io_pin_pbuf_ring(struct io_uring_buf_reg *reg, struct io_buffer_list *bl) {struct io_uring_buf_ring *br =NULL ; -int nr_pages, ret, i;struct page **pages; +int nr_pages, ret; pages = io_pin_pages(reg->ring_addr, flex_array_size(br, bufs, reg->ring_entries),
@@ -494,8 +494,7 @@ static int io_pin_pbuf_ring(struct io_uring_buf_reg *reg, bl->is_mmap =0 ;return 0 ; error_unpin: -for (i =0 ; i < nr_pages; i++) -unpin_user_page (pages[i]); +unpin_user_pages (pages, nr_pages);kvfree (pages);vunmap (br);return ret;
Signature · 1 line
2.43.0
Jens Axboe
maintainerPATCH v3 11/11 io_uring: move mapping/allocation helpers to a separate file
In reply to Jens AxboeMove the related code from io_uring.c into memmap.c. No functional changes in this patch, just cleaning it up a bit now that the full transition is done.
--- io_uring/Makefile | 3 +- io_uring/io_uring.c | 324 +----------------------------------------- io_uring/io_uring.h | 9 -- io_uring/kbuf.c | 1 + io_uring/memmap.c | 333 ++++++++++++++++++++++++++++++++++++++++++++ io_uring/memmap.h | 25 ++++ io_uring/rsrc.c | 1 + 7 files changed, 364 insertions(+), 332 deletions(-) create mode 100644 io_uring/memmap.c create mode 100644 io_uring/memmap.h
io_uring/Makefile +2 -1
@@ -8,7 +8,8 @@ obj-$(CONFIG_IO_URING) += io_uring.o opdef.o kbuf.o rsrc.o notif.o \ xattr.o nop.o fs.o splice.o sync.o \ msg_ring.o advise.o openclose.o \ epoll.o statx.o timeout.o fdinfo.o \ - cancel.o waitid.o register.o truncate.o + cancel.o waitid.o register.o \ + truncate.o memmap.o obj-$(CONFIG_IO_WQ) += io-wq.o obj-$(CONFIG_FUTEX) += futex.o obj-$(CONFIG_NET_RX_BUSY_POLL) += napi.o
io_uring/io_uring.c +2 -322
356 lines not on this page · raw message
io_uring/io_uring.h +0 -9
@@ -69,10 +69,6 @@ bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflagsbool io_req_post_cqe (struct io_kiocb *req,s32 res,u32 cflags);void __io_commit_cqring_flush (struct io_ring_ctx *ctx); -struct page **io_pin_pages (unsigned long ubuf,unsigned long len,int *npages); -int io_uring_mmap_pages (struct io_ring_ctx *ctx,struct vm_area_struct *vma, -struct page **pages,int npages); -struct file *io_file_get_normal (struct io_kiocb *req,int fd);struct file *io_file_get_fixed (struct io_kiocb *req,int fd,unsigned issue_flags);
@@ -109,11 +105,6 @@ bool __io_alloc_req_refill(struct io_ring_ctx *ctx);bool io_match_task_safe (struct io_kiocb *head,struct task_struct *task,bool cancel_all); -void *io_pages_map (struct page ***out_pages,unsigned short *npages, -size_t size); -void io_pages_unmap (void *ptr,struct page ***pages,unsigned short *npages, -bool put_pages); -enum {IO_EVENTFD_OP_SIGNAL_BIT ,IO_EVENTFD_OP_FREE_BIT ,
io_uring/kbuf.c +1 -0
@@ -15,6 +15,7 @@#include "io_uring.h" #include "opdef.h" #include "kbuf.h" +#include "memmap.h" #define IO_BUFFER_LIST_BUF_PER_PAGE (PAGE_SIZE / sizeof(struct io_uring_buf))
io_uring/memmap.c new file +333 -0
333 lines not on this page · raw message
io_uring/memmap.h new file +25 -0
@@ -0,0 +1,25 @@ +#ifndef IO_URING_MEMMAP_H +#define IO_URING_MEMMAP_H + +struct page **io_pin_pages (unsigned long ubuf,unsigned long len,int *npages); +void io_pages_free (struct page ***pages,int npages); +int io_uring_mmap_pages (struct io_ring_ctx *ctx,struct vm_area_struct *vma, +struct page **pages,int npages); + +void *io_pages_map (struct page ***out_pages,unsigned short *npages, +size_t size); +void io_pages_unmap (void *ptr,struct page ***pages,unsigned short *npages, +bool put_pages); + +void *__io_uaddr_map (struct page ***pages,unsigned short *npages, +unsigned long uaddr,size_t size); + +#ifndef CONFIG_MMU +unsigned int io_uring_nommu_mmap_capabilities (struct file *file); +#endif +unsigned long io_uring_get_unmapped_area (struct file *file,unsigned long addr, +unsigned long len,unsigned long pgoff, +unsigned long flags); +int io_uring_mmap (struct file *file,struct vm_area_struct *vma); + +#endif
io_uring/rsrc.c +1 -0
@@ -16,6 +16,7 @@#include "alloc_cache.h" #include "openclose.h" #include "rsrc.h" +#include "memmap.h" struct io_rsrc_update {struct file *file;
Signature · 1 line
2.43.0
Re: [PATCH 02/11] io_uring: get rid of remap_pfn_range() for mapping rings/sqes
In reply to Jens AxboeQuoted 36 lines · Jens Axboe
Rather than use
remap_pfn_range()for this and manually free later, switch to usingvm_insert_pages()and have it Just Work.If possible, allocate a single compound page that covers the range that is needed. If that works, then we can just use
page_address()on that page. If we fail to get a compound page, allocate single pages and usevmap()to map them into the kernel virtual address space.This just covers the rings/sqes, the other remaining user of the mmap
remap_pfn_range()user will be converted separately. Once that is done, we can kill the old alloc/free code.Signed-off-by Jens Axboe <[email protected]>--- io_uring/io_uring.c | 136 +++++++++++++++++++++++++++++++++++++++++--- io_uring/io_uring.h | 2 + 2 files changed, 130 insertions(+), 8 deletions(-)io_uring/io_uring.c +10
-0@@ -2594,6 +2594,33 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,return READ_ONCE (rings->cq.head) ==READ_ONCE (rings->cq.tail) ? ret :0 ; } +static void io_pages_unmap (void *ptr,struct page ***pages, +unsigned short *npages) +{ +bool do_vunmap = false; + +if (*npages) { +struct page **to_free = *pages; +int i; + +/*
io_uring/io_uring.c static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · line 2616
Quoted 10 lines
io_uring/io_uring.c +10
-0+* Only did vmap for the non-compound multiple page case. +* For the compound page, we just need to put the head. +*/ +if (PageCompound (to_free[0 ])) + *npages =1 ; +else if (*npages >1 ) + do_vunmap = true; +for (i =0 ; i < *npages; i++) +put_page (to_free[i]); + }
Hi Jens,
wouldn't it be simpler to handle the compound case separately as a folio? Then you folio_put the compound page here and just handle the non-continuous case after.
Signature · 1 line
Gabriel Krisman Bertazi
Jens Axboe
maintainerRe: [PATCH 02/11] io_uring: get rid of remap_pfn_range() for mapping rings/sqes
Quoted 51 lines · Gabriel Krisman Bertazi
Quoted 36 lines · Jens Axboe
Rather than use
remap_pfn_range()for this and manually free later, switch to usingvm_insert_pages()and have it Just Work.If possible, allocate a single compound page that covers the range that is needed. If that works, then we can just use
page_address()on that page. If we fail to get a compound page, allocate single pages and usevmap()to map them into the kernel virtual address space.This just covers the rings/sqes, the other remaining user of the mmap
remap_pfn_range()user will be converted separately. Once that is done, we can kill the old alloc/free code.Signed-off-by Jens Axboe <[email protected]>--- io_uring/io_uring.c | 136 +++++++++++++++++++++++++++++++++++++++++--- io_uring/io_uring.h | 2 + 2 files changed, 130 insertions(+), 8 deletions(-)io_uring/io_uring.c +10
-0@@ -2594,6 +2594,33 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,return READ_ONCE (rings->cq.head) ==READ_ONCE (rings->cq.tail) ? ret :0 ; } +static void io_pages_unmap (void *ptr,struct page ***pages, +unsigned short *npages) +{ +bool do_vunmap = false; + +if (*npages) { +struct page **to_free = *pages; +int i; + +/* io_uring/io_uring.c static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, · line 2616
Quoted 10 lines
io_uring/io_uring.c +10
-0+* Only did vmap for the non-compound multiple page case. +* For the compound page, we just need to put the head. +*/ +if (PageCompound (to_free[0 ])) + *npages =1 ; +else if (*npages >1 ) + do_vunmap = true; +for (i =0 ; i < *npages; i++) +put_page (to_free[i]); + }Hi Jens,
Quoted 3 lines
wouldn't it be simpler to handle the compound case separately as a folio? Then you
folio_putthe compound page here and just handle the non-continuous case after.
I don't think it makes sense, as we're still dealing with pages for insertion. Once there's some folio variant of inserting pages, then yeah I think it'd make sense to unify it. If not, we're doing the page <-> folio transition in one spot anyway.
Signature · 1 line
Jens Axboe