registrered buffer coalescing cleanup
cover registrered buffer coalescing cleanup
Improve how we handle huge page with registrered buffers, in particular make io_coalesce_buffer() a bit more readable.
Pavel Begunkov (3):
io_uring/rsrc: use unpin_user_folio
io_uring/rsrc: clean up io_coalesce_buffer()
io_uring/rsrc: remove null check on import
io_uring/rsrc.c | 50 ++++++++++++++++++++++--------------------------- 1 file changed, 22 insertions(+), 28 deletions(-)
Signature · 1 line
2.48.1
PATCH 1/3 io_uring/rsrc: use unpin_user_folio
We want to have a full folio to be left pinned but with only one reference, for that we "unpin" all but the first page with unpin_user_pages(), which can be confusing. There is a new helper to achieve that called unpin_user_folio(), so use that.
--- io_uring/rsrc.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-)
io_uring/rsrc.c +2 -3
@@ -699,10 +699,9 @@ static bool io_coalesce_buffer(struct page ***pages, int *nr_pages,* The pages are bound to the folio, it doesn't * actually unpin them but drops all but one reference, * which is usually put down by io_buffer_unmap(). -* Note, needs a better helper. */ if (data->nr_pages_head >1 ) -unpin_user_pages (&page_array[1 ], data->nr_pages_head -1 ); +unpin_user_folio (page_folio (new_array[0 ]), data->nr_pages_head -1 ); j = data->nr_pages_head; nr_pages_left -= data->nr_pages_head;
@@ -713,7 +712,7 @@ static bool io_coalesce_buffer(struct page ***pages, int *nr_pages, nr_unpin = min_t(unsigned int, nr_pages_left - 1, data->nr_pages_mid -1 );if (nr_unpin) -unpin_user_pages (&page_array[j+1 ], nr_unpin); +unpin_user_folio (page_folio (new_array[i]), nr_unpin); j += data->nr_pages_mid; nr_pages_left -= data->nr_pages_mid; }
Signature · 1 line
2.48.1
Re: [PATCH 1/3] io_uring/rsrc: use unpin_user_folio
PATCH 2/3 io_uring/rsrc: clean up io_coalesce_buffer()
In reply to Pavel BegunkovWe don't need special handling for the first page in io_coalesce_buffer(), move it inside the loop.
--- io_uring/rsrc.c | 47 ++++++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 25 deletions(-)
io_uring/rsrc.c +22 -25
@@ -685,37 +685,34 @@ static bool io_coalesce_buffer(struct page ***pages, int *nr_pages, struct io_imu_folio_data *data) {struct page **page_array = *pages, **new_array =NULL ; -int nr_pages_left = *nr_pages, i, j; -int nr_folios = data->nr_folios; +unsigned nr_pages_left = *nr_pages; +unsigned nr_folios = data->nr_folios; +unsigned i, j;/* Store head pages only*/ - new_array =kvmalloc_array (nr_folios,sizeof (struct page *), -GFP_KERNEL ); + new_array =kvmalloc_array (nr_folios,sizeof (struct page *),GFP_KERNEL );if (!new_array)return false; - new_array[0 ] =compound_head (page_array[0 ]); -/* -* The pages are bound to the folio, it doesn't -* actually unpin them but drops all but one reference, -* which is usually put down by io_buffer_unmap(). -*/ -if (data->nr_pages_head >1 ) -unpin_user_folio (page_folio (new_array[0 ]), data->nr_pages_head -1 ); - - j = data->nr_pages_head; - nr_pages_left -= data->nr_pages_head; -for (i =1 ; i < nr_folios; i++) { -unsigned int nr_unpin; - - new_array[i] = page_array[j]; - nr_unpin = min_t(unsigned int, nr_pages_left - 1, - data->nr_pages_mid -1 ); -if (nr_unpin) -unpin_user_folio (page_folio (new_array[i]), nr_unpin); - j += data->nr_pages_mid; - nr_pages_left -= data->nr_pages_mid; +for (i =0 , j =0 ; i < nr_folios; i++) { +struct page *p =compound_head (page_array[j]); +struct folio *folio =page_folio (p); +unsigned int nr; + +WARN_ON_ONCE (i >0 && p != page_array[j]); + + nr = i ? data->nr_pages_mid : data->nr_pages_head; + nr =min (nr, nr_pages_left); +/* Drop all but one ref, the entire folio will remain pinned. */ +if (nr >1 ) +unpin_user_folio (folio, nr -1 ); + j += nr; + nr_pages_left -= nr; + new_array[i] = p; } + +WARN_ON_ONCE (j != *nr_pages); +kvfree (page_array); *pages = new_array; *nr_pages = nr_folios;
Signature · 1 line
2.48.1
Re: [PATCH 2/3] io_uring/rsrc: clean up io_coalesce_buffer()
Re: [PATCH 2/3] io_uring/rsrc: clean up io_coalesce_buffer()
In reply to Pavel BegunkovQuoted 13 lines · Pavel Begunkov
We don't need special handling for the first page in
io_coalesce_buffer(), move it inside the loop.Signed-off-by Pavel Begunkov <[email protected]>---
io_uring/rsrc.c| 47 ++++++++++++++++++++++-------------------------1 file changed, 22 insertions(+), 25 deletions(-)io_uring/rsrc.c +0
-0
io_uring/rsrc.c static bool io_coalesce_buffer(struct page ***pages, int *nr_pages, · line 690
Quoted 8 lines
io_uring/rsrc.c +3
-2@@ -685,37 +685,34 @@ static bool io_coalesce_buffer(struct page ***pages, int *nr_pages, struct io_imu_folio_data *data) {struct page **page_array = *pages, **new_array =NULL ; -int nr_pages_left = *nr_pages, i, j; -int nr_folios = data->nr_folios; +unsigned nr_pages_left = *nr_pages; +unsigned nr_folios = data->nr_folios; +unsigned i, j;
checkpatch.pl complains about just "unsigned", "unsigned int" is preferred.
io_uring/rsrc.c static bool io_coalesce_buffer(struct page ***pages, int *nr_pages, · line 693
Quoted 5 lines
io_uring/rsrc.c +1
-2/* Store head pages only*/ - new_array =kvmalloc_array (nr_folios,sizeof (struct page *), -GFP_KERNEL ); + new_array =kvmalloc_array (nr_folios,sizeof (struct page *),GFP_KERNEL );
Not sure whether 80 line length is preferred in uring subsystem, if yes then this breaks it.
--Nitesh Shetty
Jens Axboe
maintainerRe: [PATCH 2/3] io_uring/rsrc: clean up io_coalesce_buffer()
Quoted 25 lines · Nitesh Shetty
Quoted 13 lines · Pavel Begunkov
We don't need special handling for the first page in
io_coalesce_buffer(), move it inside the loop.Signed-off-by Pavel Begunkov <[email protected]>---
io_uring/rsrc.c| 47 ++++++++++++++++++++++-------------------------1 file changed, 22 insertions(+), 25 deletions(-)io_uring/rsrc.c +0
-0io_uring/rsrc.c static bool io_coalesce_buffer(struct page ***pages, int *nr_pages, · line 690
Quoted 8 lines
io_uring/rsrc.c +3
-2@@ -685,37 +685,34 @@ static bool io_coalesce_buffer(struct page ***pages, int *nr_pages, struct io_imu_folio_data *data) {struct page **page_array = *pages, **new_array =NULL ; -int nr_pages_left = *nr_pages, i, j; -int nr_folios = data->nr_folios; +unsigned nr_pages_left = *nr_pages; +unsigned nr_folios = data->nr_folios; +unsigned i, j;checkpatch.pl complains about just "unsigned", "unsigned int" is preferred.
Quoted 1 line
Quoted 6 lines
io_uring/rsrc.c static bool io_coalesce_buffer(struct page ***pages, int *nr_pages, · line 693
Quoted 4 lines
/* Store head pages only*/io_uring/rsrc.c +1
-2- new_array =kvmalloc_array (nr_folios,sizeof (struct page *), -GFP_KERNEL ); + new_array =kvmalloc_array (nr_folios,sizeof (struct page *),GFP_KERNEL );Not sure whether 80 line length is preferred in uring subsystem, if yes then this breaks it.
It's fine to use checkpatch for new contributors, but it's generally just pretty useless imho. io_uring does exceed 80 chars regularly, if it improves readability. And we also do use unsigned, not sure why checkpatch is against that.
Signature · 1 line
Jens Axboe
PATCH 3/3 io_uring/rsrc: remove null check on import
In reply to Pavel BegunkovWARN_ON_ONCE() checking imu for NULL in io_import_fixed() is in the hot path and too protective, it's time to get rid of it.
--- io_uring/rsrc.c | 2 -- 1 file changed, 2 deletions(-)
io_uring/rsrc.c +0 -2
@@ -1058,8 +1058,6 @@ static int io_import_fixed(int ddir, struct iov_iter *iter,size_t offset;int ret; -if (WARN_ON_ONCE (!imu)) -return -EFAULT ; ret =validate_fixed_range (buf_addr, len, imu);if (unlikely (ret))return ret;
Signature · 1 line
2.48.1
Quoted 9 lines · Pavel Begunkov
Improve how we handle huge page with registrered buffers, in particular make
io_coalesce_buffer()a bit more readable.Pavel Begunkov (3):
io_uring/rsrc: useunpin_user_folio
io_uring/rsrc: clean upio_coalesce_buffer()
io_uring/rsrc: remove null check on import[...]
Applied, thanks!
[1/3] io_uring/rsrc: use unpin_user_folio
commit: ea76925614189bdcb6571e2ea8de68af409ebd56
[2/3] io_uring/rsrc: clean up io_coalesce_buffer()
commit: 9cebcf7b0c38bca1b501d8716163aa254b230559
[3/3] io_uring/rsrc: remove null check on import
commit: be6bad57b217491733754ae8113eec94a90a2769Best regards,
Signature · 1 line
Jens Axboe