implement async block discards and other ops via io_uring
Subsystems (5)
v3 cover implement async block discards and other ops via io_uring
There is an interest in having asynchronous block operations like discard and write zeroes. The series implements that as io_uring commands, which is an io_uring request type allowing to implement custom file specific operations.
First 4 are preparation patches. Patch 5 introduces the main chunk of cmd infrastructure and discard commands. Patches 6-8 implement write zeroes variants.
Branch with tests and docs:
https://github.com/isilence/liburing.git discard-cmd
The man page specifically (need to shuffle it to some cmd section):
https://github.com/isilence/liburing/commit/a6fa2bc2400bf7fcb80496e322b5db4c8b3191f0
v3: use GFP_NOWAIT for non-blocking allocation
fail oversized nowait discards in advance drop secure erase and add zero page writes renamed function name + other cosmetic changes use IOC / ioctl encoding for cmd opcodes
v2: move out of CONFIG_COMPAT
add write zeroes & secure erase
drop a note about interaction with page cache
Pavel Begunkov (8):
io_uring/cmd: expose iowq to cmds io_uring/cmd: give inline space in request to cmds
filemap: introduce filemap_invalidate_pages
block: introduce blk_validate_byte_range()
block: implement async discard as io_uring cmd
block: implement async write zeroes command
block: add nowait flag for __blkdev_issue_zero_pages
block: implement async write zero pages command
block/blk-lib.c | 25 +++- block/blk.h | 1 + block/fops.c | 2 + block/ioctl.c | 228 ++++++++++++++++++++++++++++++++--- include/linux/bio.h | 6 + include/linux/blkdev.h | 1 + include/linux/io_uring/cmd.h | 15 +++ include/linux/pagemap.h | 2 + include/uapi/linux/fs.h | 4 + io_uring/io_uring.c | 11 ++ io_uring/io_uring.h | 1 + io_uring/uring_cmd.c | 7 ++ mm/filemap.c | 17 ++- 13 files changed, 292 insertions(+), 28 deletions(-)
Signature · 1 line
2.45.2
PATCH v3 1/8 io_uring/cmd: expose iowq to cmds
When an io_uring request needs blocking context we offload it to the io_uring's thread pool called io-wq. We can get there off ->uring_cmd by returning -EAGAIN, but there is no straightforward way of doing that from an asynchronous callback. Add a helper that would transfer a command to a blocking context.
Note, we do an extra hop via task_work before io_queue_iowq(), that's a limitation of io_uring infra we have that can likely be lifted later if that would ever become a problem.
--- include/linux/io_uring/cmd.h | 6 ++++++ io_uring/io_uring.c | 11 +++++++++++ io_uring/io_uring.h | 1 + io_uring/uring_cmd.c | 7 +++++++ 4 files changed, 25 insertions(+)
include/linux/io_uring/cmd.h +6 -0
@@ -48,6 +48,9 @@ void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,void io_uring_cmd_mark_cancelable (struct io_uring_cmd *cmd,unsigned int issue_flags); +/* Execute the request from a blocking context */ +void io_uring_cmd_issue_blocking (struct io_uring_cmd *ioucmd); +#else static inline int io_uring_cmd_import_fixed (u64 ubuf,unsigned long len,int rw,struct iov_iter *iter,void *ioucmd)
@@ -67,6 +70,9 @@ static inline void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd, unsigned int issue_flags) { } +static inline void io_uring_cmd_issue_blocking (struct io_uring_cmd *ioucmd) +{ +}#endif /*
io_uring/io_uring.c +11 -0
@@ -533,6 +533,17 @@ static void io_queue_iowq(struct io_kiocb *req)io_queue_linked_timeout (link); } +static void io_req_queue_iowq_tw (struct io_kiocb *req,struct io_tw_state *ts) +{ +io_queue_iowq (req); +} + +void io_req_queue_iowq (struct io_kiocb *req) +{ + req->io_task_work.func = io_req_queue_iowq_tw; +io_req_task_work_add (req); +} +static __cold void io_queue_deferred (struct io_ring_ctx *ctx) {while (!list_empty (&ctx->defer_list)) {
io_uring/io_uring.h +1 -0
@@ -94,6 +94,7 @@ int io_uring_alloc_task_context(struct task_struct *task,int io_ring_add_registered_file (struct io_uring_task *tctx,struct file *file,int start,int end); +void io_req_queue_iowq (struct io_kiocb *req);int io_poll_issue (struct io_kiocb *req,struct io_tw_state *ts);int io_submit_sqes (struct io_ring_ctx *ctx,unsigned int nr);
io_uring/uring_cmd.c +7 -0
@@ -277,6 +277,13 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, }EXPORT_SYMBOL_GPL (io_uring_cmd_import_fixed); +void io_uring_cmd_issue_blocking (struct io_uring_cmd *ioucmd) +{ +struct io_kiocb *req =cmd_to_io_kiocb (ioucmd); + +io_req_queue_iowq (req); +} +static inline int io_uring_cmd_getsockopt (struct socket *sock,struct io_uring_cmd *cmd,unsigned int issue_flags)
Signature · 1 line
2.45.2
PATCH v3 2/8 io_uring/cmd: give inline space in request to cmds
In reply to Pavel BegunkovSome io_uring commands can use some inline space in io_kiocb. We have 32 bytes in struct io_uring_cmd, expose it.
--- include/linux/io_uring/cmd.h | 9 +++++++++ 1 file changed, 9 insertions(+)
include/linux/io_uring/cmd.h +9 -0
@@ -23,6 +23,15 @@ static inline const void *io_uring_sqe_cmd(const struct io_uring_sqe *sqe)return sqe->cmd; } +static inline void io_uring_cmd_private_sz_check (size_t cmd_sz) +{ + BUILD_BUG_ON(cmd_sz > sizeof_field(struct io_uring_cmd, pdu)); +} +#define io_uring_cmd_to_pdu (cmd, pdu_type) ( \ + io_uring_cmd_private_sz_check(sizeof(pdu_type)), \ + ((pdu_type *)&(cmd)->pdu) \ +) +#if defined(CONFIG_IO_URING )int io_uring_cmd_import_fixed (u64 ubuf,unsigned long len,int rw,struct iov_iter *iter,void *ioucmd);
Signature · 1 line
2.45.2
PATCH v3 3/8 filemap: introduce filemap_invalidate_pages
In reply to Pavel Begunkovkiocb_invalidate_pages() is useful for the write path, however not everything is backed by kiocb and we want to reuse the function for bio based discard implementation. Extract and and reuse a new helper called filemap_invalidate_pages(), which takes a argument indicating whether it should be non-blocking and might return -EAGAIN.
--- include/linux/pagemap.h | 2 ++ mm/filemap.c | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-)
include/linux/pagemap.h +2 -0
@@ -32,6 +32,8 @@ int invalidate_inode_pages2_range(struct address_space *mapping, pgoff_t start, pgoff_t end);int kiocb_invalidate_pages (struct kiocb *iocb,size_t count);void kiocb_invalidate_post_direct_write (struct kiocb *iocb,size_t count); +int filemap_invalidate_pages (struct address_space *mapping, +loff_t pos,loff_t end,bool nowait);int write_inode_now (struct inode *,int sync);int filemap_fdatawrite (struct address_space *);
mm/filemap.c +12 -5
@@ -2712,14 +2712,12 @@ int kiocb_write_and_wait(struct kiocb *iocb, size_t count) }EXPORT_SYMBOL_GPL (kiocb_write_and_wait); -int kiocb_invalidate_pages (struct kiocb *iocb,size_t count) +int filemap_invalidate_pages (struct address_space *mapping, +loff_t pos,loff_t end,bool nowait) { -struct address_space *mapping = iocb->ki_filp->f_mapping; -loff_t pos = iocb->ki_pos; -loff_t end = pos + count -1 ;int ret; -if (iocb->ki_flags &IOCB_NOWAIT ) { +if (nowait) {/* we could block if there are any pages in the range */ if (filemap_range_has_page (mapping, pos, end))return -EAGAIN ;
@@ -2738,6 +2736,15 @@ int kiocb_invalidate_pages(struct kiocb *iocb, size_t count)return invalidate_inode_pages2_range (mapping, pos >>PAGE_SHIFT , end >>PAGE_SHIFT ); } + +int kiocb_invalidate_pages (struct kiocb *iocb,size_t count) +{ +struct address_space *mapping = iocb->ki_filp->f_mapping; + +return filemap_invalidate_pages (mapping, iocb->ki_pos, + iocb->ki_pos + count -1 , + iocb->ki_flags &IOCB_NOWAIT ); +}EXPORT_SYMBOL_GPL (kiocb_invalidate_pages);/**
Signature · 1 line
2.45.2
PATCH v3 4/8 block: introduce blk_validate_byte_range()
In reply to Pavel BegunkovIn preparation to further changes extract a helper function out of blk_ioctl_discard() that validates if we can do IO against the given range of disk byte addresses.
--- block/ioctl.c | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-)
block/ioctl.c +26 -18
@@ -92,38 +92,46 @@ static int compat_blkpg_ioctl(struct block_device *bdev, }#endif +static int blk_validate_byte_range (struct block_device *bdev, +uint64_t start,uint64_t len) +{ +unsigned int bs_mask =bdev_logical_block_size (bdev) -1 ; +uint64_t end; + +if ((start | len) & bs_mask) +return -EINVAL ; +if (!len) +return -EINVAL ; +if (check_add_overflow (start, len, &end) || end >bdev_nr_bytes (bdev)) +return -EINVAL ; + +return 0 ; +} +static int blk_ioctl_discard (struct block_device *bdev,blk_mode_t mode,unsigned long arg) { -unsigned int bs_mask =bdev_logical_block_size (bdev) -1 ; -uint64_t range[2 ], start, len, end; +uint64_t range[2 ], start, len;struct bio *prev =NULL , *bio;sector_t sector, nr_sects;struct blk_plug plug;int err; -if (!(mode &BLK_OPEN_WRITE )) -return -EBADF ; - -if (!bdev_max_discard_sectors (bdev)) -return -EOPNOTSUPP ; -if (bdev_read_only (bdev)) -return -EPERM ; -if (copy_from_user (range, (void __user *)arg,sizeof (range)))return -EFAULT ; - start = range[0 ]; len = range[1 ]; -if (!len) -return -EINVAL ; -if ((start | len) & bs_mask) -return -EINVAL ; +if (!bdev_max_discard_sectors (bdev)) +return -EOPNOTSUPP ; -if (check_add_overflow (start, len, &end) || - end >bdev_nr_bytes (bdev)) -return -EINVAL ; +if (!(mode &BLK_OPEN_WRITE )) +return -EBADF ; +if (bdev_read_only (bdev)) +return -EPERM ; + err =blk_validate_byte_range (bdev, start, len); +if (err) +return err;filemap_invalidate_lock (bdev->bd_mapping); err =truncate_bdev_range (bdev, mode, start, start + len -1 );
Signature · 1 line
2.45.2
PATCH v3 5/8 block: implement async discard as io_uring cmd
In reply to Pavel Begunkovio_uring allows to implement custom file specific operations via fops->uring_cmd callback. Use it to wire up asynchronous discard commands. Normally, first it tries to do a non-blocking issue, and if fails we'd retry from a blocking context by returning -EAGAIN to core io_uring.
Note, unlike ioctl(BLKDISCARD) with stronger guarantees against races, we only do a best effort attempt to invalidate page cache, and it can race with any writes and reads and leave page cache stale. It's the same kind of races we allow to direct writes.
--- block/blk-lib.c | 3 +- block/blk.h | 1 + block/fops.c | 2 + block/ioctl.c | 102 ++++++++++++++++++++++++++++++++++++++++ include/linux/bio.h | 2 + include/uapi/linux/fs.h | 2 + 6 files changed, 111 insertions(+), 1 deletion(-)
block/blk-lib.c +2 -1
@@ -10,7 +10,8 @@#include "blk.h" -static sector_t bio_discard_limit (struct block_device *bdev,sector_t sector) +/* The maximum size of a discard that can be issued from a given sector. */ +sector_t bio_discard_limit (struct block_device *bdev,sector_t sector) {unsigned int discard_granularity =bdev_discard_granularity (bdev);sector_t granularity_aligned_sector;
block/blk.h +1 -0
@@ -605,6 +605,7 @@ blk_mode_t file_to_blk_mode(struct file *file);int truncate_bdev_range (struct block_device *bdev,blk_mode_t mode,loff_t lstart,loff_t lend);long blkdev_ioctl (struct file *file,unsigned cmd,unsigned long arg); +int blkdev_uring_cmd (struct io_uring_cmd *cmd,unsigned int issue_flags);long compat_blkdev_ioctl (struct file *file,unsigned cmd,unsigned long arg);extern const struct address_space_operations def_blk_aops;
block/fops.c +2 -0
@@ -17,6 +17,7 @@#include <linux/fs.h> #include <linux/iomap.h> #include <linux/module.h> +#include <linux/io_uring/cmd.h> #include "blk.h" static inline struct inode *bdev_file_inode (struct file *file)
@@ -873,6 +874,7 @@ const struct file_operations def_blk_fops = { .splice_read = filemap_splice_read, .splice_write = iter_file_splice_write, .fallocate = blkdev_fallocate, + .uring_cmd = blkdev_uring_cmd, .fop_flags = FOP_BUFFER_RASYNC, };
block/ioctl.c +102 -0
111 lines not on this page · raw message
include/linux/bio.h +2 -0
@@ -684,4 +684,6 @@ struct bio *bio_chain_and_submit(struct bio *prev, struct bio *new);struct bio *blk_alloc_discard_bio (struct block_device *bdev,sector_t *sector,sector_t *nr_sects,gfp_t gfp_mask); +sector_t bio_discard_limit (struct block_device *bdev,sector_t sector); +#endif /* __LINUX_BIO_H */
include/uapi/linux/fs.h +2 -0
@@ -208,6 +208,8 @@ struct fsxattr {* (see uapi/linux/blkzoned.h) */ +#define BLOCK_URING_CMD_DISCARD _IO(0x12,137) +#define BMAP_IOCTL 1/* obsolete - kept for compatibility */ #define FIBMAP _IO(0x00,1)/* bmap access */ #define FIGETBSZ _IO(0x00,2)/* get the block size used for bmap */
Signature · 1 line
2.45.2
PATCH v3 6/8 block: implement async write zeroes command
In reply to Pavel BegunkovAdd another io_uring cmd for block layer implementing asynchronous write zeroes. It reuses helpers we've added for async discards, and inherits the code structure as well as all considerations in regards to page cache races.
--- block/ioctl.c | 64 +++++++++++++++++++++++++++++++++++++++++ include/uapi/linux/fs.h | 1 + 2 files changed, 65 insertions(+)
block/ioctl.c +64 -0
@@ -772,6 +772,67 @@ static void bio_cmd_bio_end_io(struct bio *bio)bio_put (bio); } +static int blkdev_cmd_write_zeroes (struct io_uring_cmd *cmd, +struct block_device *bdev, +uint64_t start,uint64_t len,bool nowait) +{ + +sector_t bs_mask = (bdev_logical_block_size (bdev) >>SECTOR_SHIFT ) -1 ; +sector_t limit =bdev_write_zeroes_sectors (bdev); +sector_t sector = start >>SECTOR_SHIFT ; +sector_t nr_sects = len >>SECTOR_SHIFT ; +struct bio *prev =NULL , *bio; +gfp_t gfp = nowait ?GFP_NOWAIT :GFP_KERNEL ; +int err; + +if (!(file_to_blk_mode (cmd->file) &BLK_OPEN_WRITE )) +return -EBADF ; +if (bdev_read_only (bdev)) +return -EPERM ; + err =blk_validate_byte_range (bdev, start, len); +if (err) +return err; + +if (!limit) +return -EOPNOTSUPP ; +/* +* Don't allow multi-bio non-blocking submissions as subsequent bios +* may fail but we won't get a direct indication of that. Normally, +* the caller should retry from a blocking context. +*/ +if (nowait && nr_sects > limit) +return -EAGAIN ; + + err =filemap_invalidate_pages (bdev->bd_mapping, start, + start + len -1 , nowait); +if (err) +return err; + + limit =min (limit, (UINT_MAX >>SECTOR_SHIFT ) & ~bs_mask); +while (nr_sects) { +sector_t bio_sects =min (nr_sects, limit); + + bio =bio_alloc (bdev,0 ,REQ_OP_WRITE_ZEROES |REQ_NOUNMAP , gfp); +if (!bio) +break ; +if (nowait) + bio->bi_opf |=REQ_NOWAIT ; + bio->bi_iter.bi_sector = sector; + bio->bi_iter.bi_size = bio_sects <<SECTOR_SHIFT ; + sector += bio_sects; + nr_sects -= bio_sects; + + prev =bio_chain_and_submit (prev, bio); + } +if (!prev) +return -EAGAIN ; + + prev->bi_private = cmd; + prev->bi_end_io = bio_cmd_bio_end_io; +submit_bio (prev); +return -EIOCBQUEUED ; +} +static int blkdev_cmd_discard (struct io_uring_cmd *cmd,struct block_device *bdev,uint64_t start,uint64_t len,bool nowait)
@@ -841,6 +902,9 @@ int blkdev_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)switch (cmd_op) {case BLOCK_URING_CMD_DISCARD :return blkdev_cmd_discard (cmd, bdev, start, len, bic->nowait); +case BLOCK_URING_CMD_WRITE_ZEROES : +return blkdev_cmd_write_zeroes (cmd, bdev, start, len, + bic->nowait); }return -EINVAL ; }
include/uapi/linux/fs.h +1 -0
@@ -209,6 +209,7 @@ struct fsxattr {*/ #define BLOCK_URING_CMD_DISCARD _IO(0x12,137) +#define BLOCK_URING_CMD_WRITE_ZEROES _IO(0x12,138)#define BMAP_IOCTL 1/* obsolete - kept for compatibility */ #define FIBMAP _IO(0x00,1)/* bmap access */
Signature · 1 line
2.45.2
PATCH v3 7/8 block: add nowait flag for __blkdev_issue_zero_pages
In reply to Pavel BegunkovTo reuse __blkdev_issue_zero_pages() in the following patch, we need to make it work with non-blocking requests. Add a new nowait flag we can pass inside. Return errors if something went wrong, and check bio_alloc() for failures, which wasn't supposed to happen before because of what gfp flags the callers are passing. Note that there might be a bio passed back even when the function returned an error. To limit the scope of the patch, don't add return code handling to callers, that can be deferred to a follow up.
--- block/blk-lib.c | 22 ++++++++++++++++++---- include/linux/bio.h | 4 ++++ include/linux/blkdev.h | 1 + 3 files changed, 23 insertions(+), 4 deletions(-)
block/blk-lib.c +18 -4
@@ -193,20 +193,32 @@ static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects)return min (pages, (sector_t )BIO_MAX_VECS ); } -static void __blkdev_issue_zero_pages (struct block_device *bdev, +int blkdev_issue_zero_pages_bio (struct block_device *bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio **biop,unsigned int flags) { +blk_opf_t opf =REQ_OP_WRITE ; + +if (flags &BLKDEV_ZERO_PAGES_NOWAIT ) { +sector_t max_bio_sectors =BIO_MAX_VECS <<PAGE_SECTORS_SHIFT ; + +if (nr_sects > max_bio_sectors) +return -EAGAIN ; + opf |=REQ_NOWAIT ; + } +while (nr_sects) {unsigned int nr_vecs =__blkdev_sectors_to_bio_pages (nr_sects);struct bio *bio; bio =bio_alloc (bdev, nr_vecs,REQ_OP_WRITE , gfp_mask); +if (!bio) +return -ENOMEM ; bio->bi_iter.bi_sector = sector;if ((flags &BLKDEV_ZERO_KILLABLE ) &&fatal_signal_pending (current)) -break ; +return -EINTR ; do { unsigned int len, added;
@@ -223,6 +235,8 @@ static void __blkdev_issue_zero_pages(struct block_device *bdev, *biop =bio_chain_and_submit (*biop, bio);cond_resched (); } + +return 0 ; } static int blkdev_issue_zero_pages(struct block_device *bdev, sector_t sector,
@@ -236,7 +250,7 @@ static int blkdev_issue_zero_pages(struct block_device *bdev, sector_t sector,return -EOPNOTSUPP ;blk_start_plug (&plug); -__blkdev_issue_zero_pages (bdev, sector, nr_sects, gfp, &bio, flags); +blkdev_issue_zero_pages_bio (bdev, sector, nr_sects, gfp, &bio, flags);if (bio) {if ((flags &BLKDEV_ZERO_KILLABLE ) &&fatal_signal_pending (current)) {
@@ -286,7 +300,7 @@ int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, }else {if (flags &BLKDEV_ZERO_NOFALLBACK )return -EOPNOTSUPP ; -__blkdev_issue_zero_pages (bdev, sector, nr_sects, gfp_mask, +blkdev_issue_zero_pages_bio (bdev, sector, nr_sects, gfp_mask, biop, flags); }return 0 ;
include/linux/bio.h +4 -0
@@ -686,4 +686,8 @@ struct bio *blk_alloc_discard_bio(struct block_device *bdev,sector_t bio_discard_limit (struct block_device *bdev,sector_t sector); +int blkdev_issue_zero_pages_bio (struct block_device *bdev, +sector_t sector,sector_t nr_sects,gfp_t gfp_mask, +struct bio **biop,unsigned int flags); +#endif /* __LINUX_BIO_H */
include/linux/blkdev.h +1 -0
@@ -1098,6 +1098,7 @@ int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector,#define BLKDEV_ZERO_NOUNMAP (1 << 0)/* do not free blocks */ #define BLKDEV_ZERO_NOFALLBACK (1 << 1)/* don't write explicit zeroes */ #define BLKDEV_ZERO_KILLABLE (1 << 2)/* interruptible by fatal signals */ +#define BLKDEV_ZERO_PAGES_NOWAIT (1 << 3)/* non-blocking submission */ extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
Signature · 1 line
2.45.2
kernel test robot
build botRe: [PATCH v3 7/8] block: add nowait flag for __blkdev_issue_zero_pages
kernel test robot noticed the following build warnings:
Hi Pavel,
kernel test robot noticed the following build warnings:
[auto build test WARNING on axboe-block/for-next]
[also build test WARNING on akpm-mm/mm-everything linus/master v6.11-rc6 next-20240905] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Pavel-Begunkov/io_uring-cmd-expose-iowq-to-cmds/20240904-222012 base: https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux-block.git for-next patch link: https://lore.kernel.org/r/292fa1c611adb064efe16ab741aad65c2128ada8.1725459175.git.asml.silence%40gmail.com patch subject: [PATCH v3 7/8] block: add nowait flag for __blkdev_issue_zero_pages config: i386-randconfig-141-20240906 (https://download.01.org/0day-ci/archive/20240906/[email protected]/config) compiler: clang version 18.1.5 (https://github.com/llvm/llvm-project 617a15a9eac96088ae5e9134248d8236e34b91b1) reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240906/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
>> block/blk-lib.c:200:12: warning: variable 'opf' set but not used [-Wunused-but-set-variable]
200 | blk_opf_t opf = REQ_OP_WRITE;
| ^1 warning generated.
vim +/opf +200 block/blk-lib.c
195
196 int blkdev_issue_zero_pages_bio(struct block_device *bdev,
197 sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
198 struct bio **biop, unsigned int flags)
199 {
> 200 blk_opf_t opf = REQ_OP_WRITE;
201
202 if (flags & BLKDEV_ZERO_PAGES_NOWAIT) {
203 sector_t max_bio_sectors = BIO_MAX_VECS << PAGE_SECTORS_SHIFT;
204
205 if (nr_sects > max_bio_sectors)
206 return -EAGAIN;
207 opf |= REQ_NOWAIT;
208 }
209
210 while (nr_sects) {
211 unsigned int nr_vecs = __blkdev_sectors_to_bio_pages(nr_sects);
212 struct bio *bio;
213
214 bio = bio_alloc(bdev, nr_vecs, REQ_OP_WRITE, gfp_mask);
215 if (!bio)
216 return -ENOMEM;
217 bio->bi_iter.bi_sector = sector;
218
219 if ((flags & BLKDEV_ZERO_KILLABLE) &&
220 fatal_signal_pending(current))
221 return -EINTR;
222
223 do {
224 unsigned int len, added;
225
226 len = min_t(sector_t,
227 PAGE_SIZE, nr_sects << SECTOR_SHIFT);
228 added = bio_add_page(bio, ZERO_PAGE(0), len, 0);
229 if (added < len)
230 break;
231 nr_sects -= added >> SECTOR_SHIFT;
232 sector += added >> SECTOR_SHIFT;
233 } while (nr_sects);
234
235 *biop = bio_chain_and_submit(*biop, bio);
236 cond_resched();
237 }
238
239 return 0;
240 }
241
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
Jens Axboe
maintainerRe: [PATCH v3 7/8] block: add nowait flag for __blkdev_issue_zero_pages
In reply to Pavel BegunkovQuoted 18 lines · Pavel Begunkov
block/blk-lib.c +6
-1@@ -193,20 +193,32 @@ static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects)return min (pages, (sector_t )BIO_MAX_VECS ); } -static void __blkdev_issue_zero_pages (struct block_device *bdev, +int blkdev_issue_zero_pages_bio (struct block_device *bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio **biop,unsigned int flags) { +blk_opf_t opf =REQ_OP_WRITE ; + +if (flags &BLKDEV_ZERO_PAGES_NOWAIT ) { +sector_t max_bio_sectors =BIO_MAX_VECS <<PAGE_SECTORS_SHIFT ; +
block/blk-lib.c static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects) · line 214
Quoted 10 lines
block/blk-lib.c +5
-0+if (nr_sects > max_bio_sectors) +return -EAGAIN ; + opf |=REQ_NOWAIT ; + } +while (nr_sects) {unsigned int nr_vecs =__blkdev_sectors_to_bio_pages (nr_sects);struct bio *bio; bio =bio_alloc (bdev, nr_vecs,REQ_OP_WRITE , gfp_mask);
as per the kernel test bot, I guess this one should be using opf rather than REQ_OP_WRITE.
Signature · 1 line
Jens Axboe
Re: [PATCH v3 7/8] block: add nowait flag for __blkdev_issue_zero_pages
Quoted 28 lines · Jens Axboe
Quoted 27 lines · Pavel Begunkov
block/blk-lib.c +11
-1@@ -193,20 +193,32 @@ static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects)return min (pages, (sector_t )BIO_MAX_VECS ); } -static void __blkdev_issue_zero_pages (struct block_device *bdev, +int blkdev_issue_zero_pages_bio (struct block_device *bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio **biop,unsigned int flags) { +blk_opf_t opf =REQ_OP_WRITE ; + +if (flags &BLKDEV_ZERO_PAGES_NOWAIT ) { +sector_t max_bio_sectors =BIO_MAX_VECS <<PAGE_SECTORS_SHIFT ; + +if (nr_sects > max_bio_sectors) +return -EAGAIN ; + opf |=REQ_NOWAIT ; + } +while (nr_sects) {unsigned int nr_vecs =__blkdev_sectors_to_bio_pages (nr_sects);struct bio *bio;
Quoted 4 lines
block/blk-lib.c static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects) · line 214
Quoted 1 line
block/blk-lib.c +0
-0bio =bio_alloc (bdev, nr_vecs,REQ_OP_WRITE , gfp_mask);as per the kernel test bot, I guess this one should be using opf rather than
REQ_OP_WRITE.
Right, I overlooked it. I'm going to resend the series later today.
Signature · 1 line
Pavel Begunkov
PATCH v3 8/8 block: implement async write zero pages command
In reply to Pavel BegunkovAdd a command that writes the zero page to the drive. Apart from passing the zero page instead of actual data it uses the normal write path and doesn't do any further acceleration, nor it requires any special hardware support. The indended use is to have a fallback when BLOCK_URING_CMD_WRITE_ZEROES is not supported.
--- block/ioctl.c | 24 +++++++++++++++++++++--- include/uapi/linux/fs.h | 1 + 2 files changed, 22 insertions(+), 3 deletions(-)
block/ioctl.c +21 -3
@@ -774,7 +774,8 @@ static void bio_cmd_bio_end_io(struct bio *bio)static int blkdev_cmd_write_zeroes (struct io_uring_cmd *cmd,struct block_device *bdev, -uint64_t start,uint64_t len,bool nowait) +uint64_t start,uint64_t len, +bool nowait,bool zero_pages) {sector_t bs_mask = (bdev_logical_block_size (bdev) >>SECTOR_SHIFT ) -1 ;
@@ -793,6 +794,20 @@ static int blkdev_cmd_write_zeroes(struct io_uring_cmd *cmd,if (err)return err; +if (zero_pages) { +struct blk_iou_cmd *bic =io_uring_cmd_to_pdu (cmd, + struct blk_iou_cmd); + + err =blkdev_issue_zero_pages_bio (bdev, sector, nr_sects, + gfp, &prev, +BLKDEV_ZERO_PAGES_NOWAIT ); +if (!prev) +return -EAGAIN ; +if (err) + bic->res = err; + goto out_submit; + } +if (!limit)return -EOPNOTSUPP ;/*
@@ -826,7 +841,7 @@ static int blkdev_cmd_write_zeroes(struct io_uring_cmd *cmd, }if (!prev)return -EAGAIN ; - +out_submit: prev->bi_private = cmd; prev->bi_end_io = bio_cmd_bio_end_io;submit_bio (prev);
@@ -904,7 +919,10 @@ int blkdev_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)return blkdev_cmd_discard (cmd, bdev, start, len, bic->nowait);case BLOCK_URING_CMD_WRITE_ZEROES :return blkdev_cmd_write_zeroes (cmd, bdev, start, len, - bic->nowait); + bic->nowait, false); +case BLOCK_URING_CMD_WRITE_ZERO_PAGE : +return blkdev_cmd_write_zeroes (cmd, bdev, start, len, + bic->nowait, true); }return -EINVAL ; }
include/uapi/linux/fs.h +1 -0
@@ -210,6 +210,7 @@ struct fsxattr {#define BLOCK_URING_CMD_DISCARD _IO(0x12,137)#define BLOCK_URING_CMD_WRITE_ZEROES _IO(0x12,138) +#define BLOCK_URING_CMD_WRITE_ZERO_PAGE _IO(0x12,139)#define BMAP_IOCTL 1/* obsolete - kept for compatibility */ #define FIBMAP _IO(0x00,1)/* bmap access */
Signature · 1 line
2.45.2