Make generic_make_request() handle arbitrary size bios
Subsystems (21)
Jens, the following patches are for 3.15. This builds off of immutable biovecs, and this series in turn enables the DIO rewrite I've been working on off and on for ages and ages, and it also will let us delete merge_bvec_fn piecemeal (altogether merge_bvec_fn is 1500 lines of code we'll be able to delete).
This will also get us performance gains (that I've measured myself, benchmarking on a p320h) once we push the work of splitting too-big bios all the way down to the drivers, and I expect bigger gains for stacking block drivers due to just processing larger bios.
I've got a patch I'll send separately for the mtip32xx driver to make it take arbitrary size bios, it's pretty small.
Also a few related small refactoring patches.
This is on top of your for-3.15/core branch. If you want to pull these patches from my tree, here's that:
The following changes since commit c46fff2a3b29794b35d717b5680a27f31a6a6bc0:
smp: Rename __smp_call_function_single() to smp_call_function_single_async() (2014-02-24 14:47:15 -0800)
are available in the git repository at:
git://evilpiepirate.org/~kent/linux-bcache.git for-jens
for you to fetch changes up to 089f8de5c42a121f351ef9d240d66e1128fa0ea2:
iov_iter: Kill written arg to iov_iter_init() (2014-02-26 15:17:36 -0800)
----------------------------------------------------------------
Kent Overstreet (9):
block: Make generic_make_request handle arbitrary sized bios
block: Gut bio_add_page()
blk-lib.c: generic_make_request() handles large bios now
bcache: generic_make_request() handles large bios now
btrfs: generic_make_request() handles arbitrary size bios now
btrfs: Convert to bio_for_each_segment()
iov_iter: Move iov_iter to uio.h
iov_iter: Kill iov_iter_single_seg_count()
iov_iter: Kill written arg to iov_iter_init()
block/blk-core.c | 19 ++- block/blk-lib.c | 175 +++++----------------------- block/blk-merge.c | 150 ++++++++++++++++++++++-- block/blk-mq.c | 2 + drivers/block/drbd/drbd_req.c | 2 + drivers/block/mtip32xx/mtip32xx.c | 6 +- drivers/block/nvme-core.c | 2 + drivers/block/pktcdvd.c | 6 +- drivers/block/ps3vram.c | 2 + drivers/block/rsxx/dev.c | 2 + drivers/block/umem.c | 2 + drivers/block/zram/zram_drv.c | 2 + drivers/md/bcache/bcache.h | 18 --- drivers/md/bcache/io.c | 100 +--------------- drivers/md/bcache/journal.c | 4 +- drivers/md/bcache/request.c | 16 +-- drivers/md/bcache/super.c | 32 +---- drivers/md/bcache/util.h | 5 +- drivers/md/bcache/writeback.c | 4 +- drivers/md/dm.c | 2 + drivers/md/md.c | 2 + drivers/s390/block/dcssblk.c | 2 + drivers/s390/block/xpram.c | 2 + drivers/staging/lustre/lustre/llite/lloop.c | 2 + fs/bio.c | 137 +++++++++------------- fs/btrfs/extent_io.c | 12 +- fs/btrfs/file-item.c | 59 ++++------ fs/btrfs/file.c | 8 +- fs/btrfs/inode.c | 22 ++-- fs/btrfs/volumes.c | 73 ------------ fs/ceph/file.c | 7 +- fs/cifs/file.c | 4 +- fs/fuse/file.c | 17 +-- include/linux/blkdev.h | 3 + include/linux/fs.h | 32 ----- include/linux/uio.h | 46 ++++++++ mm/filemap.c | 19 +-- 37 files changed, 399 insertions(+), 599 deletions(-)
PATCH 1/9 block: Make generic_make_request handle arbitrary sized bios
The way the block layer is currently written, it goes to great lengths to avoid having to split bios; upper layer code (such as bio_add_page()) checks what the underlying device can handle and tries to always create bios that don't need to be split.
But this approach becomes unwieldy and eventually breaks down with stacked devices and devices with dynamic limits, and it adds a lot of complexity. If the block layer could split bios as needed, we could eliminate a lot of complexity elsewhere - particularly in stacked drivers. Code that creates bios can then create whatever size bios are convenient, and more importantly stacked drivers don't have to deal with both their own bio size limitations and the limitations of the (potentially multiple) devices underneath them. In the future this will let us delete merge_bvec_fn and a bunch of other code.
We do this by adding calls to blk_queue_split() to the various make_request functions that need it - a few can already handle arbitrary size bios. Note that we add the call _after_ any call to blk_queue_bounce(); this means that blk_queue_split() and blk_recalc_rq_segments() don't need to be concerned with bouncing affecting segment merging.
Some make_request_fns were simple enough to audit and verify they don't need blk_queue_split() calls. The skipped ones are:
* nfhd_make_request (arch/m68k/emu/nfblock.c)
* axon_ram_make_request (arch/powerpc/sysdev/axonram.c)
* simdisk_make_request (arch/xtensa/platforms/iss/simdisk.c)
* brd_make_request (ramdisk - drivers/block/brd.c)
* loop_make_request
* null_queue_bio
* bcache's make_request fns
Some others are almost certainly safe to remove now, but will be left for future patches.
--- block/blk-core.c | 19 ++-- block/blk-merge.c | 150 ++++++++++++++++++++++++++-- block/blk-mq.c | 2 + drivers/block/drbd/drbd_req.c | 2 + drivers/block/mtip32xx/mtip32xx.c | 6 +- drivers/block/nvme-core.c | 2 + drivers/block/pktcdvd.c | 6 +- drivers/block/ps3vram.c | 2 + drivers/block/rsxx/dev.c | 2 + drivers/block/umem.c | 2 + drivers/block/zram/zram_drv.c | 2 + drivers/md/dm.c | 2 + drivers/md/md.c | 2 + drivers/s390/block/dcssblk.c | 2 + drivers/s390/block/xpram.c | 2 + drivers/staging/lustre/lustre/llite/lloop.c | 2 + include/linux/blkdev.h | 3 + 17 files changed, 185 insertions(+), 23 deletions(-)
block/blk-core.c +9 -10
@@ -581,6 +581,10 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)if (q->id <0 ) goto fail_c; + q->bio_split =bioset_create (4 ,0 ); +if (!q->bio_split) + goto fail_id; + q->backing_dev_info.ra_pages = (VM_MAX_READAHEAD *1024 ) /PAGE_CACHE_SIZE ; q->backing_dev_info.state =0 ;
@@ -590,7 +594,7 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) err =bdi_init (&q->backing_dev_info);if (err) - goto fail_id; + goto fail_split;setup_timer (&q->backing_dev_info.laptop_mode_wb_timer, laptop_mode_timer_fn, (unsigned long ) q);
@@ -635,6 +639,8 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) fail_bdi:bdi_destroy (&q->backing_dev_info); +fail_split: +bioset_free (q->bio_split); fail_id:ida_simple_remove (&blk_queue_ida, q->id); fail_c:
@@ -1501,6 +1507,8 @@ void blk_queue_bio(struct request_queue *q, struct bio *bio)struct request *req;unsigned int request_count =0 ; +blk_queue_split (q, &bio, q->bio_split); +/* * low level driver can indicate that it wants pages above a * certain limit bounced to low memory (ie for highmem, or even
@@ -1723,15 +1731,6 @@ generic_make_request_checks(struct bio *bio) goto end_io; } -if (likely (bio_is_rw (bio) && - nr_sectors >queue_max_hw_sectors (q))) { -printk (KERN_ERR "bio too big device %s (%u > %u)\n" , -bdevname (bio->bi_bdev, b), -bio_sectors (bio), -queue_max_hw_sectors (q)); - goto end_io; - } - part = bio->bi_bdev->bd_part;if (should_fail_request (part, bio->bi_iter.bi_size) || should_fail_request(&part_to_disk(part)->part0,
block/blk-merge.c +141 -9
173 lines not on this page · raw message
block/blk-mq.c +2 -0
@@ -915,6 +915,8 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio)return ; } +blk_queue_split (q, &bio, q->bio_split); +if (use_plug &&blk_attempt_plug_merge (q, bio, &request_count))return ;
drivers/block/drbd/drbd_req.c +2 -0
@@ -1275,6 +1275,8 @@ void drbd_make_request(struct request_queue *q, struct bio *bio)struct drbd_conf *mdev = (struct drbd_conf *) q->queuedata;unsigned long start_time; +blk_queue_split (q, &bio, q->bio_split); + start_time = jiffies;/*
drivers/block/mtip32xx/mtip32xx.c +4 -2
@@ -4033,6 +4033,10 @@ static void mtip_make_request(struct request_queue *queue, struct bio *bio)int nents =0 ;int tag =0 , unaligned =0 ; +blk_queue_bounce (queue, &bio); + +blk_queue_split (queue, &bio, queue->bio_split); +if (unlikely (dd->dd_flag &MTIP_DDF_STOP_IO )) {if (unlikely (test_bit (MTIP_DDF_REMOVE_PENDING_BIT , &dd->dd_flag))) {
@@ -4082,8 +4086,6 @@ static void mtip_make_request(struct request_queue *queue, struct bio *bio) sg =mtip_hw_get_scatterlist (dd, &tag, unaligned);if (likely (sg !=NULL )) { -blk_queue_bounce (queue, &bio); -if (unlikely ((bio)->bi_vcnt >MTIP_MAX_SG )) {dev_warn (&dd->pdev->dev,"Maximum number of SGL entries exceeded\n" );
drivers/block/nvme-core.c +2 -0
@@ -737,6 +737,8 @@ static void nvme_make_request(struct request_queue *q, struct bio *bio)struct nvme_queue *nvmeq =get_nvmeq (ns->dev);int result = -EBUSY ; +blk_queue_split (q, &bio, q->bio_split); +if (!nvmeq) {put_nvmeq (NULL );bio_endio (bio, -EIO );
drivers/block/pktcdvd.c +4 -2
@@ -2444,6 +2444,10 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio)char b[BDEVNAME_SIZE ];struct bio *split; +blk_queue_bounce (q, &bio); + +blk_queue_split (q, &bio, q->bio_split); + pd = q->queuedata;if (!pd) { pr_err("%s incorrect request queue\n",
@@ -2474,8 +2478,6 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio) goto end_io; } -blk_queue_bounce (q, &bio); - do { sector_t zone = get_zone(bio->bi_iter.bi_sector, pd); sector_t last_zone = get_zone(bio_end_sector(bio) - 1, pd);
drivers/block/ps3vram.c +2 -0
@@ -603,6 +603,8 @@ static void ps3vram_make_request(struct request_queue *q, struct bio *bio)struct ps3vram_priv *priv =ps3_system_bus_get_drvdata (dev);int busy; +blk_queue_split (q, &bio, q->bio_split); +dev_dbg (&dev->core,"%s\n" , __func__);spin_lock_irq (&priv->lock);
drivers/block/rsxx/dev.c +2 -0
@@ -169,6 +169,8 @@ static void rsxx_make_request(struct request_queue *q, struct bio *bio)struct rsxx_bio_meta *bio_meta;int st = -EINVAL ; +blk_queue_split (q, &bio, q->bio_split); +might_sleep ();if (!card)
drivers/block/umem.c +2 -0
@@ -531,6 +531,8 @@ static void mm_make_request(struct request_queue *q, struct bio *bio) (unsigned long long )bio->bi_iter.bi_sector, bio->bi_iter.bi_size); +blk_queue_split (q, &bio, q->bio_split); +spin_lock_irq (&card->lock); *card->biotail = bio; bio->bi_next =NULL ;
drivers/block/zram/zram_drv.c +2 -0
@@ -733,6 +733,8 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio) {struct zram *zram = queue->queuedata; +blk_queue_split (queue, &bio, queue->bio_split); +down_read (&zram->init_lock);if (unlikely (!zram->init_done)) goto error;
drivers/md/dm.c +2 -0
@@ -1500,6 +1500,8 @@ static void dm_request(struct request_queue *q, struct bio *bio) {struct mapped_device *md = q->queuedata; +blk_queue_split (q, &bio, q->bio_split); +if (dm_request_based (md))blk_queue_bio (q, bio);else
drivers/md/md.c +2 -0
@@ -256,6 +256,8 @@ static void md_make_request(struct request_queue *q, struct bio *bio)int cpu;unsigned int sectors; +blk_queue_split (q, &bio, q->bio_split); +if (mddev ==NULL || mddev->pers ==NULL || !mddev->ready) {bio_io_error (bio);
drivers/s390/block/dcssblk.c +2 -0
@@ -815,6 +815,8 @@ dcssblk_make_request(struct request_queue *q, struct bio *bio)unsigned long source_addr;unsigned long bytes_done; +blk_queue_split (q, &bio, q->bio_split); + bytes_done =0 ; dev_info = bio->bi_bdev->bd_disk->private_data;if (dev_info ==NULL )
drivers/s390/block/xpram.c +2 -0
@@ -190,6 +190,8 @@ static void xpram_make_request(struct request_queue *q, struct bio *bio)unsigned long page_addr;unsigned long bytes; +blk_queue_split (q, &bio, q->bio_split); +if ((bio->bi_iter.bi_sector &7 ) !=0 || (bio->bi_iter.bi_size &4095 ) !=0 )/* Request is not page-aligned. */
drivers/staging/lustre/lustre/llite/lloop.c +2 -0
@@ -344,6 +344,8 @@ static void loop_make_request(struct request_queue *q, struct bio *old_bio)int rw =bio_rw (old_bio);int inactive; +blk_queue_split (q, &old_bio, q->bio_split); +if (!lo) goto err;
include/linux/blkdev.h +3 -0
@@ -470,6 +470,7 @@ struct request_queue {wait_queue_head_t mq_freeze_wq;struct percpu_counter mq_usage_counter;struct list_head all_q_node; +struct bio_set *bio_split; };#define QUEUE_FLAG_QUEUED 1/* uses generic tag queueing */
@@ -781,6 +782,8 @@ extern void blk_rq_unprep_clone(struct request *rq);extern int blk_insert_cloned_request (struct request_queue *q,struct request *rq);extern void blk_delay_queue (struct request_queue *,unsigned long ); +extern void blk_queue_split (struct request_queue *,struct bio **, +struct bio_set *);extern void blk_recount_segments (struct request_queue *,struct bio *);extern int scsi_verify_blk_ioctl (struct block_device *,unsigned int ); extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
Signature · 1 line
1.9.0
Re: [PATCH 1/9] block: Make generic_make_request handle arbitrary sized bios
Quoted 11 lines · Kent Overstreet
We do this by adding calls to
blk_queue_split()to the variousmake_requestfunctions that need it - a few can already handle arbitrary size bios. Note that we add the call _after_ any call toblk_queue_bounce(); this means thatblk_queue_split()andblk_recalc_rq_segments()don't need to be concerned with bouncing affecting segment merging.drivers/block/nvme-core.c +0
-0
drivers/block/nvme-core.c static void nvme_make_request(struct request_queue *q, struct bio *bio) · line 744
Quoted 8 lines
drivers/block/nvme-core.c +2
-0@@ -737,6 +737,8 @@ static void nvme_make_request(struct request_queue *q, struct bio *bio)struct nvme_queue *nvmeq =get_nvmeq (ns->dev);int result = -EBUSY ; +blk_queue_split (q, &bio, q->bio_split); +if (!nvmeq) {put_nvmeq (NULL );bio_endio (bio, -EIO );
I'd suggest that we do:
- struct nvme_queue *nvmeq = get_nvmeq(ns->dev);
+ struct nvme_queue *nvmeq;
int result = -EBUSY;
+ blk_queue_split(q, &bio, q->bio_split);
+
+ nvmeq = get_nvmeq(ns->dev);
if (!nvmeq) {
so that we're running the blk_queue_split() code outside the get_cpu() call.
Now, the NVMe driver has its own rules about when BIOs have to be split. Right now, that's way down inside the nvme_map_bio() call when we're walking the bio to compose the scatterlist. Should we instead have an nvme_bio_split() routine that is called instead of blk_queue_split(), and we can simplify nvme_map_bio() since it'll know that it's working with bios that don't have to be split.
In fact, I think it would have little NVMe-specific in it at that point, so we could name __blk_bios_map_sg() better, export it to drivers and call it from nvme_map_bio(), which I think would make everybody happier.
Quoted 164 lines
drivers/block/pktcdvd.c +4
-2@@ -2444,6 +2444,10 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio)char b[BDEVNAME_SIZE ];struct bio *split; +blk_queue_bounce (q, &bio); + +blk_queue_split (q, &bio, q->bio_split); + pd = q->queuedata;if (!pd) { pr_err("%s incorrect request queue\n",@@ -2474,8 +2478,6 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio) goto end_io; } -blk_queue_bounce (q, &bio); - do { sector_t zone = get_zone(bio->bi_iter.bi_sector, pd); sector_t last_zone = get_zone(bio_end_sector(bio) - 1, pd);drivers/block/ps3vram.c +2
-0@@ -603,6 +603,8 @@ static void ps3vram_make_request(struct request_queue *q, struct bio *bio)struct ps3vram_priv *priv =ps3_system_bus_get_drvdata (dev);int busy; +blk_queue_split (q, &bio, q->bio_split); +dev_dbg (&dev->core,"%s\n" , __func__);spin_lock_irq (&priv->lock);drivers/block/rsxx/dev.c +2
-0@@ -169,6 +169,8 @@ static void rsxx_make_request(struct request_queue *q, struct bio *bio)struct rsxx_bio_meta *bio_meta;int st = -EINVAL ; +blk_queue_split (q, &bio, q->bio_split); +might_sleep ();if (!card)drivers/block/umem.c +2
-0@@ -531,6 +531,8 @@ static void mm_make_request(struct request_queue *q, struct bio *bio) (unsigned long long )bio->bi_iter.bi_sector, bio->bi_iter.bi_size); +blk_queue_split (q, &bio, q->bio_split); +spin_lock_irq (&card->lock); *card->biotail = bio; bio->bi_next =NULL ;drivers/block/zram/zram_drv.c +2
-0@@ -733,6 +733,8 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio) {struct zram *zram = queue->queuedata; +blk_queue_split (queue, &bio, queue->bio_split); +down_read (&zram->init_lock);if (unlikely (!zram->init_done)) goto error;drivers/md/dm.c +2
-0@@ -1500,6 +1500,8 @@ static void dm_request(struct request_queue *q, struct bio *bio) {struct mapped_device *md = q->queuedata; +blk_queue_split (q, &bio, q->bio_split); +if (dm_request_based (md))blk_queue_bio (q, bio);else drivers/md/md.c +2
-0@@ -256,6 +256,8 @@ static void md_make_request(struct request_queue *q, struct bio *bio)int cpu;unsigned int sectors; +blk_queue_split (q, &bio, q->bio_split); +if (mddev ==NULL || mddev->pers ==NULL || !mddev->ready) {bio_io_error (bio);drivers/s390/block/dcssblk.c +2
-0@@ -815,6 +815,8 @@ dcssblk_make_request(struct request_queue *q, struct bio *bio)unsigned long source_addr;unsigned long bytes_done; +blk_queue_split (q, &bio, q->bio_split); + bytes_done =0 ; dev_info = bio->bi_bdev->bd_disk->private_data;if (dev_info ==NULL )drivers/s390/block/xpram.c +2
-0@@ -190,6 +190,8 @@ static void xpram_make_request(struct request_queue *q, struct bio *bio)unsigned long page_addr;unsigned long bytes; +blk_queue_split (q, &bio, q->bio_split); +if ((bio->bi_iter.bi_sector &7 ) !=0 || (bio->bi_iter.bi_size &4095 ) !=0 )/* Request is not page-aligned. */ drivers/staging/lustre/lustre/llite/lloop.c +2
-0@@ -344,6 +344,8 @@ static void loop_make_request(struct request_queue *q, struct bio *old_bio)int rw =bio_rw (old_bio);int inactive; +blk_queue_split (q, &old_bio, q->bio_split); +if (!lo) goto err;include/linux/blkdev.h +3
-0@@ -470,6 +470,7 @@ struct request_queue {wait_queue_head_t mq_freeze_wq;struct percpu_counter mq_usage_counter;struct list_head all_q_node; +struct bio_set *bio_split; };#define QUEUE_FLAG_QUEUED 1/* uses generic tag queueing */ @@ -781,6 +782,8 @@ extern void blk_rq_unprep_clone(struct request *rq);extern int blk_insert_cloned_request (struct request_queue *q,struct request *rq);extern void blk_delay_queue (struct request_queue *,unsigned long ); +extern void blk_queue_split (struct request_queue *,struct bio **, +struct bio_set *);extern void blk_recount_segments (struct request_queue *,struct bio *);extern int scsi_verify_blk_ioctl (struct block_device *,unsigned int ); extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,Signature · 1 line
1.9.0
Re: [PATCH 1/9] block: Make generic_make_request handle arbitrary sized bios
Quoted 32 lines · Matthew Wilcox
Quoted 11 lines · Kent Overstreet
We do this by adding calls to
blk_queue_split()to the variousmake_requestfunctions that need it - a few can already handle arbitrary size bios. Note that we add the call _after_ any call toblk_queue_bounce(); this means thatblk_queue_split()andblk_recalc_rq_segments()don't need to be concerned with bouncing affecting segment merging.drivers/block/nvme-core.c +0
-0drivers/block/nvme-core.c static void nvme_make_request(struct request_queue *q, struct bio *bio) · line 744
Quoted 8 lines
drivers/block/nvme-core.c +2
-0@@ -737,6 +737,8 @@ static void nvme_make_request(struct request_queue *q, struct bio *bio)struct nvme_queue *nvmeq =get_nvmeq (ns->dev);int result = -EBUSY ; +blk_queue_split (q, &bio, q->bio_split); +if (!nvmeq) {put_nvmeq (NULL );bio_endio (bio, -EIO );I'd suggest that we do:
drivers/block/nvme-core.c +4
-1-struct nvme_queue *nvmeq =get_nvmeq (ns->dev); +struct nvme_queue *nvmeq;int result = -EBUSY ; +blk_queue_split (q, &bio, q->bio_split); + + nvmeq =get_nvmeq (ns->dev);if (!nvmeq) {
Quoted 2 lines
so that we're running the
blk_queue_split()code outside theget_cpu()call.
Whoops, that's definitely a bug.
Quoted 10 lines
Now, the NVMe driver has its own rules about when BIOs have to be split. Right now, that's way down inside the
nvme_map_bio()call when we're walking the bio to compose the scatterlist. Should we instead have annvme_bio_split()routine that is called instead ofblk_queue_split(), and we can simplifynvme_map_bio()since it'll know that it's working with bios that don't have to be split.In fact, I think it would have little NVMe-specific in it at that point, so we could name
__blk_bios_map_sg()better, export it to drivers and call it fromnvme_map_bio(), which I think would make everybody happier.
Yes, definitely - and by doing it there we shoudn't even have to split the bios, we can just process them incrementally. I can write a patch for it later if you want to test it.
Re: [PATCH 1/9] block: Make generic_make_request handle arbitrary sized bios
In reply to Matthew WilcoxQuoted 42 lines · Matthew Wilcox
Quoted 11 lines · Kent Overstreet
We do this by adding calls to
blk_queue_split()to the variousmake_requestfunctions that need it - a few can already handle arbitrary size bios. Note that we add the call _after_ any call toblk_queue_bounce(); this means thatblk_queue_split()andblk_recalc_rq_segments()don't need to be concerned with bouncing affecting segment merging.drivers/block/nvme-core.c +0
-0drivers/block/nvme-core.c static void nvme_make_request(struct request_queue *q, struct bio *bio) · line 744
Quoted 8 lines
drivers/block/nvme-core.c +2
-0@@ -737,6 +737,8 @@ static void nvme_make_request(struct request_queue *q, struct bio *bio)struct nvme_queue *nvmeq =get_nvmeq (ns->dev);int result = -EBUSY ; +blk_queue_split (q, &bio, q->bio_split); +if (!nvmeq) {put_nvmeq (NULL );bio_endio (bio, -EIO );I'd suggest that we do:
drivers/block/nvme-core.c +4
-1-struct nvme_queue *nvmeq =get_nvmeq (ns->dev); +struct nvme_queue *nvmeq;int result = -EBUSY ; +blk_queue_split (q, &bio, q->bio_split); + + nvmeq =get_nvmeq (ns->dev);if (!nvmeq) {so that we're running the
blk_queue_split()code outside theget_cpu()call.Now, the NVMe driver has its own rules about when BIOs have to be split. Right now, that's way down inside the
nvme_map_bio()call when we're walking the bio to compose the scatterlist. Should we instead have annvme_bio_split()routine that is called instead ofblk_queue_split(), and we can simplifynvme_map_bio()since it'll know that it's working with bios that don't have to be split.
Quoted 3 lines
In fact, I think it would have little NVMe-specific in it at that point, so we could name
__blk_bios_map_sg()better, export it to drivers and call it fromnvme_map_bio(), which I think would make everybody happier.
Actually, reading nvme_map_bio() (it's different since last I looked at it) it looks like nvme should already be able to handle arbitrary size bios?
I do intend to rework the blk_bio_map_sg() (or add a new one?) to incrementally map as much of a bio as will fit in the provided scatterlist, but it looks like nvme has some odd restrictions where it's using BIOVEC_PHYS_MERGABLE()/BIOVEC_NOT_VIRT_MERGABLE() so I dunno if it's worth bothering to try and have it use generic code.
However we don't need an explicit split here: if the sg fills up (i.e. the places nvme_split_and_submit() is called), we can just mark the bio as partially completed (set bio->bi_iter = iter, i.e. use the iterator you passed to bio_for_each_segment), then increment bi_remaining (which just counts completions, i.e. bio_endio() calls before the bio is really completed) and resubmit the original bio. No need to allocate a split bio, or loop over the bio again in bio_split().
Re: [PATCH 1/9] block: Make generic_make_request handle arbitrary sized bios
Quoted 51 lines · Kent Overstreet
Quoted 42 lines · Matthew Wilcox
Quoted 11 lines · Kent Overstreet
We do this by adding calls to
blk_queue_split()to the variousmake_requestfunctions that need it - a few can already handle arbitrary size bios. Note that we add the call _after_ any call toblk_queue_bounce(); this means thatblk_queue_split()andblk_recalc_rq_segments()don't need to be concerned with bouncing affecting segment merging.drivers/block/nvme-core.c +0
-0drivers/block/nvme-core.c static void nvme_make_request(struct request_queue *q, struct bio *bio) · line 744
Quoted 8 lines
drivers/block/nvme-core.c +2
-0@@ -737,6 +737,8 @@ static void nvme_make_request(struct request_queue *q, struct bio *bio)struct nvme_queue *nvmeq =get_nvmeq (ns->dev);int result = -EBUSY ; +blk_queue_split (q, &bio, q->bio_split); +if (!nvmeq) {put_nvmeq (NULL );bio_endio (bio, -EIO );I'd suggest that we do:
drivers/block/nvme-core.c +4
-1-struct nvme_queue *nvmeq =get_nvmeq (ns->dev); +struct nvme_queue *nvmeq;int result = -EBUSY ; +blk_queue_split (q, &bio, q->bio_split); + + nvmeq =get_nvmeq (ns->dev);if (!nvmeq) {so that we're running the
blk_queue_split()code outside theget_cpu()call.Now, the NVMe driver has its own rules about when BIOs have to be split. Right now, that's way down inside the
nvme_map_bio()call when we're walking the bio to compose the scatterlist. Should we instead have annvme_bio_split()routine that is called instead ofblk_queue_split(), and we can simplifynvme_map_bio()since it'll know that it's working with bios that don't have to be split.Quoted 3 lines
In fact, I think it would have little NVMe-specific in it at that point, so we could name
__blk_bios_map_sg()better, export it to drivers and call it fromnvme_map_bio(), which I think would make everybody happier.Actually, reading
nvme_map_bio()(it's different since last I looked at it) it looks like nvme should already be able to handle arbitrary size bios?
Quoted 5 lines
I do intend to rework the
blk_bio_map_sg()(or add a new one?) to incrementally map as much of a bio as will fit in the provided scatterlist, but it looks like nvme has some odd restrictions where it's using BIOVEC_PHYS_MERGABLE()/BIOVEC_NOT_VIRT_MERGABLE() so I dunno if it's worth bothering to try and have it use generic code.
Is nvme the only driver that has these kinds of restrictions on segment address offsets? If so, I guess there's no reason to make it generic.
Quoted 7 lines
However we don't need an explicit split here: if the sg fills up (i.e. the places
nvme_split_and_submit()is called), we can just mark the bio as partially completed (setbio->bi_iter= iter, i.e. use the iterator you passed tobio_for_each_segment), then incrementbi_remaining(which just counts completions, i.e.bio_endio()calls before the bio is really completed) and resubmit the original bio. No need to allocate a split bio, or loop over the bio again inbio_split().
We used to manipulate the original bio to track partial completions, but I changed that for reasons that haven't quite yet materialized. If we move the bio's bi_iter, it will make it difficult to retry the original request on intermittent failures, and it will break the integrity verify if the device format supports protection information. It's also more performant to submit all parts at once rather than wait for the previous part to complete before sending the next.
PATCH 1/9 block: Make generic_make_request handle arbitrary sized bios
In reply to Kent OverstreetQuoted 57 lines · Kent Overstreet
Quoted 42 lines · Matthew Wilcox
Quoted 11 lines · Kent Overstreet
We do this by adding calls to
blk_queue_split()to the variousmake_requestfunctions that need it - a few can already handle arbitrary size bios. Note that we add the call _after_ any call toblk_queue_bounce(); this means thatblk_queue_split()andblk_recalc_rq_segments()don't need to be concerned with bouncing affecting segment merging.drivers/block/nvme-core.c +0
-0drivers/block/nvme-core.c static void nvme_make_request(struct request_queue *q, struct bio *bio) · line 744
Quoted 8 lines
drivers/block/nvme-core.c +2
-0@@ -737,6 +737,8 @@ static void nvme_make_request(struct request_queue *q, struct bio *bio)struct nvme_queue *nvmeq =get_nvmeq (ns->dev);int result = -EBUSY ; +blk_queue_split (q, &bio, q->bio_split); +if (!nvmeq) {put_nvmeq (NULL );bio_endio (bio, -EIO );I'd suggest that we do:
(quoted diff) +4
-1- struct nvme_queue *nvmeq = get_nvmeq(ns->dev); + struct nvme_queue *nvmeq; int result = -EBUSY; + blk_queue_split(q, &bio, q->bio_split); + + nvmeq = get_nvmeq(ns->dev); if (!nvmeq) {so that we're running the
blk_queue_split()code outside theget_cpu()call.Now, the NVMe driver has its own rules about when BIOs have to be split. Right now, that's way down inside the
nvme_map_bio()call when we're walking the bio to compose the scatterlist. Should we instead have annvme_bio_split()routine that is called instead ofblk_queue_split(), and we can simplifynvme_map_bio()since it'll know that it's working with bios that don't have to be split.Quoted 3 lines
In fact, I think it would have little NVMe-specific in it at that point, so we could name
__blk_bios_map_sg()better, export it to drivers and call it fromnvme_map_bio(), which I think would make everybody happier.Actually, reading
nvme_map_bio()(it's different since last I looked at it) it looks like nvme should already be able to handle arbitrary size bios?I do intend to rework the
blk_bio_map_sg()(or add a new one?) to incrementally map as much of a bio as will fit in the provided scatterlist, but it looks like nvme has some odd restrictions where it's using BIOVEC_PHYS_MERGABLE()/BIOVEC_NOT_VIRT_MERGABLE() so I dunno if it's worth bothering to try and have it use generic code.
Quoted 7 lines
However we don't need an explicit split here: if the sg fills up (i.e. the places
nvme_split_and_submit()is called), we can just mark the bio as partially completed (setbio->bi_iter= iter, i.e. use the iterator you passed tobio_for_each_segment), then incrementbi_remaining(which just counts completions, i.e.bio_endio()calls before the bio is really completed) and resubmit the original bio. No need to allocate a split bio, or loop over the bio again inbio_split().
Hi Kent,
I wanted to see about getting rid of nvme_map_bio and do all the splits generically. I built this on top of your original patch and added additional split criteria for NVMe, which includes the "segment" address offset and the vendor specific "stripe_size" (I think "chunk_size" may have been a more appropriate description, but I used someone else's term for this). This is what I came up with; I hoped the additions would be useful, but maybe the additional split criteria applies only to nvme.
block/blk-merge.c +25 -25
@@ -66,11 +66,9 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,struct bio *bio, struct bio_set *bs) { - struct bio *split; struct bio_vec bv, bvprv;struct bvec_iter iter;unsigned seg_size =0 , nsegs =0 ; -int prev =0 ; struct bvec_merge_data bvm = { .bi_bdev = bio->bi_bdev,
@@ -79,17 +77,26 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, .bi_rw = bio->bi_rw, }; +if (bio->bi_iter.bi_size >>9 >queue_max_sectors (q)) { + bvm.bi_size =queue_max_sectors (q) <<9 ; + goto split; + } +if (queue_stripe_sectors (q)) { +unsigned split_len = + (queue_stripe_sectors (q) - + (bio->bi_iter.bi_sector & + (queue_stripe_sectors (q) -1 ))) <<9 ; +if (split_len < bio->bi_iter.bi_size) { + bvm.bi_size = split_len; + goto split; + } + } +bio_for_each_segment (bv, bio, iter) {if (q->merge_bvec_fn && q->merge_bvec_fn (q, &bvm, &bv) < (int ) bv.bv_len) goto split; - - bvm.bi_size += bv.bv_len; - -if (bvm.bi_size >>9 >queue_max_sectors (q)) - goto split; - -if (prev &&blk_queue_cluster (q)) { +if (nsegs &&blk_queue_cluster (q)) {if (seg_size + bv.bv_len >queue_max_segment_size (q)) goto new_segment;if (!BIOVEC_PHYS_MERGEABLE (&bvprv, &bv))
@@ -97,34 +104,26 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,if (!BIOVEC_SEG_BOUNDARY (q, &bvprv, &bv)) goto new_segment; + bvm.bi_size += bv.bv_len; seg_size += bv.bv_len; bvprv = bv; - prev =1 ;continue ; } new_segment: +if (nsegs &&BIOVEC_SEG_OFFSET (q, &bvprv, &bv)) + goto split;if (nsegs ==queue_max_segments (q)) goto split; + bvm.bi_size += bv.bv_len; nsegs++; bvprv = bv; - prev =1 ; seg_size = bv.bv_len; }return NULL ; split: - split =bio_clone_bioset (bio,GFP_NOIO , bs); - - split->bi_iter.bi_size -= iter.bi_size; - bio->bi_iter = iter; - -if (bio_integrity (bio)) { -bio_integrity_advance (bio, split->bi_iter.bi_size); -bio_integrity_trim (split,0 ,bio_sectors (split)); - } - -return split; +return bio_split (bio, bvm.bi_size >>9 ,GFP_ATOMIC , bs); } void blk_queue_split(struct request_queue *q, struct bio **bio,
@@ -303,7 +302,7 @@ new_segment: *bvprv = *bvec; } -static int __blk_bios_map_sg (struct request_queue *q,struct bio *bio, +int blk_bios_map_sg (struct request_queue *q,struct bio *bio,struct scatterlist *sglist,struct scatterlist **sg) {
@@ -344,6 +343,7 @@ single_segment:return nsegs; } +EXPORT_SYMBOL (blk_bios_map_sg);/* * map a request to scatterlist, return number of sg entries setup. Caller
@@ -356,7 +356,7 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq,int nsegs =0 ;if (rq->bio) - nsegs =__blk_bios_map_sg (q, rq->bio, sglist, &sg); + nsegs =blk_bios_map_sg (q, rq->bio, sglist, &sg);if (unlikely (rq->cmd_flags &REQ_COPY_USER ) && (blk_rq_bytes (rq) & q->dma_pad_mask)) {
@@ -407,7 +407,7 @@ int blk_bio_map_sg(struct request_queue *q, struct bio *bio,struct bio *next = bio->bi_next; bio->bi_next =NULL ; - nsegs =__blk_bios_map_sg (q, bio, sglist, &sg); + nsegs =blk_bios_map_sg (q, bio, sglist, &sg); bio->bi_next = next;if (sg)sg_mark_end (sg);
block/blk-settings.c +36 -1
@@ -111,8 +111,10 @@ void blk_set_default_limits(struct queue_limits *lim) lim->max_segments =BLK_MAX_SEGMENTS ; lim->max_integrity_segments =0 ; lim->seg_boundary_mask =BLK_SEG_BOUNDARY_MASK ; + lim->seg_offset_mask =0 ; lim->max_segment_size =BLK_MAX_SEGMENT_SIZE ; lim->max_sectors = lim->max_hw_sectors =BLK_SAFE_MAX_SECTORS ; + lim->stripe_sectors =0 ; lim->max_write_same_sectors =0 ; lim->max_discard_sectors =0 ; lim->discard_granularity =0 ;
@@ -146,6 +148,7 @@ void blk_set_stacking_limits(struct queue_limits *lim) lim->max_hw_sectors =UINT_MAX ; lim->max_segment_size =UINT_MAX ; lim->max_sectors =UINT_MAX ; + lim->stripe_sectors =0 ; lim->max_write_same_sectors =UINT_MAX ; }EXPORT_SYMBOL (blk_set_stacking_limits);
@@ -262,6 +265,18 @@ void blk_limits_max_hw_sectors(struct queue_limits *limits, unsigned int max_hw_ }EXPORT_SYMBOL (blk_limits_max_hw_sectors); +void blk_queue_stripe_sectors (struct request_queue *q,unsigned int hw_stripe_sectors) +{ +if (hw_stripe_sectors && ((hw_stripe_sectors <<9 ) <PAGE_CACHE_SIZE || + !is_power_of_2 (hw_stripe_sectors))) { + hw_stripe_sectors =1 << (PAGE_CACHE_SHIFT -9 ); +printk (KERN_INFO "%s: set to minimum %d\n" , + __func__, hw_stripe_sectors); + } + q->limits.stripe_sectors = hw_stripe_sectors; +} +EXPORT_SYMBOL (blk_queue_stripe_sectors); +/** * blk_queue_max_hw_sectors - set max sectors for a request for this queue * @q: the request queue for the device
@@ -527,10 +542,13 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, t->max_hw_sectors =min_not_zero (t->max_hw_sectors, b->max_hw_sectors); t->max_write_same_sectors =min (t->max_write_same_sectors, b->max_write_same_sectors); + t->stripe_sectors =min_not_zero (t->stripe_sectors, b->stripe_sectors); t->bounce_pfn =min_not_zero (t->bounce_pfn, b->bounce_pfn); t->seg_boundary_mask =min_not_zero (t->seg_boundary_mask, b->seg_boundary_mask); + t->seg_offset_mask =min_not_zero (t->seg_offset_mask, + b->seg_offset_mask); t->max_segments =min_not_zero (t->max_segments, b->max_segments); t->max_integrity_segments = min_not_zero(t->max_integrity_segments,
@@ -758,7 +776,7 @@ EXPORT_SYMBOL_GPL(blk_queue_dma_drain);**/ void blk_queue_segment_boundary (struct request_queue *q,unsigned long mask) { -if (mask <PAGE_CACHE_SIZE -1 ) { +if (mask < (PAGE_CACHE_SIZE -1 )) { mask =PAGE_CACHE_SIZE -1 ;printk (KERN_INFO "%s: set to minimum %lx\n" , __func__, mask);
@@ -769,6 +787,23 @@ void blk_queue_segment_boundary(struct request_queue *q, unsigned long mask)EXPORT_SYMBOL (blk_queue_segment_boundary);/** +* blk_queue_segment_offset - set boundary rules for segment start address +* @q: the request queue for the device +* @mask: the memory offset mask at which the request must be split +**/ +void blk_queue_segment_offset (struct request_queue *q,unsigned long mask) +{ +if (mask && mask < (PAGE_CACHE_SIZE -1 )) { +printk (KERN_INFO "%s: mask:%lx set to minimum %lx\n" , + __func__, mask,PAGE_CACHE_SIZE -1 ); + mask =PAGE_CACHE_SIZE -1 ; + } + + q->limits.seg_offset_mask = mask; +} +EXPORT_SYMBOL (blk_queue_segment_offset); + +/** * blk_queue_dma_alignment - set dma length and memory alignment * @q: the request queue for the device * @mask: alignment mask
drivers/block/nvme-core.c +16 -70
130 lines not on this page · raw message
include/linux/bio.h +5 -0
@@ -186,6 +186,11 @@ static inline void *bio_data(struct bio *bio)#define BIOVEC_SEG_BOUNDARY (q, b1, b2) \ __BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q))) +#define __BIO_SEG_OFFSET (addr1, addr2, mask) \ + (((addr1) & (mask)) || (((addr2) & (mask)))) +#define BIOVEC_SEG_OFFSET (q, b1, b2) \ + __BIO_SEG_OFFSET(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_offset((q))) +#define bio_io_error (bio) bio_endio((bio), -EIO)/*
include/linux/blkdev.h +16 -0
@@ -267,9 +267,11 @@ struct blk_queue_tag {struct queue_limits {unsigned long bounce_pfn;unsigned long seg_boundary_mask; +unsigned long seg_offset_mask;unsigned int max_hw_sectors;unsigned int max_sectors; +unsigned int stripe_sectors;unsigned int max_segment_size;unsigned int physical_block_size;unsigned int alignment_offset;
@@ -987,6 +989,8 @@ extern int blk_queue_dma_drain(struct request_queue *q, void *buf, unsigned int size);extern void blk_queue_lld_busy (struct request_queue *q,lld_busy_fn *fn);extern void blk_queue_segment_boundary (struct request_queue *,unsigned long ); +extern void blk_queue_segment_offset (struct request_queue *,unsigned long ); +extern void blk_queue_stripe_sectors (struct request_queue *,unsigned int );extern void blk_queue_prep_rq (struct request_queue *,prep_rq_fn *pfn);extern void blk_queue_unprep_rq (struct request_queue *,unprep_rq_fn *ufn);extern void blk_queue_merge_bvec (struct request_queue *,merge_bvec_fn *);
@@ -999,6 +1003,8 @@ extern void blk_queue_flush(struct request_queue *q, unsigned int flush);extern void blk_queue_flush_queueable (struct request_queue *q,bool queueable);extern struct backing_dev_info *blk_get_backing_dev_info (struct block_device *bdev); +extern int blk_bios_map_sg (struct request_queue *q,struct bio *bio,struct scatterlist *sglist, +struct scatterlist **sg);extern int blk_rq_map_sg (struct request_queue *,struct request *,struct scatterlist *);extern int blk_bio_map_sg (struct request_queue *q,struct bio *bio,struct scatterlist *sglist);
@@ -1159,11 +1165,21 @@ static inline unsigned long queue_segment_boundary(struct request_queue *q)return q->limits.seg_boundary_mask; } +static inline unsigned long queue_segment_offset (struct request_queue *q) +{ +return q->limits.seg_offset_mask; +} +static inline unsigned int queue_max_sectors (struct request_queue *q) {return q->limits.max_sectors; } +static inline unsigned int queue_stripe_sectors (struct request_queue *q) +{ +return q->limits.stripe_sectors; +} +static inline unsigned int queue_max_hw_sectors (struct request_queue *q) {return q->limits.max_hw_sectors;
Re: [PATCH 1/9] block: Make generic_make_request handle arbitrary sized bios
In reply to Kent OverstreetKent,
The blk_queue_split(), splits a bio into at most two bios right? So, if the original bio spans larger space than two bios can cover (restriction by the lower driver in the stack), this might not work? Am I reading it incorrectly?
Thanks!
Regards,
Muthu
Quoted 547 lines · Kent Overstreet
The way the block layer is currently written, it goes to great lengths to avoid having to split bios; upper layer code (such as
bio_add_page()) checks what the underlying device can handle and tries to always create bios that don't need to be split.But this approach becomes unwieldy and eventually breaks down with stacked devices and devices with dynamic limits, and it adds a lot of complexity. If the block layer could split bios as needed, we could eliminate a lot of complexity elsewhere - particularly in stacked drivers. Code that creates bios can then create whatever size bios are convenient, and more importantly stacked drivers don't have to deal with both their own bio size limitations and the limitations of the (potentially multiple) devices underneath them. In the future this will let us delete
merge_bvec_fnand a bunch of other code.We do this by adding calls to
blk_queue_split()to the variousmake_requestfunctions that need it - a few can already handle arbitrary size bios. Note that we add the call _after_ any call toblk_queue_bounce(); this means thatblk_queue_split()andblk_recalc_rq_segments()don't need to be concerned with bouncing affecting segment merging.Some
make_request_fnswere simple enough to audit and verify they don't needblk_queue_split()calls. The skipped ones are:*
nfhd_make_request(arch/m68k/emu/nfblock.c)
*axon_ram_make_request(arch/powerpc/sysdev/axonram.c)
*simdisk_make_request(arch/xtensa/platforms/iss/simdisk.c)
*brd_make_request(ramdisk -drivers/block/brd.c)
*loop_make_request
*null_queue_bio
* bcache'smake_requestfnsSome others are almost certainly safe to remove now, but will be left for future patches.
Signed-off-by Kent Overstreet <[email protected]>Cc Jens Axboe <[email protected]>Cc Neil Brown <[email protected]>Cc Alasdair Kergon <[email protected]>Cc Lars Ellenberg <[email protected]>Cc Asai Thambi S P <[email protected]>Cc Sam Bradshaw <[email protected]>Cc Matthew Wilcox <[email protected]>Cc Jiri Kosina <[email protected]>Cc Geoff Levand <[email protected]>Cc Jim Paris <[email protected]>Cc Joshua Morris <[email protected]>Cc Philip Kelleher <[email protected]>Cc Minchan Kim <[email protected]>Cc Nitin Gupta <[email protected]>Cc Martin Schwidefsky <[email protected]>Cc Heiko Carstens <[email protected]>Cc Peng Tao <[email protected]>--- block/blk-core.c | 19 ++-- block/blk-merge.c | 150 ++++++++++++++++++++++++++-- block/blk-mq.c | 2 + drivers/block/drbd/drbd_req.c | 2 + drivers/block/mtip32xx/mtip32xx.c | 6 +- drivers/block/nvme-core.c | 2 + drivers/block/pktcdvd.c | 6 +- drivers/block/ps3vram.c | 2 + drivers/block/rsxx/dev.c | 2 + drivers/block/umem.c | 2 + drivers/block/zram/zram_drv.c | 2 + drivers/md/dm.c | 2 + drivers/md/md.c | 2 + drivers/s390/block/dcssblk.c | 2 + drivers/s390/block/xpram.c | 2 + drivers/staging/lustre/lustre/llite/lloop.c | 2 + include/linux/blkdev.h | 3 + 17 files changed, 185 insertions(+), 23 deletions(-)block/blk-core.c +9
-10@@ -581,6 +581,10 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)if (q->id <0 ) goto fail_c; + q->bio_split =bioset_create (4 ,0 ); +if (!q->bio_split) + goto fail_id; + q->backing_dev_info.ra_pages = (VM_MAX_READAHEAD *1024 ) /PAGE_CACHE_SIZE ; q->backing_dev_info.state =0 ;@@ -590,7 +594,7 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) err =bdi_init (&q->backing_dev_info);if (err) - goto fail_id; + goto fail_split;setup_timer (&q->backing_dev_info.laptop_mode_wb_timer, laptop_mode_timer_fn, (unsigned long ) q);@@ -635,6 +639,8 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) fail_bdi:bdi_destroy (&q->backing_dev_info); +fail_split: +bioset_free (q->bio_split); fail_id:ida_simple_remove (&blk_queue_ida, q->id); fail_c:@@ -1501,6 +1507,8 @@ void blk_queue_bio(struct request_queue *q, struct bio *bio)struct request *req;unsigned int request_count =0 ; +blk_queue_split (q, &bio, q->bio_split); +/* * low level driver can indicate that it wants pages above a * certain limit bounced to low memory (ie for highmem, or even @@ -1723,15 +1731,6 @@ generic_make_request_checks(struct bio *bio) goto end_io; } -if (likely (bio_is_rw (bio) && - nr_sectors >queue_max_hw_sectors (q))) { -printk (KERN_ERR "bio too big device %s (%u > %u)\n" , -bdevname (bio->bi_bdev, b), -bio_sectors (bio), -queue_max_hw_sectors (q)); - goto end_io; - } - part = bio->bi_bdev->bd_part;if (should_fail_request (part, bio->bi_iter.bi_size) || should_fail_request(&part_to_disk(part)->part0,block/blk-merge.c +141
-9@@ -9,11 +9,149 @@#include "blk.h" +static struct bio *blk_bio_discard_split (struct request_queue *q, +struct bio *bio, +struct bio_set *bs) +{ +unsigned int max_discard_sectors, granularity; +int alignment; +sector_t tmp; +unsigned split_sectors; + +/* Zero-sector (unknown) and one-sector granularities are the same. */ + granularity =max (q->limits.discard_granularity >>9 ,1U ); + + max_discard_sectors =min (q->limits.max_discard_sectors,UINT_MAX >>9 ); + max_discard_sectors -= max_discard_sectors % granularity; + +if (unlikely (!max_discard_sectors)) { +/* XXX: warn */ +return NULL ; + } + +if (bio_sectors (bio) <= max_discard_sectors) +return NULL ; + + split_sectors = max_discard_sectors; + +/* +* If the next starting sector would be misaligned, stop the discard at +* the previous aligned sector. +*/ + alignment = (q->limits.discard_alignment >>9 ) % granularity; + + tmp = bio->bi_iter.bi_sector + split_sectors - alignment; + tmp =sector_div (tmp, granularity); + +if (split_sectors > tmp) + split_sectors -= tmp; + +return bio_split (bio, split_sectors,GFP_NOIO , bs); +} + +static struct bio *blk_bio_write_same_split (struct request_queue *q, +struct bio *bio, +struct bio_set *bs) +{ +if (!q->limits.max_write_same_sectors) +return NULL ; + +if (bio_sectors (bio) <= q->limits.max_write_same_sectors) +return NULL ; + +return bio_split (bio, q->limits.max_write_same_sectors,GFP_NOIO , bs); +} + +static struct bio *blk_bio_segment_split (struct request_queue *q, +struct bio *bio, +struct bio_set *bs) +{ +struct bio *split; +struct bio_vec bv, bvprv; +struct bvec_iter iter; +unsigned seg_size =0 , nsegs =0 ; +int prev =0 ; + +struct bvec_merge_data bvm = { + .bi_bdev = bio->bi_bdev, + .bi_sector = bio->bi_iter.bi_sector, + .bi_size =0 , + .bi_rw = bio->bi_rw, + }; + +bio_for_each_segment (bv, bio, iter) { +if (q->merge_bvec_fn && + q->merge_bvec_fn (q, &bvm, &bv) < (int ) bv.bv_len) + goto split; + + bvm.bi_size += bv.bv_len; + +if (bvm.bi_size >>9 >queue_max_sectors (q)) + goto split; + +if (prev &&blk_queue_cluster (q)) { +if (seg_size + bv.bv_len >queue_max_segment_size (q)) + goto new_segment; +if (!BIOVEC_PHYS_MERGEABLE (&bvprv, &bv)) + goto new_segment; +if (!BIOVEC_SEG_BOUNDARY (q, &bvprv, &bv)) + goto new_segment; + + seg_size += bv.bv_len; + bvprv = bv; + prev =1 ; +continue ; + } +new_segment: +if (nsegs ==queue_max_segments (q)) + goto split; + + nsegs++; + bvprv = bv; + prev =1 ; + seg_size = bv.bv_len; + } + +return NULL ; +split: + split =bio_clone_bioset (bio,GFP_NOIO , bs); + + split->bi_iter.bi_size -= iter.bi_size; + bio->bi_iter = iter; + +if (bio_integrity (bio)) { +bio_integrity_advance (bio, split->bi_iter.bi_size); +bio_integrity_trim (split,0 ,bio_sectors (split)); + } + +return split; +} + +void blk_queue_split (struct request_queue *q,struct bio **bio, +struct bio_set *bs) +{ +struct bio *split; + +if ((*bio)->bi_rw &REQ_DISCARD ) + split =blk_bio_discard_split (q, *bio, bs); +else if ((*bio)->bi_rw &REQ_WRITE_SAME ) + split =blk_bio_write_same_split (q, *bio, bs); +else + split =blk_bio_segment_split (q, *bio, q->bio_split); + +if (split) { +bio_chain (split, *bio); +generic_make_request (*bio); + *bio = split; + } +} +EXPORT_SYMBOL (blk_queue_split); +static unsigned int __blk_recalc_rq_segments (struct request_queue *q,struct bio *bio) {struct bio_vec bv, bvprv = {NULL }; -int cluster, high, highprv =1 ; +int cluster, prev =0 ;unsigned int seg_size, nr_phys_segs;struct bio *fbio, *bbio;struct bvec_iter iter;@@ -37,13 +175,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, nr_phys_segs =0 ;for_each_bio (bio) {bio_for_each_segment (bv, bio, iter) { -/* -* the trick here is making sure that a high page is -* never considered part of another segment, since that -* might change with the bounce page. -*/ - high =page_to_pfn (bv.bv_page) >queue_bounce_pfn (q); -if (!high && !highprv && cluster) { +if (prev && cluster) {if (seg_size + bv.bv_len >queue_max_segment_size (q)) goto new_segment;@@ -63,8 +195,8 @@ new_segment: nr_phys_segs++; bvprv = bv; + prev =1 ; seg_size = bv.bv_len; - highprv = high; } bbio = bio; }block/blk-mq.c +2
-0@@ -915,6 +915,8 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio)return ; } +blk_queue_split (q, &bio, q->bio_split); +if (use_plug &&blk_attempt_plug_merge (q, bio, &request_count))return ;drivers/block/drbd/drbd_req.c +2
-0@@ -1275,6 +1275,8 @@ void drbd_make_request(struct request_queue *q, struct bio *bio)struct drbd_conf *mdev = (struct drbd_conf *) q->queuedata;unsigned long start_time; +blk_queue_split (q, &bio, q->bio_split); + start_time = jiffies;/* drivers/block/mtip32xx/mtip32xx.c +4
-2@@ -4033,6 +4033,10 @@ static void mtip_make_request(struct request_queue *queue, struct bio *bio)int nents =0 ;int tag =0 , unaligned =0 ; +blk_queue_bounce (queue, &bio); + +blk_queue_split (queue, &bio, queue->bio_split); +if (unlikely (dd->dd_flag &MTIP_DDF_STOP_IO )) {if (unlikely (test_bit (MTIP_DDF_REMOVE_PENDING_BIT , &dd->dd_flag))) {@@ -4082,8 +4086,6 @@ static void mtip_make_request(struct request_queue *queue, struct bio *bio) sg =mtip_hw_get_scatterlist (dd, &tag, unaligned);if (likely (sg !=NULL )) { -blk_queue_bounce (queue, &bio); -if (unlikely ((bio)->bi_vcnt >MTIP_MAX_SG )) {dev_warn (&dd->pdev->dev,"Maximum number of SGL entries exceeded\n" );drivers/block/nvme-core.c +2
-0@@ -737,6 +737,8 @@ static void nvme_make_request(struct request_queue *q, struct bio *bio)struct nvme_queue *nvmeq =get_nvmeq (ns->dev);int result = -EBUSY ; +blk_queue_split (q, &bio, q->bio_split); +if (!nvmeq) {put_nvmeq (NULL );bio_endio (bio, -EIO );drivers/block/pktcdvd.c +4
-2@@ -2444,6 +2444,10 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio)char b[BDEVNAME_SIZE ];struct bio *split; +blk_queue_bounce (q, &bio); + +blk_queue_split (q, &bio, q->bio_split); + pd = q->queuedata;if (!pd) { pr_err("%s incorrect request queue\n",@@ -2474,8 +2478,6 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio) goto end_io; } -blk_queue_bounce (q, &bio); - do { sector_t zone = get_zone(bio->bi_iter.bi_sector, pd); sector_t last_zone = get_zone(bio_end_sector(bio) - 1, pd);drivers/block/ps3vram.c +2
-0@@ -603,6 +603,8 @@ static void ps3vram_make_request(struct request_queue *q, struct bio *bio)struct ps3vram_priv *priv =ps3_system_bus_get_drvdata (dev);int busy; +blk_queue_split (q, &bio, q->bio_split); +dev_dbg (&dev->core,"%s\n" , __func__);spin_lock_irq (&priv->lock);drivers/block/rsxx/dev.c +2
-0@@ -169,6 +169,8 @@ static void rsxx_make_request(struct request_queue *q, struct bio *bio)struct rsxx_bio_meta *bio_meta;int st = -EINVAL ; +blk_queue_split (q, &bio, q->bio_split); +might_sleep ();if (!card)drivers/block/umem.c +2
-0@@ -531,6 +531,8 @@ static void mm_make_request(struct request_queue *q, struct bio *bio) (unsigned long long )bio->bi_iter.bi_sector, bio->bi_iter.bi_size); +blk_queue_split (q, &bio, q->bio_split); +spin_lock_irq (&card->lock); *card->biotail = bio; bio->bi_next =NULL ;drivers/block/zram/zram_drv.c +2
-0@@ -733,6 +733,8 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio) {struct zram *zram = queue->queuedata; +blk_queue_split (queue, &bio, queue->bio_split); +down_read (&zram->init_lock);if (unlikely (!zram->init_done)) goto error;drivers/md/dm.c +2
-0@@ -1500,6 +1500,8 @@ static void dm_request(struct request_queue *q, struct bio *bio) {struct mapped_device *md = q->queuedata; +blk_queue_split (q, &bio, q->bio_split); +if (dm_request_based (md))blk_queue_bio (q, bio);else drivers/md/md.c +2
-0@@ -256,6 +256,8 @@ static void md_make_request(struct request_queue *q, struct bio *bio)int cpu;unsigned int sectors; +blk_queue_split (q, &bio, q->bio_split); +if (mddev ==NULL || mddev->pers ==NULL || !mddev->ready) {bio_io_error (bio);drivers/s390/block/dcssblk.c +2
-0@@ -815,6 +815,8 @@ dcssblk_make_request(struct request_queue *q, struct bio *bio)unsigned long source_addr;unsigned long bytes_done; +blk_queue_split (q, &bio, q->bio_split); + bytes_done =0 ; dev_info = bio->bi_bdev->bd_disk->private_data;if (dev_info ==NULL )drivers/s390/block/xpram.c +2
-0@@ -190,6 +190,8 @@ static void xpram_make_request(struct request_queue *q, struct bio *bio)unsigned long page_addr;unsigned long bytes; +blk_queue_split (q, &bio, q->bio_split); +if ((bio->bi_iter.bi_sector &7 ) !=0 || (bio->bi_iter.bi_size &4095 ) !=0 )/* Request is not page-aligned. */ drivers/staging/lustre/lustre/llite/lloop.c +2
-0@@ -344,6 +344,8 @@ static void loop_make_request(struct request_queue *q, struct bio *old_bio)int rw =bio_rw (old_bio);int inactive; +blk_queue_split (q, &old_bio, q->bio_split); +if (!lo) goto err;include/linux/blkdev.h +3
-1@@ -470,6 +470,7 @@ struct request_queue {wait_queue_head_t mq_freeze_wq;struct percpu_counter mq_usage_counter;struct list_head all_q_node; +struct bio_set *bio_split; };#define QUEUE_FLAG_QUEUED 1/* uses generic tag queueing */ @@ -781,6 +782,8 @@ extern void blk_rq_unprep_clone(struct request *rq);extern int blk_insert_cloned_request (struct request_queue *q,struct request *rq);extern void blk_delay_queue (struct request_queue *,unsigned long ); +extern void blk_queue_split (struct request_queue *,struct bio **, +struct bio_set *);extern void blk_recount_segments (struct request_queue *,struct bio *);extern int scsi_verify_blk_ioctl (struct block_device *,unsigned int ); extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t, --1.9.0
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Re: [PATCH 1/9] block: Make generic_make_request handle arbitrary sized bios
Never mind...
The following code covers it:
+ if (split) {
+ bio_chain(split, *bio); + generic_make_request(*bio); + *bio = split;
+ }
My other question is, can we avoid calling the queue_split from individual drivers make_request()? Can we move the functionality into generic_make_request()?
Thanks.
Regards,
Muthu
Quoted 561 lines · Muthu Kumar
Kent,
Theblk_queue_split(), splits a bio into at most two bios right? So, if the original bio spans larger space than two bios can cover (restriction by the lower driver in the stack), this might not work? Am I reading it incorrectly?Thanks!
Regards,
MuthuQuoted 547 lines · Kent Overstreet
The way the block layer is currently written, it goes to great lengths to avoid having to split bios; upper layer code (such as
bio_add_page()) checks what the underlying device can handle and tries to always create bios that don't need to be split.But this approach becomes unwieldy and eventually breaks down with stacked devices and devices with dynamic limits, and it adds a lot of complexity. If the block layer could split bios as needed, we could eliminate a lot of complexity elsewhere - particularly in stacked drivers. Code that creates bios can then create whatever size bios are convenient, and more importantly stacked drivers don't have to deal with both their own bio size limitations and the limitations of the (potentially multiple) devices underneath them. In the future this will let us delete
merge_bvec_fnand a bunch of other code.We do this by adding calls to
blk_queue_split()to the variousmake_requestfunctions that need it - a few can already handle arbitrary size bios. Note that we add the call _after_ any call toblk_queue_bounce(); this means thatblk_queue_split()andblk_recalc_rq_segments()don't need to be concerned with bouncing affecting segment merging.Some
make_request_fnswere simple enough to audit and verify they don't needblk_queue_split()calls. The skipped ones are:*
nfhd_make_request(arch/m68k/emu/nfblock.c)
*axon_ram_make_request(arch/powerpc/sysdev/axonram.c)
*simdisk_make_request(arch/xtensa/platforms/iss/simdisk.c)
*brd_make_request(ramdisk -drivers/block/brd.c)
*loop_make_request
*null_queue_bio
* bcache'smake_requestfnsSome others are almost certainly safe to remove now, but will be left for future patches.
Signed-off-by Kent Overstreet <[email protected]>Cc Jens Axboe <[email protected]>Cc Neil Brown <[email protected]>Cc Alasdair Kergon <[email protected]>Cc Lars Ellenberg <[email protected]>Cc Asai Thambi S P <[email protected]>Cc Sam Bradshaw <[email protected]>Cc Matthew Wilcox <[email protected]>Cc Jiri Kosina <[email protected]>Cc Geoff Levand <[email protected]>Cc Jim Paris <[email protected]>Cc Joshua Morris <[email protected]>Cc Philip Kelleher <[email protected]>Cc Minchan Kim <[email protected]>Cc Nitin Gupta <[email protected]>Cc Martin Schwidefsky <[email protected]>Cc Heiko Carstens <[email protected]>Cc Peng Tao <[email protected]>--- block/blk-core.c | 19 ++-- block/blk-merge.c | 150 ++++++++++++++++++++++++++-- block/blk-mq.c | 2 + drivers/block/drbd/drbd_req.c | 2 + drivers/block/mtip32xx/mtip32xx.c | 6 +- drivers/block/nvme-core.c | 2 + drivers/block/pktcdvd.c | 6 +- drivers/block/ps3vram.c | 2 + drivers/block/rsxx/dev.c | 2 + drivers/block/umem.c | 2 + drivers/block/zram/zram_drv.c | 2 + drivers/md/dm.c | 2 + drivers/md/md.c | 2 + drivers/s390/block/dcssblk.c | 2 + drivers/s390/block/xpram.c | 2 + drivers/staging/lustre/lustre/llite/lloop.c | 2 + include/linux/blkdev.h | 3 + 17 files changed, 185 insertions(+), 23 deletions(-)block/blk-core.c +9
-10@@ -581,6 +581,10 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)if (q->id <0 ) goto fail_c; + q->bio_split =bioset_create (4 ,0 ); +if (!q->bio_split) + goto fail_id; + q->backing_dev_info.ra_pages = (VM_MAX_READAHEAD *1024 ) /PAGE_CACHE_SIZE ; q->backing_dev_info.state =0 ;@@ -590,7 +594,7 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) err =bdi_init (&q->backing_dev_info);if (err) - goto fail_id; + goto fail_split;setup_timer (&q->backing_dev_info.laptop_mode_wb_timer, laptop_mode_timer_fn, (unsigned long ) q);@@ -635,6 +639,8 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) fail_bdi:bdi_destroy (&q->backing_dev_info); +fail_split: +bioset_free (q->bio_split); fail_id:ida_simple_remove (&blk_queue_ida, q->id); fail_c:@@ -1501,6 +1507,8 @@ void blk_queue_bio(struct request_queue *q, struct bio *bio)struct request *req;unsigned int request_count =0 ; +blk_queue_split (q, &bio, q->bio_split); +/* * low level driver can indicate that it wants pages above a * certain limit bounced to low memory (ie for highmem, or even @@ -1723,15 +1731,6 @@ generic_make_request_checks(struct bio *bio) goto end_io; } -if (likely (bio_is_rw (bio) && - nr_sectors >queue_max_hw_sectors (q))) { -printk (KERN_ERR "bio too big device %s (%u > %u)\n" , -bdevname (bio->bi_bdev, b), -bio_sectors (bio), -queue_max_hw_sectors (q)); - goto end_io; - } - part = bio->bi_bdev->bd_part;if (should_fail_request (part, bio->bi_iter.bi_size) || should_fail_request(&part_to_disk(part)->part0,block/blk-merge.c +141
-9@@ -9,11 +9,149 @@#include "blk.h" +static struct bio *blk_bio_discard_split (struct request_queue *q, +struct bio *bio, +struct bio_set *bs) +{ +unsigned int max_discard_sectors, granularity; +int alignment; +sector_t tmp; +unsigned split_sectors; + +/* Zero-sector (unknown) and one-sector granularities are the same. */ + granularity =max (q->limits.discard_granularity >>9 ,1U ); + + max_discard_sectors =min (q->limits.max_discard_sectors,UINT_MAX >>9 ); + max_discard_sectors -= max_discard_sectors % granularity; + +if (unlikely (!max_discard_sectors)) { +/* XXX: warn */ +return NULL ; + } + +if (bio_sectors (bio) <= max_discard_sectors) +return NULL ; + + split_sectors = max_discard_sectors; + +/* +* If the next starting sector would be misaligned, stop the discard at +* the previous aligned sector. +*/ + alignment = (q->limits.discard_alignment >>9 ) % granularity; + + tmp = bio->bi_iter.bi_sector + split_sectors - alignment; + tmp =sector_div (tmp, granularity); + +if (split_sectors > tmp) + split_sectors -= tmp; + +return bio_split (bio, split_sectors,GFP_NOIO , bs); +} + +static struct bio *blk_bio_write_same_split (struct request_queue *q, +struct bio *bio, +struct bio_set *bs) +{ +if (!q->limits.max_write_same_sectors) +return NULL ; + +if (bio_sectors (bio) <= q->limits.max_write_same_sectors) +return NULL ; + +return bio_split (bio, q->limits.max_write_same_sectors,GFP_NOIO , bs); +} + +static struct bio *blk_bio_segment_split (struct request_queue *q, +struct bio *bio, +struct bio_set *bs) +{ +struct bio *split; +struct bio_vec bv, bvprv; +struct bvec_iter iter; +unsigned seg_size =0 , nsegs =0 ; +int prev =0 ; + +struct bvec_merge_data bvm = { + .bi_bdev = bio->bi_bdev, + .bi_sector = bio->bi_iter.bi_sector, + .bi_size =0 , + .bi_rw = bio->bi_rw, + }; + +bio_for_each_segment (bv, bio, iter) { +if (q->merge_bvec_fn && + q->merge_bvec_fn (q, &bvm, &bv) < (int ) bv.bv_len) + goto split; + + bvm.bi_size += bv.bv_len; + +if (bvm.bi_size >>9 >queue_max_sectors (q)) + goto split; + +if (prev &&blk_queue_cluster (q)) { +if (seg_size + bv.bv_len >queue_max_segment_size (q)) + goto new_segment; +if (!BIOVEC_PHYS_MERGEABLE (&bvprv, &bv)) + goto new_segment; +if (!BIOVEC_SEG_BOUNDARY (q, &bvprv, &bv)) + goto new_segment; + + seg_size += bv.bv_len; + bvprv = bv; + prev =1 ; +continue ; + } +new_segment: +if (nsegs ==queue_max_segments (q)) + goto split; + + nsegs++; + bvprv = bv; + prev =1 ; + seg_size = bv.bv_len; + } + +return NULL ; +split: + split =bio_clone_bioset (bio,GFP_NOIO , bs); + + split->bi_iter.bi_size -= iter.bi_size; + bio->bi_iter = iter; + +if (bio_integrity (bio)) { +bio_integrity_advance (bio, split->bi_iter.bi_size); +bio_integrity_trim (split,0 ,bio_sectors (split)); + } + +return split; +} + +void blk_queue_split (struct request_queue *q,struct bio **bio, +struct bio_set *bs) +{ +struct bio *split; + +if ((*bio)->bi_rw &REQ_DISCARD ) + split =blk_bio_discard_split (q, *bio, bs); +else if ((*bio)->bi_rw &REQ_WRITE_SAME ) + split =blk_bio_write_same_split (q, *bio, bs); +else + split =blk_bio_segment_split (q, *bio, q->bio_split); + +if (split) { +bio_chain (split, *bio); +generic_make_request (*bio); + *bio = split; + } +} +EXPORT_SYMBOL (blk_queue_split); +static unsigned int __blk_recalc_rq_segments (struct request_queue *q,struct bio *bio) {struct bio_vec bv, bvprv = {NULL }; -int cluster, high, highprv =1 ; +int cluster, prev =0 ;unsigned int seg_size, nr_phys_segs;struct bio *fbio, *bbio;struct bvec_iter iter;@@ -37,13 +175,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, nr_phys_segs =0 ;for_each_bio (bio) {bio_for_each_segment (bv, bio, iter) { -/* -* the trick here is making sure that a high page is -* never considered part of another segment, since that -* might change with the bounce page. -*/ - high =page_to_pfn (bv.bv_page) >queue_bounce_pfn (q); -if (!high && !highprv && cluster) { +if (prev && cluster) {if (seg_size + bv.bv_len >queue_max_segment_size (q)) goto new_segment;@@ -63,8 +195,8 @@ new_segment: nr_phys_segs++; bvprv = bv; + prev =1 ; seg_size = bv.bv_len; - highprv = high; } bbio = bio; }block/blk-mq.c +2
-0@@ -915,6 +915,8 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio)return ; } +blk_queue_split (q, &bio, q->bio_split); +if (use_plug &&blk_attempt_plug_merge (q, bio, &request_count))return ;drivers/block/drbd/drbd_req.c +2
-0@@ -1275,6 +1275,8 @@ void drbd_make_request(struct request_queue *q, struct bio *bio)struct drbd_conf *mdev = (struct drbd_conf *) q->queuedata;unsigned long start_time; +blk_queue_split (q, &bio, q->bio_split); + start_time = jiffies;/* drivers/block/mtip32xx/mtip32xx.c +4
-2@@ -4033,6 +4033,10 @@ static void mtip_make_request(struct request_queue *queue, struct bio *bio)int nents =0 ;int tag =0 , unaligned =0 ; +blk_queue_bounce (queue, &bio); + +blk_queue_split (queue, &bio, queue->bio_split); +if (unlikely (dd->dd_flag &MTIP_DDF_STOP_IO )) {if (unlikely (test_bit (MTIP_DDF_REMOVE_PENDING_BIT , &dd->dd_flag))) {@@ -4082,8 +4086,6 @@ static void mtip_make_request(struct request_queue *queue, struct bio *bio) sg =mtip_hw_get_scatterlist (dd, &tag, unaligned);if (likely (sg !=NULL )) { -blk_queue_bounce (queue, &bio); -if (unlikely ((bio)->bi_vcnt >MTIP_MAX_SG )) {dev_warn (&dd->pdev->dev,"Maximum number of SGL entries exceeded\n" );drivers/block/nvme-core.c +2
-0@@ -737,6 +737,8 @@ static void nvme_make_request(struct request_queue *q, struct bio *bio)struct nvme_queue *nvmeq =get_nvmeq (ns->dev);int result = -EBUSY ; +blk_queue_split (q, &bio, q->bio_split); +if (!nvmeq) {put_nvmeq (NULL );bio_endio (bio, -EIO );drivers/block/pktcdvd.c +4
-2@@ -2444,6 +2444,10 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio)char b[BDEVNAME_SIZE ];struct bio *split; +blk_queue_bounce (q, &bio); + +blk_queue_split (q, &bio, q->bio_split); + pd = q->queuedata;if (!pd) { pr_err("%s incorrect request queue\n",@@ -2474,8 +2478,6 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio) goto end_io; } -blk_queue_bounce (q, &bio); - do { sector_t zone = get_zone(bio->bi_iter.bi_sector, pd); sector_t last_zone = get_zone(bio_end_sector(bio) - 1, pd);drivers/block/ps3vram.c +2
-0@@ -603,6 +603,8 @@ static void ps3vram_make_request(struct request_queue *q, struct bio *bio)struct ps3vram_priv *priv =ps3_system_bus_get_drvdata (dev);int busy; +blk_queue_split (q, &bio, q->bio_split); +dev_dbg (&dev->core,"%s\n" , __func__);spin_lock_irq (&priv->lock);drivers/block/rsxx/dev.c +2
-0@@ -169,6 +169,8 @@ static void rsxx_make_request(struct request_queue *q, struct bio *bio)struct rsxx_bio_meta *bio_meta;int st = -EINVAL ; +blk_queue_split (q, &bio, q->bio_split); +might_sleep ();if (!card)drivers/block/umem.c +2
-0@@ -531,6 +531,8 @@ static void mm_make_request(struct request_queue *q, struct bio *bio) (unsigned long long )bio->bi_iter.bi_sector, bio->bi_iter.bi_size); +blk_queue_split (q, &bio, q->bio_split); +spin_lock_irq (&card->lock); *card->biotail = bio; bio->bi_next =NULL ;drivers/block/zram/zram_drv.c +2
-0@@ -733,6 +733,8 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio) {struct zram *zram = queue->queuedata; +blk_queue_split (queue, &bio, queue->bio_split); +down_read (&zram->init_lock);if (unlikely (!zram->init_done)) goto error;drivers/md/dm.c +2
-0@@ -1500,6 +1500,8 @@ static void dm_request(struct request_queue *q, struct bio *bio) {struct mapped_device *md = q->queuedata; +blk_queue_split (q, &bio, q->bio_split); +if (dm_request_based (md))blk_queue_bio (q, bio);else drivers/md/md.c +2
-0@@ -256,6 +256,8 @@ static void md_make_request(struct request_queue *q, struct bio *bio)int cpu;unsigned int sectors; +blk_queue_split (q, &bio, q->bio_split); +if (mddev ==NULL || mddev->pers ==NULL || !mddev->ready) {bio_io_error (bio);drivers/s390/block/dcssblk.c +2
-0@@ -815,6 +815,8 @@ dcssblk_make_request(struct request_queue *q, struct bio *bio)unsigned long source_addr;unsigned long bytes_done; +blk_queue_split (q, &bio, q->bio_split); + bytes_done =0 ; dev_info = bio->bi_bdev->bd_disk->private_data;if (dev_info ==NULL )drivers/s390/block/xpram.c +2
-0@@ -190,6 +190,8 @@ static void xpram_make_request(struct request_queue *q, struct bio *bio)unsigned long page_addr;unsigned long bytes; +blk_queue_split (q, &bio, q->bio_split); +if ((bio->bi_iter.bi_sector &7 ) !=0 || (bio->bi_iter.bi_size &4095 ) !=0 )/* Request is not page-aligned. */ drivers/staging/lustre/lustre/llite/lloop.c +2
-0@@ -344,6 +344,8 @@ static void loop_make_request(struct request_queue *q, struct bio *old_bio)int rw =bio_rw (old_bio);int inactive; +blk_queue_split (q, &old_bio, q->bio_split); +if (!lo) goto err;include/linux/blkdev.h +3
-1@@ -470,6 +470,7 @@ struct request_queue {wait_queue_head_t mq_freeze_wq;struct percpu_counter mq_usage_counter;struct list_head all_q_node; +struct bio_set *bio_split; };#define QUEUE_FLAG_QUEUED 1/* uses generic tag queueing */ @@ -781,6 +782,8 @@ extern void blk_rq_unprep_clone(struct request *rq);extern int blk_insert_cloned_request (struct request_queue *q,struct request *rq);extern void blk_delay_queue (struct request_queue *,unsigned long ); +extern void blk_queue_split (struct request_queue *,struct bio **, +struct bio_set *);extern void blk_recount_segments (struct request_queue *,struct bio *);extern int scsi_verify_blk_ioctl (struct block_device *,unsigned int ); extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t, --1.9.0
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
PATCH 2/9 block: Gut bio_add_page()
In reply to Kent OverstreetSince generic_make_request() can now handle arbitrary size bios, all we have to do is make sure the bvec array doesn't overflow.
--- fs/bio.c | 137 ++++++++++++++++++++++++++------------------------------------- 1 file changed, 57 insertions(+), 80 deletions(-)
fs/bio.c +57 -80
@@ -694,9 +694,23 @@ int bio_get_nr_vecs(struct block_device *bdev) }EXPORT_SYMBOL (bio_get_nr_vecs); -static int __bio_add_page (struct request_queue *q,struct bio *bio,struct page - *page,unsigned int len,unsigned int offset, -unsigned int max_sectors) +/** +* bio_add_pc_page - attempt to add page to bio +* @q: the target queue +* @bio: destination bio +* @page: page to add +* @len: vec entry length +* @offset: vec entry offset +* +* Attempt to add a page to the bio_vec maplist. This can fail for a +* number of reasons, such as the bio being full or target block device +* limitations. The target block device must allow bio's up to PAGE_SIZE, +* so it is always possible to add a single page to an empty bio. +* +* This should only be used by REQ_PC bios. +*/ +int bio_add_pc_page (struct request_queue *q,struct bio *bio,struct page + *page,unsigned int len,unsigned int offset) {int retried_segments =0 ;struct bio_vec *bvec;
@@ -707,7 +721,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct pageif (unlikely (bio_flagged (bio,BIO_CLONED )))return 0 ; -if (((bio->bi_iter.bi_size + len) >>9 ) > max_sectors) +if (((bio->bi_iter.bi_size + len) >>9 ) >queue_max_hw_sectors (q))return 0 ;/*
@@ -720,28 +734,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct pageif (page == prev->bv_page && offset == prev->bv_offset + prev->bv_len) { -unsigned int prev_bv_len = prev->bv_len; prev->bv_len += len; - -if (q->merge_bvec_fn) { -struct bvec_merge_data bvm = { -/* prev_bvec is already charged in -bi_size, discharge it in order to -simulate merging updated prev_bvec -as new bvec. */ - .bi_bdev = bio->bi_bdev, - .bi_sector = bio->bi_iter.bi_sector, - .bi_size = bio->bi_iter.bi_size - - prev_bv_len, - .bi_rw = bio->bi_rw, - }; - -if (q->merge_bvec_fn (q, &bvm, prev) < prev->bv_len) { - prev->bv_len -= len; -return 0 ; - } - } - goto done; } }
@@ -772,31 +765,6 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page bvec->bv_len = len; bvec->bv_offset = offset; -/* -* if queue has other restrictions (eg varying max sector size -* depending on offset), it can specify a merge_bvec_fn in the -* queue to get further control -*/ -if (q->merge_bvec_fn) { -struct bvec_merge_data bvm = { - .bi_bdev = bio->bi_bdev, - .bi_sector = bio->bi_iter.bi_sector, - .bi_size = bio->bi_iter.bi_size, - .bi_rw = bio->bi_rw, - }; - -/* -* merge_bvec_fn() returns number of bytes it can accept -* at this offset -*/ -if (q->merge_bvec_fn (q, &bvm, bvec) < bvec->bv_len) { - bvec->bv_page =NULL ; - bvec->bv_len =0 ; - bvec->bv_offset =0 ; -return 0 ; - } - } -/* If we may be able to merge these biovecs, force a recount */ if (bio->bi_vcnt && (BIOVEC_PHYS_MERGEABLE (bvec-1 , bvec))) bio->bi_flags &= ~(1 <<BIO_SEG_VALID );
@@ -807,28 +775,6 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page bio->bi_iter.bi_size += len;return len; } - -/** -* bio_add_pc_page - attempt to add page to bio -* @q: the target queue -* @bio: destination bio -* @page: page to add -* @len: vec entry length -* @offset: vec entry offset -* -* Attempt to add a page to the bio_vec maplist. This can fail for a -* number of reasons, such as the bio being full or target block device -* limitations. The target block device must allow bio's up to PAGE_SIZE, -* so it is always possible to add a single page to an empty bio. -* -* This should only be used by REQ_PC bios. -*/ -int bio_add_pc_page (struct request_queue *q,struct bio *bio,struct page *page, -unsigned int len,unsigned int offset) -{ -return __bio_add_page (q, bio, page, len, offset, -queue_max_hw_sectors (q)); -}EXPORT_SYMBOL (bio_add_pc_page);/**
@@ -838,16 +784,47 @@ EXPORT_SYMBOL(bio_add_pc_page);* @len: vec entry length * @offset: vec entry offset * -* Attempt to add a page to the bio_vec maplist. This can fail for a -* number of reasons, such as the bio being full or target block device -* limitations. The target block device must allow bio's up to PAGE_SIZE, -* so it is always possible to add a single page to an empty bio. +* Attempt to add a page to the bio_vec maplist. This will only fail if +* bio->bi_vcnt == bio->bi_max_vecs. */ -int bio_add_page (struct bio *bio,struct page *page,unsigned int len, -unsigned int offset) +int bio_add_page (struct bio *bio,struct page *page, +unsigned int len,unsigned int offset) { -struct request_queue *q =bdev_get_queue (bio->bi_bdev); -return __bio_add_page (q, bio, page, len, offset,queue_max_sectors (q)); +struct bio_vec *bv; + +/* +* cloned bio must not modify vec list +*/ +if (unlikely (bio_flagged (bio,BIO_CLONED ))) +return 0 ; + +/* +* For filesystems with a blocksize smaller than the pagesize +* we will often be called with the same page as last time and +* a consecutive offset. Optimize this special case. +*/ +if (bio->bi_vcnt >0 ) { + bv = &bio->bi_io_vec[bio->bi_vcnt -1 ]; + +if (page == bv->bv_page && + offset == bv->bv_offset + bv->bv_len) { + bv->bv_len += len; + goto done; + } + } + +if (bio->bi_vcnt >= bio->bi_max_vecs) +return 0 ; + + bv = &bio->bi_io_vec[bio->bi_vcnt]; + bv->bv_page = page; + bv->bv_len = len; + bv->bv_offset = offset; + + bio->bi_vcnt++; +done: + bio->bi_iter.bi_size += len; +return len; }EXPORT_SYMBOL (bio_add_page);
Signature · 1 line
1.9.0
PATCH 3/9 blk-lib.c: generic_make_request() handles large bios now
In reply to Kent Overstreetgeneric_make_request() will now do for us what the code in blk-lib.c was doing manually, with the bio_batch stuff - we still need some looping in case we're trying to discard/zeroout more than around a gigabyte, but when we can submit that much at a time doing the submissions in parallel really shouldn't matter.
--- block/blk-lib.c | 175 ++++++++++---------------------------------------------- 1 file changed, 30 insertions(+), 145 deletions(-)
block/blk-lib.c +30 -145
@@ -9,23 +9,6 @@#include "blk.h" -struct bio_batch { -atomic_t done; -unsigned long flags; -struct completion *wait; -}; - -static void bio_batch_end_io (struct bio *bio,int err) -{ -struct bio_batch *bb = bio->bi_private; - -if (err && (err != -EOPNOTSUPP )) -clear_bit (BIO_UPTODATE , &bb->flags); -if (atomic_dec_and_test (&bb->done)) -complete (bb->wait); -bio_put (bio); -} -/** * blkdev_issue_discard - queue a discard * @bdev: blockdev to issue discard for
@@ -40,15 +23,10 @@ static void bio_batch_end_io(struct bio *bio, int err)int blkdev_issue_discard (struct block_device *bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,unsigned long flags) { -DECLARE_COMPLETION_ONSTACK (wait);struct request_queue *q =bdev_get_queue (bdev);int type =REQ_WRITE |REQ_DISCARD ; -unsigned int max_discard_sectors, granularity; -int alignment; -struct bio_batch bb;struct bio *bio;int ret =0 ; -struct blk_plug plug;if (!q)return -ENXIO ;
@@ -56,69 +34,27 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector,if (!blk_queue_discard (q))return -EOPNOTSUPP ; -/* Zero-sector (unknown) and one-sector granularities are the same. */ - granularity =max (q->limits.discard_granularity >>9 ,1U ); - alignment = (bdev_discard_alignment (bdev) >>9 ) % granularity; - -/* -* Ensure that max_discard_sectors is of the proper -* granularity, so that requests stay aligned after a split. -*/ - max_discard_sectors =min (q->limits.max_discard_sectors,UINT_MAX >>9 ); - max_discard_sectors -= max_discard_sectors % granularity; -if (unlikely (!max_discard_sectors)) { -/* Avoid infinite loop below. Being cautious never hurts. */ -return -EOPNOTSUPP ; - } -if (flags &BLKDEV_DISCARD_SECURE ) {if (!blk_queue_secdiscard (q))return -EOPNOTSUPP ; type |=REQ_SECURE ; } -atomic_set (&bb.done,1 ); - bb.flags =1 <<BIO_UPTODATE ; - bb.wait = &wait; - -blk_start_plug (&plug);while (nr_sects) { -unsigned int req_sects; -sector_t end_sect, tmp; - bio =bio_alloc (gfp_mask,1 ); -if (!bio) { - ret = -ENOMEM ; -break ; - } - - req_sects =min_t (sector_t, nr_sects, max_discard_sectors); +if (!bio) +return -ENOMEM ; -/* -* If splitting a request, and the next starting sector would be -* misaligned, stop the discard at the previous aligned sector. -*/ - end_sect = sector + req_sects; - tmp = end_sect; -if (req_sects < nr_sects && -sector_div (tmp, granularity) != alignment) { - end_sect = end_sect - alignment; -sector_div (end_sect, granularity); - end_sect = end_sect * granularity + alignment; - req_sects = end_sect - sector; - } - - bio->bi_iter.bi_sector = sector; - bio->bi_end_io = bio_batch_end_io; bio->bi_bdev = bdev; - bio->bi_private = &bb; + bio->bi_iter.bi_sector = sector; + bio->bi_iter.bi_size =min_t (sector_t, nr_sects,1 <<20 ) <<9 ; - bio->bi_iter.bi_size = req_sects <<9 ; - nr_sects -= req_sects; - sector = end_sect; + sector +=bio_sectors (bio); + nr_sects -=bio_sectors (bio); -atomic_inc (&bb.done); -submit_bio (type, bio); + ret =submit_bio_wait (type, bio); +if (ret) +break ;/* * We can loop for a long time in here, if someone does
@@ -128,14 +64,6 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector,*/ cond_resched (); } -blk_finish_plug (&plug); - -/* Wait for bios in-flight */ -if (!atomic_dec_and_test (&bb.done)) -wait_for_completion_io (&wait); - -if (!test_bit (BIO_UPTODATE , &bb.flags)) - ret = -EIO ;return ret; }
@@ -156,61 +84,37 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, struct page *page) { -DECLARE_COMPLETION_ONSTACK (wait ); struct request_queue *q = bdev_get_queue(bdev); -unsigned int max_write_same_sectors; -struct bio_batch bb;struct bio *bio;int ret =0 ;if (!q)return -ENXIO ; - max_write_same_sectors = q->limits.max_write_same_sectors; - -if (max_write_same_sectors ==0 ) +if (!q->limits.max_write_same_sectors)return -EOPNOTSUPP ; -atomic_set (&bb.done,1 ); - bb.flags =1 <<BIO_UPTODATE ; - bb.wait = &wait; -while (nr_sects) { bio =bio_alloc (gfp_mask,1 ); -if (!bio) { - ret = -ENOMEM ; -break ; - } +if (!bio) +return -ENOMEM ; - bio->bi_iter.bi_sector = sector; - bio->bi_end_io = bio_batch_end_io; bio->bi_bdev = bdev; - bio->bi_private = &bb; + bio->bi_iter.bi_sector = sector; + bio->bi_iter.bi_size =min_t (sector_t, nr_sects,1 <<20 ) <<9 ; bio->bi_vcnt =1 ; bio->bi_io_vec->bv_page = page; bio->bi_io_vec->bv_offset =0 ; bio->bi_io_vec->bv_len =bdev_logical_block_size (bdev); -if (nr_sects > max_write_same_sectors) { - bio->bi_iter.bi_size = max_write_same_sectors <<9 ; - nr_sects -= max_write_same_sectors; - sector += max_write_same_sectors; - }else { - bio->bi_iter.bi_size = nr_sects <<9 ; - nr_sects =0 ; - } + sector +=bio_sectors (bio); + nr_sects -=bio_sectors (bio); -atomic_inc (&bb.done); -submit_bio (REQ_WRITE |REQ_WRITE_SAME , bio); + ret =submit_bio_wait (REQ_WRITE |REQ_WRITE_SAME , bio); +if (ret) +break ; } -/* Wait for bios in-flight */ -if (!atomic_dec_and_test (&bb.done)) -wait_for_completion_io (&wait); - -if (!test_bit (BIO_UPTODATE , &bb.flags)) - ret = -ENOTSUPP ; -return ret; }EXPORT_SYMBOL (blkdev_issue_write_same);
@@ -225,33 +129,22 @@ EXPORT_SYMBOL(blkdev_issue_write_same);* Description: * Generate and issue number of bios with zerofiled pages. */ -int __blkdev_issue_zeroout (struct block_device *bdev,sector_t sector, -sector_t nr_sects,gfp_t gfp_mask) +sector_t nr_sects,gfp_t gfp_mask) { -int ret; +int ret =0 ;struct bio *bio; -struct bio_batch bb;unsigned int sz; -DECLARE_COMPLETION_ONSTACK (wait); - -atomic_set (&bb.done,1 ); - bb.flags =1 <<BIO_UPTODATE ; - bb.wait = &wait; - ret =0 ; -while (nr_sects !=0 ) { +while (nr_sects) { bio =bio_alloc (gfp_mask, -min (nr_sects, (sector_t )BIO_MAX_PAGES )); -if (!bio) { - ret = -ENOMEM ; -break ; - } +min (nr_sects / (PAGE_SIZE >>9 ), + (sector_t )BIO_MAX_PAGES )); +if (!bio) +return -ENOMEM ; bio->bi_iter.bi_sector = sector; bio->bi_bdev = bdev; - bio->bi_end_io = bio_batch_end_io; - bio->bi_private = &bb;while (nr_sects !=0 ) { sz =min ((sector_t )PAGE_SIZE >>9 , nr_sects);
@@ -261,18 +154,11 @@ int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,if (ret < (sz <<9 ))break ; } - ret =0 ; -atomic_inc (&bb.done); -submit_bio (WRITE , bio); - } - -/* Wait for bios in-flight */ -if (!atomic_dec_and_test (&bb.done)) -wait_for_completion_io (&wait); -if (!test_bit (BIO_UPTODATE , &bb.flags)) -/* One of bios in the batch was completed with error.*/ - ret = -EIO ; + ret =submit_bio_wait (WRITE , bio); +if (ret) +break ; + }return ret; }
@@ -287,7 +173,6 @@ int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,* Description: * Generate and issue number of bios with zerofiled pages. */ -int blkdev_issue_zeroout (struct block_device *bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask) {
Signature · 1 line
1.9.0
PATCH 4/9 bcache: generic_make_request() handles large bios now
In reply to Kent OverstreetSo we get to delete our hacky workaround.
--- drivers/md/bcache/bcache.h | 18 -------- drivers/md/bcache/io.c | 100 +----------------------------------------- drivers/md/bcache/journal.c | 4 +- drivers/md/bcache/request.c | 16 +++---- drivers/md/bcache/super.c | 32 +------------- drivers/md/bcache/util.h | 5 ++- drivers/md/bcache/writeback.c | 4 +- 7 files changed, 18 insertions(+), 161 deletions(-)
drivers/md/bcache/bcache.h +0 -18
@@ -245,19 +245,6 @@ struct keybuf { DECLARE_ARRAY_ALLOCATOR(struct keybuf_key, freelist, KEYBUF_NR); }; -struct bio_split_pool { -struct bio_set *bio_split; -mempool_t *bio_split_hook; -}; - -struct bio_split_hook { -struct closure cl; -struct bio_split_pool *p; -struct bio *bio; -bio_end_io_t *bi_end_io; -void *bi_private; -}; -struct bcache_device {struct closure cl;
@@ -290,8 +277,6 @@ struct bcache_device {int (*cache_miss)(struct btree *,struct search *,struct bio *,unsigned );int (*ioctl) (struct bcache_device *,fmode_t ,unsigned ,unsigned long ); - -struct bio_split_pool bio_split_hook; };struct io {
@@ -467,8 +452,6 @@ struct cache {atomic_long_t meta_sectors_written;atomic_long_t btree_sectors_written;atomic_long_t sectors_written; - -struct bio_split_pool bio_split_hook; };struct gc_stat {
@@ -893,7 +876,6 @@ void bch_bbio_endio(struct cache_set *, struct bio *, int, const char *);void bch_bbio_free (struct bio *,struct cache_set *);struct bio *bch_bbio_alloc (struct cache_set *); -void bch_generic_make_request (struct bio *,struct bio_split_pool *);void __bch_submit_bbio (struct bio *,struct cache_set *);void bch_submit_bbio (struct bio *,struct cache_set *,struct bkey *,unsigned );
drivers/md/bcache/io.c +1 -99
112 lines not on this page · raw message
drivers/md/bcache/journal.c +2 -2
@@ -60,7 +60,7 @@ reread: left = ca->sb.bucket_size - offset; bio->bi_private = &cl;bch_bio_map (bio, data); -closure_bio_submit (bio, &cl, ca); +closure_bio_submit (bio, &cl);closure_sync (&cl);/* This function could be simpler now since we no longer write
@@ -642,7 +642,7 @@ static void journal_write_unlocked(struct closure *cl)spin_unlock (&c->journal.lock);while ((bio =bio_list_pop (&list))) -closure_bio_submit (bio, cl, c->cache[0 ]); +closure_bio_submit (bio, cl);continue_at (cl, journal_write_done,NULL ); }
drivers/md/bcache/request.c +8 -8
@@ -873,7 +873,7 @@ static void cached_dev_read_error(struct closure *cl)/* XXX: invalidate cache */ -closure_bio_submit (bio, cl, s->d); +closure_bio_submit (bio, cl); }continue_at (cl, cached_dev_cache_miss_done,NULL );
@@ -996,7 +996,7 @@ static int cached_dev_cache_miss(struct btree *b, struct search *s, s->cache_miss = miss; s->iop.bio = cache_bio;bio_get (cache_bio); -closure_bio_submit (cache_bio, &s->cl, s->d); +closure_bio_submit (cache_bio, &s->cl);return ret; out_put:
@@ -1004,7 +1004,7 @@ out_put: out_submit: miss->bi_end_io = request_endio; miss->bi_private = &s->cl; -closure_bio_submit (miss, &s->cl, s->d); +closure_bio_submit (miss, &s->cl);return ret; }
@@ -1069,7 +1069,7 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s)if (!(bio->bi_rw &REQ_DISCARD ) ||blk_queue_discard (bdev_get_queue (dc->bdev))) -closure_bio_submit (bio, cl, s->d); +closure_bio_submit (bio, cl); }else if (s->iop.writeback) {bch_writeback_add (dc); s->iop.bio = bio;
@@ -1084,12 +1084,12 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s) flush->bi_end_io = request_endio; flush->bi_private = cl; -closure_bio_submit (flush, cl, s->d); +closure_bio_submit (flush, cl); } }else { s->iop.bio =bio_clone_fast (bio,GFP_NOIO , dc->disk.bio_split); -closure_bio_submit (bio, cl, s->d); +closure_bio_submit (bio, cl); }closure_call (&s->iop.cl, bch_data_insert,NULL , cl);
@@ -1105,7 +1105,7 @@ static void cached_dev_nodata(struct closure *cl)bch_journal_meta (s->iop.c, cl);/* If it's a flush, we send the flush to the backing device too */ -closure_bio_submit (bio, cl, s->d); +closure_bio_submit (bio, cl);continue_at (cl, cached_dev_bio_complete,NULL ); }
@@ -1152,7 +1152,7 @@ static void cached_dev_make_request(struct request_queue *q, struct bio *bio) !blk_queue_discard(bdev_get_queue(dc->bdev)))bio_endio (bio,0 ); else - bch_generic_make_request(bio, &d->bio_split_hook); +generic_make_request (bio ); } }
drivers/md/bcache/super.c +2 -30
@@ -59,29 +59,6 @@ struct workqueue_struct *bcache_wq;#define BTREE_MAX_PAGES (256 * 1024 / PAGE_SIZE) -static void bio_split_pool_free (struct bio_split_pool *p) -{ -if (p->bio_split_hook) -mempool_destroy (p->bio_split_hook); - -if (p->bio_split) -bioset_free (p->bio_split); -} - -static int bio_split_pool_init (struct bio_split_pool *p) -{ - p->bio_split =bioset_create (4 ,0 ); -if (!p->bio_split) -return -ENOMEM ; - - p->bio_split_hook =mempool_create_kmalloc_pool (4 , -sizeof (struct bio_split_hook )); -if (!p->bio_split_hook) -return -ENOMEM ; - -return 0 ; -} -/* Superblock */ static const char *read_super(struct cache_sb *sb, struct block_device *bdev,
@@ -537,7 +514,7 @@ static void prio_io(struct cache *ca, uint64_t bucket, unsigned long rw) bio->bi_private = ca;bch_bio_map (bio, ca->disk_buckets); -closure_bio_submit (bio, &ca->prio, ca); +closure_bio_submit (bio, &ca->prio);closure_sync (cl); }
@@ -763,7 +740,6 @@ static void bcache_device_free(struct bcache_device *d)put_disk (d->disk); } -bio_split_pool_free (&d->bio_split_hook);if (d->bio_split)bioset_free (d->bio_split);if (is_vmalloc_addr (d->full_dirty_stripes))
@@ -816,7 +792,6 @@ static int bcache_device_init(struct bcache_device *d, unsigned block_size,return minor;if (!(d->bio_split =bioset_create (4 , offsetof(struct bbio , bio))) || -bio_split_pool_init (&d->bio_split_hook) || !(d->disk =alloc_disk (1 ))) {ida_simple_remove (&bcache_minor, minor);return -ENOMEM ;
@@ -1775,8 +1750,6 @@ void bch_cache_release(struct kobject *kobj)if (ca->set) ca->set->cache[ca->sb.nr_this_dev] =NULL ; -bio_split_pool_free (&ca->bio_split_hook); -free_pages ((unsigned long ) ca->disk_buckets,ilog2 (bucket_pages (ca)));kfree (ca->prio_buckets);vfree (ca->buckets);
@@ -1825,8 +1798,7 @@ static int cache_alloc(struct cache_sb *sb, struct cache *ca) ca->sb.nbuckets)) || !(ca->prio_buckets =kzalloc (sizeof (uint64_t ) *prio_buckets (ca) *2 ,GFP_KERNEL )) || - !(ca->disk_buckets =alloc_bucket_pages (GFP_KERNEL , ca)) || - bio_split_pool_init(&ca->bio_split_hook)) + !(ca->disk_buckets = alloc_bucket_pages(GFP_KERNEL, ca))) return -ENOMEM; ca->prio_last_buckets = ca->prio_buckets +prio_buckets (ca);
drivers/md/bcache/util.h +3 -2
@@ -4,6 +4,7 @@#include <linux/blkdev.h> #include <linux/errno.h> +#include <linux/blkdev.h> #include <linux/kernel.h> #include <linux/llist.h> #include <linux/ratelimit.h>
@@ -576,10 +577,10 @@ static inline sector_t bdev_sectors(struct block_device *bdev)return bdev->bd_inode->i_size >>9 ; } -#define closure_bio_submit (bio, cl, dev) \ +#define closure_bio_submit (bio, cl) \ do { \ closure_get(cl); \ - bch_generic_make_request(bio, &(dev)->bio_split_hook); \ + generic_make_request(bio); \ } while (0)uint64_t bch_crc64_update (uint64_t ,const void *,size_t );
drivers/md/bcache/writeback.c +2 -2
@@ -188,7 +188,7 @@ static void write_dirty(struct closure *cl) io->bio.bi_bdev = io->dc->bdev; io->bio.bi_end_io = dirty_endio; -closure_bio_submit (&io->bio, cl, &io->dc->disk); +closure_bio_submit (&io->bio, cl);continue_at (cl, write_dirty_finish, system_wq); }
@@ -208,7 +208,7 @@ static void read_dirty_submit(struct closure *cl) { struct dirty_io *io = container_of(cl, struct dirty_io, cl); -closure_bio_submit (&io->bio, cl, &io->dc->disk); +closure_bio_submit (&io->bio, cl);continue_at (cl, write_dirty, system_wq); }
Signature · 1 line
1.9.0
PATCH 5/9 btrfs: generic_make_request() handles arbitrary size bios now
In reply to Kent OverstreetSo there's no need for btrfs to break up bios for device limits anymore
--- fs/btrfs/volumes.c | 73 ------------------------------------------------------ 1 file changed, 73 deletions(-)
fs/btrfs/volumes.c +0 -73
@@ -5395,34 +5395,6 @@ static noinline void btrfs_schedule_bio(struct btrfs_root *root, &device->work); } -static int bio_size_ok (struct block_device *bdev,struct bio *bio, -sector_t sector) -{ -struct bio_vec *prev; -struct request_queue *q =bdev_get_queue (bdev); -unsigned int max_sectors =queue_max_sectors (q); -struct bvec_merge_data bvm = { - .bi_bdev = bdev, - .bi_sector = sector, - .bi_rw = bio->bi_rw, - }; - -if (WARN_ON (bio->bi_vcnt ==0 )) -return 1 ; - - prev = &bio->bi_io_vec[bio->bi_vcnt -1 ]; -if (bio_sectors (bio) > max_sectors) -return 0 ; - -if (!q->merge_bvec_fn) -return 1 ; - - bvm.bi_size = bio->bi_iter.bi_size - prev->bv_len; -if (q->merge_bvec_fn (q, &bvm, prev) < prev->bv_len) -return 0 ; -return 1 ; -} -static void submit_stripe_bio (struct btrfs_root *root,struct btrfs_bio *bbio,struct bio *bio,u64 physical,int dev_nr,int rw,int async)
@@ -5453,38 +5425,6 @@ static void submit_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio,btrfsic_submit_bio (rw, bio); } -static int breakup_stripe_bio (struct btrfs_root *root,struct btrfs_bio *bbio, -struct bio *first_bio,struct btrfs_device *dev, -int dev_nr,int rw,int async) -{ -struct bio_vec *bvec = first_bio->bi_io_vec; -struct bio *bio; -int nr_vecs =bio_get_nr_vecs (dev->bdev); -u64 physical = bbio->stripes[dev_nr].physical; - -again: - bio =btrfs_bio_alloc (dev->bdev, physical >>9 , nr_vecs,GFP_NOFS ); -if (!bio) -return -ENOMEM ; - -while (bvec <= (first_bio->bi_io_vec + first_bio->bi_vcnt -1 )) { -if (bio_add_page (bio, bvec->bv_page, bvec->bv_len, - bvec->bv_offset) < bvec->bv_len) { -u64 len = bio->bi_iter.bi_size; - -atomic_inc (&bbio->stripes_pending); -submit_stripe_bio (root, bbio, bio, physical, dev_nr, - rw, async); - physical += len; - goto again; - } - bvec++; - } - -submit_stripe_bio (root, bbio, bio, physical, dev_nr, rw, async); -return 0 ; -} -static void bbio_error (struct btrfs_bio *bbio,struct bio *bio,u64 logical) {atomic_inc (&bbio->error);
@@ -5553,19 +5493,6 @@ int btrfs_map_bio(struct btrfs_root *root, int rw, struct bio *bio,continue ; } -/* -* Check and see if we're ok with this bio based on it's size -* and offset with the given device. -*/ -if (!bio_size_ok (dev->bdev, first_bio, - bbio->stripes[dev_nr].physical >>9 )) { - ret =breakup_stripe_bio (root, bbio, first_bio, dev, - dev_nr, rw, async_submit); -BUG_ON (ret); - dev_nr++; -continue ; - } -if (dev_nr < total_devs -1 ) { bio =btrfs_bio_clone (first_bio,GFP_NOFS );BUG_ON (!bio);/* -ENOMEM */
Signature · 1 line
1.9.0
PATCH 6/9 btrfs: Convert to bio_for_each_segment()
In reply to Kent OverstreetThis is going to be important for future (hopeful) block layer refactoring, and using the standard primitives makes the code easier to audit.
--- fs/btrfs/extent_io.c | 12 ++++++++--- fs/btrfs/file-item.c | 59 ++++++++++++++++++++-------------------------------- fs/btrfs/inode.c | 22 +++++++------------- 3 files changed, 39 insertions(+), 54 deletions(-)
fs/btrfs/extent_io.c +9 -3
@@ -2632,12 +2632,18 @@ static int __must_check submit_one_bio(int rw, struct bio *bio, int mirror_num, unsigned long bio_flags) { int ret = 0; -struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt -1 ; -struct page *page = bvec->bv_page; +struct bio_vec bvec = {0 }; +struct bvec_iter iter; +struct page *page;struct extent_io_tree *tree = bio->bi_private;u64 start; - start =page_offset (page) + bvec->bv_offset; +bio_for_each_segment (bvec, bio, iter) +if (bio_iter_last (bvec, iter)) +break ; + + page = bvec.bv_page; + start =page_offset (page) + bvec.bv_offset; bio->bi_private =NULL ;
fs/btrfs/file-item.c +22 -37
145 lines not on this page · raw message
fs/btrfs/inode.c +8 -14
@@ -7161,12 +7161,11 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip,struct btrfs_root *root =BTRFS_I (inode)->root;struct bio *bio;struct bio *orig_bio = dip->orig_bio; -struct bio_vec *bvec = orig_bio->bi_io_vec; +struct bio_vec bvec; +struct bvec_iter iter;u64 start_sector = orig_bio->bi_iter.bi_sector;u64 file_offset = dip->logical_offset; -u64 submit_len =0 ;u64 map_length; -int nr_pages =0 ;int ret =0 ;int async_submit =0 ;
@@ -7197,10 +7196,12 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip, bio->bi_end_io = btrfs_end_dio_bio;atomic_inc (&dip->pending_bios); -while (bvec <= (orig_bio->bi_io_vec + orig_bio->bi_vcnt -1 )) { -if (unlikely (map_length < submit_len + bvec->bv_len || -bio_add_page (bio, bvec->bv_page, bvec->bv_len, - bvec->bv_offset) < bvec->bv_len)) { +bio_for_each_segment (bvec, orig_bio, iter) { +if (unlikely (map_length < bio->bi_iter.bi_size + bvec.bv_len || +bio_add_page (bio, bvec.bv_page, bvec.bv_len, + bvec.bv_offset) < bvec.bv_len)) { +unsigned submit_len = bio->bi_iter.bi_size; +/* * inc the count before we submit the bio so * we know the end IO handler won't happen before
@@ -7220,9 +7221,6 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip, start_sector += submit_len >>9 ; file_offset += submit_len; - submit_len =0 ; - nr_pages =0 ; - bio =btrfs_dio_bio_alloc (orig_bio->bi_bdev, start_sector,GFP_NOFS );if (!bio)
@@ -7238,10 +7236,6 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip,bio_put (bio); goto out_err; } - }else { - submit_len += bvec->bv_len; - nr_pages++; - bvec++; } }
Signature · 1 line
1.9.0
PATCH 7/9 iov_iter: Move iov_iter to uio.h
In reply to Kent OverstreetGoing to be consolidating all the iov iter in one place, and fs.h is way too big. This also adds a new helper, iovec iov_iter_iovec().
--- include/linux/fs.h | 32 -------------------------------- include/linux/uio.h | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 32 deletions(-)
include/linux/fs.h +0 -32
@@ -290,38 +290,6 @@ struct page;struct address_space ;struct writeback_control ; -struct iov_iter { -const struct iovec *iov; -unsigned long nr_segs; -size_t iov_offset; -size_t count; -}; - -size_t iov_iter_copy_from_user_atomic (struct page *page, -struct iov_iter *i,unsigned long offset,size_t bytes); -size_t iov_iter_copy_from_user (struct page *page, -struct iov_iter *i,unsigned long offset,size_t bytes); -void iov_iter_advance (struct iov_iter *i,size_t bytes); -int iov_iter_fault_in_readable (struct iov_iter *i,size_t bytes); -size_t iov_iter_single_seg_count (const struct iov_iter *i); - -static inline void iov_iter_init (struct iov_iter *i, -const struct iovec *iov,unsigned long nr_segs, -size_t count,size_t written) -{ - i->iov = iov; - i->nr_segs = nr_segs; - i->iov_offset =0 ; - i->count = count + written; - -iov_iter_advance (i, written); -} - -static inline size_t iov_iter_count (struct iov_iter *i) -{ -return i->count; -} -/* * "descriptor" for what we're up to with a read. * This allows us to use the same read code yet
include/linux/uio.h +50 -0
72 lines not on this page · raw message
Signature · 1 line
1.9.0
PATCH 8/9 iov_iter: Kill iov_iter_single_seg_count()
In reply to Kent OverstreetThe new iov_iter_iovec() is a more general replacement.
--- fs/fuse/file.c | 6 +++--- include/linux/uio.h | 1 - mm/filemap.c | 15 +-------------- 3 files changed, 4 insertions(+), 18 deletions(-)
fs/fuse/file.c +3 -3
@@ -1013,7 +1013,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_req *req,if (!tmp) {unlock_page (page);page_cache_release (page); - bytes =min (bytes,iov_iter_single_seg_count (ii)); + bytes =min (bytes,iov_iter_iovec (ii).iov_len); goto again; }
@@ -1204,7 +1204,7 @@ static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)static inline size_t fuse_get_frag_size (const struct iov_iter *ii,size_t max_size) { -return min (iov_iter_single_seg_count (ii), max_size); +return min (iov_iter_iovec (ii).iov_len, max_size); } static int fuse_get_user_pages(struct fuse_req *req, struct iov_iter *ii,
@@ -1278,7 +1278,7 @@ static inline int fuse_iter_npages(const struct iov_iter *ii_p)while (iov_iter_count (&ii) && npages <FUSE_MAX_PAGES_PER_REQ ) {unsigned long user_addr =fuse_get_user_addr (&ii);unsigned offset = user_addr & ~PAGE_MASK ; -size_t frag_size =iov_iter_single_seg_count (&ii); +size_t frag_size =iov_iter_iovec (&ii).iov_len; npages += (frag_size + offset +PAGE_SIZE -1 ) >>PAGE_SHIFT ;iov_iter_advance (&ii, frag_size);
include/linux/uio.h +0 -1
@@ -66,7 +66,6 @@ size_t iov_iter_copy_from_user(struct page *page, struct iov_iter *i, unsigned long offset, size_t bytes);void iov_iter_advance (struct iov_iter *i,size_t bytes);int iov_iter_fault_in_readable (struct iov_iter *i,size_t bytes); -size_t iov_iter_single_seg_count (const struct iov_iter *i); static inline void iov_iter_init(struct iov_iter *i, const struct iovec *iov, unsigned long nr_segs,
mm/filemap.c +1 -14
@@ -2073,19 +2073,6 @@ int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)EXPORT_SYMBOL (iov_iter_fault_in_readable);/* -* Return the count of just the current iov_iter segment. -*/ -size_t iov_iter_single_seg_count (const struct iov_iter *i) -{ -const struct iovec *iov = i->iov; -if (i->nr_segs ==1 ) -return i->count; -else -return min (i->count, iov->iov_len - i->iov_offset); -} -EXPORT_SYMBOL (iov_iter_single_seg_count); - -/* * Performs necessary checks before doing a write * * Can adjust writing position or amount of bytes to write.
@@ -2373,7 +2360,7 @@ again:* once without a pagefault. */ bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset, -iov_iter_single_seg_count (i)); +iov_iter_iovec (i).iov_len); goto again; } pos += copied;
Signature · 1 line
1.9.0
PATCH 9/9 iov_iter: Kill written arg to iov_iter_init()
In reply to Kent OverstreetThis gets rid of a usually needless call to iov_iter_advance().
--- fs/btrfs/file.c | 8 +++++--- fs/ceph/file.c | 7 ++++--- fs/cifs/file.c | 4 ++-- fs/fuse/file.c | 11 ++++++----- include/linux/uio.h | 9 +++------ mm/filemap.c | 4 +++- 6 files changed, 23 insertions(+), 20 deletions(-)
fs/btrfs/file.c +5 -3
@@ -1652,8 +1652,9 @@ static ssize_t __btrfs_direct_write(struct kiocb *iocb,return written; pos += written; - count -= written; -iov_iter_init (&i, iov, nr_segs, count, written); +iov_iter_init (&i, iov, nr_segs, count); +iov_iter_advance (&i, written); + written_buffered =__btrfs_buffered_write (file, &i, pos);if (written_buffered <0 ) { err = written_buffered;
@@ -1768,7 +1769,8 @@ static ssize_t btrfs_file_aio_write(struct kiocb *iocb, }else {struct iov_iter i; -iov_iter_init (&i, iov, nr_segs, count, num_written); +iov_iter_init (&i, iov, nr_segs, count + num_written); +iov_iter_advance (&i, num_written); num_written =__btrfs_buffered_write (file, &i, pos);if (num_written >0 )
fs/ceph/file.c +4 -3
@@ -580,7 +580,7 @@ ceph_sync_direct_write(struct kiocb *iocb, const struct iovec *iov,CEPH_OSD_FLAG_ONDISK |CEPH_OSD_FLAG_WRITE ; -iov_iter_init (&i, iov, nr_segs, count,0 ); +iov_iter_init (&i, iov, nr_segs, count);while (iov_iter_count (&i) >0 ) {void __user *data = i.iov->iov_base + i.iov_offset;
@@ -701,7 +701,7 @@ static ssize_t ceph_sync_write(struct kiocb *iocb, const struct iovec *iov,CEPH_OSD_FLAG_WRITE |CEPH_OSD_FLAG_ACK ; -iov_iter_init (&i, iov, nr_segs, count,0 ); +iov_iter_init (&i, iov, nr_segs, count);while ((len =iov_iter_count (&i)) >0 ) {size_t left;
@@ -834,7 +834,8 @@ again: goto out; } -iov_iter_init (&i, iov, nr_segs, len, read); +iov_iter_init (&i, iov, nr_segs, len); +iov_iter_advance (&i, read);/* hmm, this isn't really async... */ ret =ceph_sync_read (iocb, &i, &checkeof);
fs/cifs/file.c +2 -2
@@ -2424,7 +2424,7 @@ cifs_iovec_write(struct file *file, const struct iovec *iov,else pid = current->tgid; -iov_iter_init (&it, iov, nr_segs, len,0 ); +iov_iter_init (&it, iov, nr_segs, len); do { size_t save_len;
@@ -2729,7 +2729,7 @@ cifs_readdata_to_iov(struct cifs_readdata *rdata, const struct iovec *iov,unsigned int i;/* set up iov_iter and advance to the correct offset */ -iov_iter_init (&ii, iov, nr_segs,iov_length (iov, nr_segs),0 ); +iov_iter_init (&ii, iov, nr_segs,iov_length (iov, nr_segs));iov_iter_advance (&ii, pos); *copied =0 ;
fs/fuse/file.c +6 -5
@@ -1152,9 +1152,10 @@ static ssize_t fuse_file_aio_write(struct kiocb *iocb, const struct iovec *iov, goto out; pos += written; - count -= written; -iov_iter_init (&i, iov, nr_segs, count, written); +iov_iter_init (&i, iov, nr_segs, count); +iov_iter_advance (&i, written); + written_buffered =fuse_perform_write (file, mapping, &i, pos);if (written_buffered <0 ) { err = written_buffered;
@@ -1174,7 +1175,7 @@ static ssize_t fuse_file_aio_write(struct kiocb *iocb, const struct iovec *iov, written += written_buffered; iocb->ki_pos = pos + written_buffered; }else { -iov_iter_init (&i, iov, nr_segs, count,0 ); +iov_iter_init (&i, iov, nr_segs, count); written =fuse_perform_write (file, mapping, &i, pos);if (written >=0 ) iocb->ki_pos = pos + written;
@@ -1300,7 +1301,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, const struct iovec *iov,struct fuse_req *req;struct iov_iter ii; -iov_iter_init (&ii, iov, nr_segs, count,0 ); +iov_iter_init (&ii, iov, nr_segs, count);if (io->async) req =fuse_get_req_for_background (fc,fuse_iter_npages (&ii));
@@ -2188,7 +2189,7 @@ static int fuse_ioctl_copy_user(struct page **pages, struct iovec *iov,if (!bytes)return 0 ; -iov_iter_init (&ii, iov, nr_segs, bytes,0 ); +iov_iter_init (&ii, iov, nr_segs, bytes);while (iov_iter_count (&ii)) {struct page *page = pages[page_idx++];
include/linux/uio.h +3 -6
@@ -67,16 +67,13 @@ size_t iov_iter_copy_from_user(struct page *page,void iov_iter_advance (struct iov_iter *i,size_t bytes);int iov_iter_fault_in_readable (struct iov_iter *i,size_t bytes); -static inline void iov_iter_init (struct iov_iter *i, -const struct iovec *iov,unsigned long nr_segs, -size_t count,size_t written) +static inline void iov_iter_init (struct iov_iter *i,const struct iovec *iov, +unsigned long nr_segs,size_t count) { i->iov = iov; i->nr_segs = nr_segs; i->iov_offset =0 ; - i->count = count + written; - -iov_iter_advance (i, written); + i->count = count; }static inline size_t iov_iter_count (struct iov_iter *i)
mm/filemap.c +3 -1
@@ -2385,7 +2385,9 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,ssize_t status;struct iov_iter i; -iov_iter_init (&i, iov, nr_segs, count, written); +iov_iter_init (&i, iov, nr_segs, count + written); +iov_iter_advance (&i, written); + status =generic_perform_write (file, &i, pos);if (likely (status >=0 )) {
Signature · 1 line
1.9.0