block/dm: use BIOSET_PERCPU_CACHE from bio_alloc_bioset
Subsystems (2)
Mike Snitzer
maintainerv3 cover block/dm: use BIOSET_PERCPU_CACHE from bio_alloc_bioset
Hi Jens,
This v3 is a rebase of the previous v2 series ontop of the revised v2 patch that Christoph provided.
Linus hasn't pulled the for-5.18/dm-changes branch yet, so the 3rd DM patch cannot be applied yet. But feel free to pickup the first 2 block patches for 5.19 and I'll rebase dm-5.19 on block accordingly.
Thanks,
Mike
v3: tweaked some code comments, refined patch headers and folded DM
patches so only one DM patch now.
v2: add REQ_ALLOC_CACHE and move use of bio_alloc_percpu_cache to
bio_alloc_bioset
Mike Snitzer (3):
block: allow using the per-cpu bio cache from bio_alloc_bioset
block: allow use of per-cpu bio alloc cache by block drivers
dm: conditionally enable BIOSET_PERCPU_CACHE for dm_io bioset
block/bio.c | 88 +++++++++++++++++++++++------------------------ block/blk.h | 7 ---- block/fops.c | 11 ++++-- drivers/md/dm-table.c | 11 ++++-- drivers/md/dm.c | 8 ++--- drivers/md/dm.h | 4 +-- include/linux/bio.h | 8 +++-- include/linux/blk_types.h | 3 +- 8 files changed, 73 insertions(+), 67 deletions(-)
Signature · 1 line
2.15.0
Mike Snitzer
maintainerPATCH v3 1/3 block: allow using the per-cpu bio cache from bio_alloc_bioset
Replace the BIO_PERCPU_CACHE bio-internal flag with a REQ_ALLOC_CACHE one that can be passed to bio_alloc / bio_alloc_bioset, and implement the percpu cache allocation logic in a helper called from bio_alloc_bioset. This allows any bio_alloc_bioset user to use the percpu caches instead of having the functionality tied to struct kiocb.
--- block/bio.c | 86 ++++++++++++++++++++++------------------------- block/blk.h | 3 +- block/fops.c | 11 ++++-- include/linux/bio.h | 2 -- include/linux/blk_types.h | 3 +- 5 files changed, 52 insertions(+), 53 deletions(-)
block/bio.c +41 -45
@@ -420,6 +420,28 @@ static void punt_bios_to_rescuer(struct bio_set *bs)queue_work (bs->rescue_workqueue, &bs->rescue_work); } +static struct bio *bio_alloc_percpu_cache (struct block_device *bdev, +unsigned short nr_vecs,unsigned int opf,gfp_t gfp, +struct bio_set *bs) +{ +struct bio_alloc_cache *cache; +struct bio *bio; + + cache =per_cpu_ptr (bs->cache,get_cpu ()); +if (!cache->free_list) { +put_cpu (); +return NULL ; + } + bio = cache->free_list; + cache->free_list = bio->bi_next; + cache->nr--; +put_cpu (); + +bio_init (bio, bdev, nr_vecs ? bio->bi_inline_vecs :NULL , nr_vecs, opf); + bio->bi_pool = bs; +return bio; +} +/** * bio_alloc_bioset - allocate a bio for I/O * @bdev: block device to allocate the bio for (can be %NULL)
@@ -452,6 +474,9 @@ static void punt_bios_to_rescuer(struct bio_set *bs)* submit_bio_noacct() should be avoided - instead, use bio_set's front_pad * for per bio allocations. * +* If REQ_ALLOC_CACHE is set, the final put of the bio MUST be done from process +* context, not hard/soft IRQ. +* * Returns: Pointer to new bio on success, NULL on failure. */ struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,
@@ -466,6 +491,21 @@ struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs,if (WARN_ON_ONCE (!mempool_initialized (&bs->bvec_pool) && nr_vecs >0 ))return NULL ; +if (opf &REQ_ALLOC_CACHE ) { +if (bs->cache && nr_vecs <=BIO_INLINE_VECS ) { + bio =bio_alloc_percpu_cache (bdev, nr_vecs, opf, + gfp_mask, bs); +if (bio) +return bio; +/* +* No cached bio available, bio returned below marked with +* REQ_ALLOC_CACHE to particpate in per-cpu alloc cache. +*/ + }else { + opf &= ~REQ_ALLOC_CACHE ; + } + } +/* * submit_bio_noacct() converts recursion to iteration; this means if * we're running beneath it, any bios we allocate and submit will not be
@@ -712,7 +752,7 @@ void bio_put(struct bio *bio)return ; } -if (bio_flagged (bio,BIO_PERCPU_CACHE )) { +if (bio->bi_opf &REQ_ALLOC_CACHE ) {struct bio_alloc_cache *cache;bio_uninit (bio);
@@ -1734,50 +1774,6 @@ int bioset_init_from_src(struct bio_set *bs, struct bio_set *src) }EXPORT_SYMBOL (bioset_init_from_src); -/** -* bio_alloc_kiocb - Allocate a bio from bio_set based on kiocb -* @kiocb: kiocb describing the IO -* @bdev: block device to allocate the bio for (can be %NULL) -* @nr_vecs: number of iovecs to pre-allocate -* @opf: operation and flags for bio -* @bs: bio_set to allocate from -* -* Description: -* Like @bio_alloc_bioset, but pass in the kiocb. The kiocb is only -* used to check if we should dip into the per-cpu bio_set allocation -* cache. The allocation uses GFP_KERNEL internally. On return, the -* bio is marked BIO_PERCPU_CACHEABLE, and the final put of the bio -* MUST be done from process context, not hard/soft IRQ. -* -*/ -struct bio *bio_alloc_kiocb (struct kiocb *kiocb,struct block_device *bdev, -unsigned short nr_vecs,unsigned int opf,struct bio_set *bs) -{ -struct bio_alloc_cache *cache; -struct bio *bio; - -if (!(kiocb->ki_flags &IOCB_ALLOC_CACHE ) || nr_vecs >BIO_INLINE_VECS ) -return bio_alloc_bioset (bdev, nr_vecs, opf,GFP_KERNEL , bs); - - cache =per_cpu_ptr (bs->cache,get_cpu ()); -if (cache->free_list) { - bio = cache->free_list; - cache->free_list = bio->bi_next; - cache->nr--; -put_cpu (); -bio_init (bio, bdev, nr_vecs ? bio->bi_inline_vecs :NULL , - nr_vecs, opf); - bio->bi_pool = bs; -bio_set_flag (bio,BIO_PERCPU_CACHE ); -return bio; - } -put_cpu (); - bio =bio_alloc_bioset (bdev, nr_vecs, opf,GFP_KERNEL , bs); -bio_set_flag (bio,BIO_PERCPU_CACHE ); -return bio; -} -EXPORT_SYMBOL_GPL (bio_alloc_kiocb); -static int __init init_bio (void ) {int i;
block/blk.h +1 -2
@@ -454,8 +454,7 @@ extern struct device_attribute dev_attr_events_poll_msecs;static inline void bio_clear_polled (struct bio *bio) {/* can't support alloc cache if we turn off polling */ -bio_clear_flag (bio,BIO_PERCPU_CACHE ); - bio->bi_opf &= ~REQ_POLLED ; + bio->bi_opf &= ~(REQ_POLLED |REQ_ALLOC_CACHE ); }long blkdev_ioctl (struct file *file,unsigned cmd,unsigned long arg);
block/fops.c +8 -3
@@ -198,8 +198,10 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, (bdev_logical_block_size(bdev) - 1))return -EINVAL ; - bio =bio_alloc_kiocb (iocb, bdev, nr_pages, opf, &blkdev_dio_pool); - +if (iocb->ki_flags &IOCB_ALLOC_CACHE ) + opf |=REQ_ALLOC_CACHE ; + bio =bio_alloc_bioset (bdev, nr_pages, opf,GFP_KERNEL , + &blkdev_dio_pool); dio = container_of(bio, struct blkdev_dio, bio);atomic_set (&dio->ref,1 );/*
@@ -320,7 +322,10 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, (bdev_logical_block_size(bdev) - 1))return -EINVAL ; - bio =bio_alloc_kiocb (iocb, bdev, nr_pages, opf, &blkdev_dio_pool); +if (iocb->ki_flags &IOCB_ALLOC_CACHE ) + opf |=REQ_ALLOC_CACHE ; + bio =bio_alloc_bioset (bdev, nr_pages, opf,GFP_KERNEL , + &blkdev_dio_pool); dio = container_of(bio, struct blkdev_dio, bio); dio->flags =0 ; dio->iocb = iocb;
include/linux/bio.h +0 -2
@@ -408,8 +408,6 @@ extern int bioset_init_from_src(struct bio_set *bs, struct bio_set *src);struct bio *bio_alloc_bioset (struct block_device *bdev,unsigned short nr_vecs,unsigned int opf,gfp_t gfp_mask,struct bio_set *bs); -struct bio *bio_alloc_kiocb (struct kiocb *kiocb,struct block_device *bdev, -unsigned short nr_vecs,unsigned int opf,struct bio_set *bs);struct bio *bio_kmalloc (gfp_t gfp_mask,unsigned short nr_iovecs);extern void bio_put (struct bio *);
include/linux/blk_types.h +2 -1
@@ -328,7 +328,6 @@ enum {BIO_QOS_MERGED ,/* but went through rq_qos merge path */ BIO_REMAPPED ,BIO_ZONE_WRITE_LOCKED ,/* Owns a zoned device zone write lock */ -BIO_PERCPU_CACHE ,/* can participate in per-cpu alloc cache */ BIO_FLAG_LAST };
@@ -415,6 +414,7 @@ enum req_flag_bits { __REQ_NOUNMAP,/* do not free blocks when zeroing */ __REQ_POLLED,/* caller polls for completion using bio_poll */ + __REQ_ALLOC_CACHE,/* allocate IO from cache if available */ /* for driver use */ __REQ_DRV,
@@ -440,6 +440,7 @@ enum req_flag_bits {#define REQ_NOUNMAP (1ULL << __REQ_NOUNMAP)#define REQ_POLLED (1ULL << __REQ_POLLED) +#define REQ_ALLOC_CACHE (1ULL << __REQ_ALLOC_CACHE)#define REQ_DRV (1ULL << __REQ_DRV)#define REQ_SWAP (1ULL << __REQ_SWAP)
Signature · 1 line
2.15.0
Mike Snitzer
maintainerPATCH v3 2/3 block: allow use of per-cpu bio alloc cache by block drivers
In reply to Mike SnitzerRefine per-cpu bio alloc cache interfaces so that DM and other block drivers can properly create and use the cache:
DM uses bioset_init_from_src() to do its final bioset initialization, so must update bioset_init_from_src() to set BIOSET_PERCPU_CACHE if %src bioset has a cache.
Also move bio_clear_polled() to include/linux/bio.h to allow users outside of block core.
--- block/bio.c | 2 ++ block/blk.h | 6 ------ include/linux/bio.h | 6 ++++++ 3 files changed, 8 insertions(+), 6 deletions(-)
block/bio.c +2 -0
@@ -1769,6 +1769,8 @@ int bioset_init_from_src(struct bio_set *bs, struct bio_set *src) flags |=BIOSET_NEED_BVECS ;if (src->rescue_workqueue) flags |=BIOSET_NEED_RESCUER ; +if (src->cache) + flags |=BIOSET_PERCPU_CACHE ;return bioset_init (bs, src->bio_pool.min_nr, src->front_pad, flags); }
block/blk.h +0 -6
@@ -451,12 +451,6 @@ extern struct device_attribute dev_attr_events;extern struct device_attribute dev_attr_events_async;extern struct device_attribute dev_attr_events_poll_msecs; -static inline void bio_clear_polled (struct bio *bio) -{ -/* can't support alloc cache if we turn off polling */ - bio->bi_opf &= ~(REQ_POLLED |REQ_ALLOC_CACHE ); -} -long blkdev_ioctl (struct file *file,unsigned cmd,unsigned long arg);long compat_blkdev_ioctl (struct file *file,unsigned cmd,unsigned long arg);
include/linux/bio.h +6 -0
@@ -783,6 +783,12 @@ static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb) bio->bi_opf |=REQ_NOWAIT ; } +static inline void bio_clear_polled (struct bio *bio) +{ +/* can't support alloc cache if we turn off polling */ + bio->bi_opf &= ~(REQ_POLLED |REQ_ALLOC_CACHE ); +} +struct bio *blk_next_bio (struct bio *bio,struct block_device *bdev,unsigned int nr_pages,unsigned int opf,gfp_t gfp);
Signature · 1 line
2.15.0
Re: [PATCH v3 2/3] block: allow use of per-cpu bio alloc cache by block drivers
Quoted 11 lines · Mike Snitzer
Refine per-cpu bio alloc cache interfaces so that DM and other block drivers can properly create and use the cache:
DM uses
bioset_init_from_src()to do its final bioset initialization, so must updatebioset_init_from_src()to setBIOSET_PERCPU_CACHEif %src bioset has a cache.Also move
bio_clear_polled()toinclude/linux/bio.hto allow users outside of block core.Signed-off-by Mike Snitzer <[email protected]>
Looks good:
Mike Snitzer
maintainerPATCH v3 3/3 dm: conditionally enable BIOSET_PERCPU_CACHE for dm_io bioset
In reply to Mike SnitzerA bioset's per-cpu alloc cache may have broader utility in the future but for now constrain it to being tightly coupled to QUEUE_FLAG_POLL.
Also change dm_io_complete() to use bio_clear_polled() so that it properly clears all associated bio state on requeue.
This commit improves DM's hipri bio polling (REQ_POLLED) perf by 7 - 20% depending on the system.
--- drivers/md/dm-table.c | 11 ++++++++--- drivers/md/dm.c | 8 ++++---- drivers/md/dm.h | 4 ++-- 3 files changed, 14 insertions(+), 9 deletions(-)
drivers/md/dm-table.c +8 -3
@@ -1002,6 +1002,8 @@ bool dm_table_request_based(struct dm_table *t)return __table_type_request_based (dm_table_get_type (t)); } +static int dm_table_supports_poll (struct dm_table *t); +static int dm_table_alloc_md_mempools (struct dm_table *t,struct mapped_device *md) {enum dm_queue_mode type =dm_table_get_type (t);
@@ -1009,21 +1011,24 @@ static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *unsigned min_pool_size =0 ;struct dm_target *ti;unsigned i; +bool poll_supported = false;if (unlikely (type ==DM_TYPE_NONE )) {DMWARN ("no table type is set, can't allocate mempools" );return -EINVAL ; } -if (__table_type_bio_based (type)) +if (__table_type_bio_based (type)) {for (i =0 ; i < t->num_targets; i++) { ti = t->targets + i; per_io_data_size =max (per_io_data_size, ti->per_io_data_size); min_pool_size =max (min_pool_size, ti->num_flush_bios); } + poll_supported = !!dm_table_supports_poll (t); + } - t->mempools =dm_alloc_md_mempools (md, type, t->integrity_supported, - per_io_data_size, min_pool_size); + t->mempools =dm_alloc_md_mempools (md, type, per_io_data_size, min_pool_size, + t->integrity_supported, poll_supported);if (!t->mempools)return -ENOMEM ;
drivers/md/dm.c +4 -4
@@ -898,7 +898,7 @@ static void dm_io_complete(struct dm_io *io)* may only reflect a subset of the pre-split original, * so clear REQ_POLLED in case of requeue */ - bio->bi_opf &= ~REQ_POLLED ; +bio_clear_polled (bio);return ; }
@@ -2915,8 +2915,8 @@ int dm_noflush_suspending(struct dm_target *ti)EXPORT_SYMBOL_GPL (dm_noflush_suspending);struct dm_md_mempools *dm_alloc_md_mempools (struct mapped_device *md,enum dm_queue_mode type, -unsigned integrity,unsigned per_io_data_size, -unsigned min_pool_size) +unsigned per_io_data_size,unsigned min_pool_size, +bool integrity,bool poll) {struct dm_md_mempools *pools =kzalloc_node (sizeof (*pools),GFP_KERNEL , md->numa_node_id);unsigned int pool_size =0 ;
@@ -2932,7 +2932,7 @@ struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_qu pool_size =max (dm_get_reserved_bio_based_ios (), min_pool_size); front_pad =roundup (per_io_data_size, __alignof__(struct dm_target_io )) +DM_TARGET_IO_BIO_OFFSET ; io_front_pad =roundup (per_io_data_size, __alignof__(struct dm_io )) +DM_IO_BIO_OFFSET ; - ret =bioset_init (&pools->io_bs, pool_size, io_front_pad,0 ); + ret =bioset_init (&pools->io_bs, pool_size, io_front_pad, poll ?BIOSET_PERCPU_CACHE :0 );if (ret) goto out;if (integrity &&bioset_integrity_create (&pools->io_bs, pool_size))
drivers/md/dm.h +2 -2
@@ -221,8 +221,8 @@ void dm_kcopyd_exit(void);* Mempool operations */ struct dm_md_mempools *dm_alloc_md_mempools (struct mapped_device *md,enum dm_queue_mode type, -unsigned integrity,unsigned per_bio_data_size, -unsigned min_pool_size); +unsigned per_io_data_size,unsigned min_pool_size, +bool integrity,bool poll);void dm_free_md_mempools (struct dm_md_mempools *pools);/*
Signature · 1 line
2.15.0
Re: (subset) [PATCH v3 0/3] block/dm: use BIOSET_PERCPU_CACHE from bio_alloc_bioset
In reply to Mike SnitzerQuoted 8 lines · Mike Snitzer
This v3 is a rebase of the previous v2 series ontop of the revised v2 patch that Christoph provided.
Linus hasn't pulled the for-5.18/dm-changes branch yet, so the 3rd DM patch cannot be applied yet. But feel free to pickup the first 2 block patches for 5.19 and I'll rebase dm-5.19 on block accordingly.
[...]
Applied, thanks!
[1/3] block: allow using the per-cpu bio cache from bio_alloc_bioset
commit: a147e4805855e34f8e1027b88baf59a7f7c8b8d3
[2/3] block: allow use of per-cpu bio alloc cache by block drivers
commit: e866e4dbad251b4dd1e134c295afd862333864bcBest regards,
Signature · 1 line
Jens Axboe