block: move bio_map_* to blk-map.c
PATCH 1/1 block: move bio_map_* to blk-map.c
The bio_map_* helpers are just the low-level helpers for the blk_rq_map_* APIs. Move them together for better logical grouping, as no there isn't much overlap with other code in bio.c.
Signed-off-by Christoph Hellwig <[email protected]>
--- block/bio.c | 510 +------------------------------------------- block/blk-map.c | 508 +++++++++++++++++++++++++++++++++++++++++++ block/blk.h | 4 + include/linux/bio.h | 14 -- 4 files changed, 513 insertions(+), 523 deletions(-)
block/bio.c +1 -509
@@ -780,7 +780,7 @@ static bool bio_try_merge_pc_page(struct request_queue *q, struct bio *bio,* * This should only be used by passthrough bios. */ -static int __bio_add_pc_page (struct request_queue *q,struct bio *bio, +int __bio_add_pc_page (struct request_queue *q,struct bio *bio,struct page *page,unsigned int len,unsigned int offset,bool *same_page) {
@@ -1194,90 +1194,6 @@ void bio_list_copy_data(struct bio *dst, struct bio *src) }EXPORT_SYMBOL (bio_list_copy_data); -struct bio_map_data { -int is_our_pages; -struct iov_iter iter; -struct iovec iov[]; -}; - -static struct bio_map_data *bio_alloc_map_data (struct iov_iter *data, -gfp_t gfp_mask) -{ -struct bio_map_data *bmd; -if (data->nr_segs >UIO_MAXIOV ) -return NULL ; - - bmd =kmalloc (struct_size (bmd, iov, data->nr_segs), gfp_mask); -if (!bmd) -return NULL ; -memcpy (bmd->iov, data->iov,sizeof (struct iovec ) * data->nr_segs); - bmd->iter = *data; - bmd->iter.iov = bmd->iov; -return bmd; -} - -/** -* bio_copy_from_iter - copy all pages from iov_iter to bio -* @bio: The &struct bio which describes the I/O as destination -* @iter: iov_iter as source -* -* Copy all pages from iov_iter to bio. -* Returns 0 on success, or error on failure. -*/ -static int bio_copy_from_iter (struct bio *bio,struct iov_iter *iter) -{ -struct bio_vec *bvec; -struct bvec_iter_all iter_all; - -bio_for_each_segment_all (bvec, bio, iter_all) { -ssize_t ret; - - ret =copy_page_from_iter (bvec->bv_page, - bvec->bv_offset, - bvec->bv_len, - iter); - -if (!iov_iter_count (iter)) -break ; - -if (ret < bvec->bv_len) -return -EFAULT ; - } - -return 0 ; -} - -/** -* bio_copy_to_iter - copy all pages from bio to iov_iter -* @bio: The &struct bio which describes the I/O as source -* @iter: iov_iter as destination -* -* Copy all pages from bio to iov_iter. -* Returns 0 on success, or error on failure. -*/ -static int bio_copy_to_iter (struct bio *bio,struct iov_iter iter) -{ -struct bio_vec *bvec; -struct bvec_iter_all iter_all; - -bio_for_each_segment_all (bvec, bio, iter_all) { -ssize_t ret; - - ret =copy_page_to_iter (bvec->bv_page, - bvec->bv_offset, - bvec->bv_len, - &iter); - -if (!iov_iter_count (&iter)) -break ; - -if (ret < bvec->bv_len) -return -EFAULT ; - } - -return 0 ; -} -void bio_free_pages (struct bio *bio) {struct bio_vec *bvec;
@@ -1288,430 +1204,6 @@ void bio_free_pages(struct bio *bio) }EXPORT_SYMBOL (bio_free_pages); -/** -* bio_uncopy_user - finish previously mapped bio -* @bio: bio being terminated -* -* Free pages allocated from bio_copy_user_iov() and write back data -* to user space in case of a read. -*/ -int bio_uncopy_user (struct bio *bio) -{ -struct bio_map_data *bmd = bio->bi_private; -int ret =0 ; - -if (!bio_flagged (bio,BIO_NULL_MAPPED )) { -/* -* if we're in a workqueue, the request is orphaned, so -* don't copy into a random user address space, just free -* and return -EINTR so user space doesn't expect any data. -*/ -if (!current->mm) - ret = -EINTR ; -else if (bio_data_dir (bio) ==READ ) - ret =bio_copy_to_iter (bio, bmd->iter); -if (bmd->is_our_pages) -bio_free_pages (bio); - } -kfree (bmd); -bio_put (bio); -return ret; -} - -/** -* bio_copy_user_iov - copy user data to bio -* @q: destination block queue -* @map_data: pointer to the rq_map_data holding pages (if necessary) -* @iter: iovec iterator -* @gfp_mask: memory allocation flags -* -* Prepares and returns a bio for indirect user io, bouncing data -* to/from kernel pages as necessary. Must be paired with -* call bio_uncopy_user() on io completion. -*/ -struct bio *bio_copy_user_iov (struct request_queue *q, -struct rq_map_data *map_data, -struct iov_iter *iter, -gfp_t gfp_mask) -{ -struct bio_map_data *bmd; -struct page *page; -struct bio *bio; -int i =0 , ret; -int nr_pages; -unsigned int len = iter->count; -unsigned int offset = map_data ?offset_in_page (map_data->offset) :0 ; - - bmd =bio_alloc_map_data (iter, gfp_mask); -if (!bmd) -return ERR_PTR (-ENOMEM ); - -/* -* We need to do a deep copy of the iov_iter including the iovecs. -* The caller provided iov might point to an on-stack or otherwise -* shortlived one. -*/ - bmd->is_our_pages = map_data ?0 :1 ; - - nr_pages =DIV_ROUND_UP (offset + len,PAGE_SIZE ); -if (nr_pages >BIO_MAX_PAGES ) - nr_pages =BIO_MAX_PAGES ; - - ret = -ENOMEM ; - bio =bio_kmalloc (gfp_mask, nr_pages); -if (!bio) - goto out_bmd; - - ret =0 ; - -if (map_data) { - nr_pages =1 << map_data->page_order; - i = map_data->offset /PAGE_SIZE ; - } -while (len) { -unsigned int bytes =PAGE_SIZE ; - - bytes -= offset; - -if (bytes > len) - bytes = len; - -if (map_data) { -if (i == map_data->nr_entries * nr_pages) { - ret = -ENOMEM ; -break ; - } - - page = map_data->pages[i / nr_pages]; - page += (i % nr_pages); - - i++; - }else { - page =alloc_page (q->bounce_gfp | gfp_mask); -if (!page) { - ret = -ENOMEM ; -break ; - } - } - -if (bio_add_pc_page (q, bio, page, bytes, offset) < bytes) { -if (!map_data) -__free_page (page); -break ; - } - - len -= bytes; - offset =0 ; - } - -if (ret) - goto cleanup; - -if (map_data) - map_data->offset += bio->bi_iter.bi_size; - -/* -* success -*/ -if ((iov_iter_rw (iter) ==WRITE && (!map_data || !map_data->null_mapped)) || - (map_data && map_data->from_user)) { - ret =bio_copy_from_iter (bio, iter); -if (ret) - goto cleanup; - }else { -if (bmd->is_our_pages) -zero_fill_bio (bio); -iov_iter_advance (iter, bio->bi_iter.bi_size); - } - - bio->bi_private = bmd; -if (map_data && map_data->null_mapped) -bio_set_flag (bio,BIO_NULL_MAPPED ); -return bio; -cleanup: -if (!map_data) -bio_free_pages (bio); -bio_put (bio); -out_bmd: -kfree (bmd); -return ERR_PTR (ret); -} - -/** -* bio_map_user_iov - map user iovec into bio -* @q: the struct request_queue for the bio -* @iter: iovec iterator -* @gfp_mask: memory allocation flags -* -* Map the user space address into a bio suitable for io to a block -* device. Returns an error pointer in case of error. -*/ -struct bio *bio_map_user_iov (struct request_queue *q, -struct iov_iter *iter, -gfp_t gfp_mask) -{ -int j; -struct bio *bio; -int ret; - -if (!iov_iter_count (iter)) -return ERR_PTR (-EINVAL ); - - bio =bio_kmalloc (gfp_mask,iov_iter_npages (iter,BIO_MAX_PAGES )); -if (!bio) -return ERR_PTR (-ENOMEM ); - -while (iov_iter_count (iter)) { -struct page **pages; -ssize_t bytes; -size_t offs, added =0 ; -int npages; - - bytes =iov_iter_get_pages_alloc (iter, &pages,LONG_MAX , &offs); -if (unlikely (bytes <=0 )) { - ret = bytes ? bytes : -EFAULT ; - goto out_unmap; - } - - npages =DIV_ROUND_UP (offs + bytes,PAGE_SIZE ); - -if (unlikely (offs &queue_dma_alignment (q))) { - ret = -EINVAL ; - j =0 ; - }else { -for (j =0 ; j < npages; j++) { -struct page *page = pages[j]; -unsigned int n =PAGE_SIZE - offs; -bool same_page = false; - -if (n > bytes) - n = bytes; - -if (!__bio_add_pc_page (q, bio, page, n, offs, - &same_page)) { -if (same_page) -put_page (page); -break ; - } - - added += n; - bytes -= n; - offs =0 ; - } -iov_iter_advance (iter, added); - } -/* -* release the pages we didn't map into the bio, if any -*/ -while (j < npages) -put_page (pages[j++]); -kvfree (pages); -/* couldn't stuff something into bio? */ -if (bytes) -break ; - } - -bio_set_flag (bio,BIO_USER_MAPPED ); - -/* -* subtle -- if bio_map_user_iov() ended up bouncing a bio, -* it would normally disappear when its bi_end_io is run. -* however, we need it for the unmap, so grab an extra -* reference to it -*/ -bio_get (bio); -return bio; - - out_unmap: -bio_release_pages (bio, false); -bio_put (bio); -return ERR_PTR (ret); -} - -/** -* bio_unmap_user - unmap a bio -* @bio: the bio being unmapped -* -* Unmap a bio previously mapped by bio_map_user_iov(). Must be called from -* process context. -* -* bio_unmap_user() may sleep. -*/ -void bio_unmap_user (struct bio *bio) -{ -bio_release_pages (bio,bio_data_dir (bio) ==READ ); -bio_put (bio); -bio_put (bio); -} - -static void bio_invalidate_vmalloc_pages (struct bio *bio) -{ -#ifdef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE -if (bio->bi_private && !op_is_write (bio_op (bio))) { -unsigned long i, len =0 ; - -for (i =0 ; i < bio->bi_vcnt; i++) - len += bio->bi_io_vec[i].bv_len; -invalidate_kernel_vmap_range (bio->bi_private, len); - } -#endif -} - -static void bio_map_kern_endio (struct bio *bio) -{ -bio_invalidate_vmalloc_pages (bio); -bio_put (bio); -} - -/** -* bio_map_kern - map kernel address into bio -* @q: the struct request_queue for the bio -* @data: pointer to buffer to map -* @len: length in bytes -* @gfp_mask: allocation flags for bio allocation -* -* Map the kernel address into a bio suitable for io to a block -* device. Returns an error pointer in case of error. -*/ -struct bio *bio_map_kern (struct request_queue *q,void *data,unsigned int len, -gfp_t gfp_mask) -{ -unsigned long kaddr = (unsigned long )data; -unsigned long end = (kaddr + len +PAGE_SIZE -1 ) >>PAGE_SHIFT ; -unsigned long start = kaddr >>PAGE_SHIFT ; -const int nr_pages = end - start; -bool is_vmalloc =is_vmalloc_addr (data); -struct page *page; -int offset, i; -struct bio *bio; - - bio =bio_kmalloc (gfp_mask, nr_pages); -if (!bio) -return ERR_PTR (-ENOMEM ); - -if (is_vmalloc) { -flush_kernel_vmap_range (data, len); - bio->bi_private = data; - } - - offset =offset_in_page (kaddr); -for (i =0 ; i < nr_pages; i++) { -unsigned int bytes =PAGE_SIZE - offset; - -if (len <=0 ) -break ; - -if (bytes > len) - bytes = len; - -if (!is_vmalloc) - page =virt_to_page (data); -else - page =vmalloc_to_page (data); -if (bio_add_pc_page (q, bio, page, bytes, - offset) < bytes) { -/* we don't support partial mappings */ -bio_put (bio); -return ERR_PTR (-EINVAL ); - } - - data += bytes; - len -= bytes; - offset =0 ; - } - - bio->bi_end_io = bio_map_kern_endio; -return bio; -} - -static void bio_copy_kern_endio (struct bio *bio) -{ -bio_free_pages (bio); -bio_put (bio); -} - -static void bio_copy_kern_endio_read (struct bio *bio) -{ -char *p = bio->bi_private; -struct bio_vec *bvec; -struct bvec_iter_all iter_all; - -bio_for_each_segment_all (bvec, bio, iter_all) { -memcpy (p,page_address (bvec->bv_page), bvec->bv_len); - p += bvec->bv_len; - } - -bio_copy_kern_endio (bio); -} - -/** -* bio_copy_kern - copy kernel address into bio -* @q: the struct request_queue for the bio -* @data: pointer to buffer to copy -* @len: length in bytes -* @gfp_mask: allocation flags for bio and page allocation -* @reading: data direction is READ -* -* copy the kernel address into a bio suitable for io to a block -* device. Returns an error pointer in case of error. -*/ -struct bio *bio_copy_kern (struct request_queue *q,void *data,unsigned int len, -gfp_t gfp_mask,int reading) -{ -unsigned long kaddr = (unsigned long )data; -unsigned long end = (kaddr + len +PAGE_SIZE -1 ) >>PAGE_SHIFT ; -unsigned long start = kaddr >>PAGE_SHIFT ; -struct bio *bio; -void *p = data; -int nr_pages =0 ; - -/* -* Overflow, abort -*/ -if (end < start) -return ERR_PTR (-EINVAL ); - - nr_pages = end - start; - bio =bio_kmalloc (gfp_mask, nr_pages); -if (!bio) -return ERR_PTR (-ENOMEM ); - -while (len) { -struct page *page; -unsigned int bytes =PAGE_SIZE ; - -if (bytes > len) - bytes = len; - - page =alloc_page (q->bounce_gfp | gfp_mask); -if (!page) - goto cleanup; - -if (!reading) -memcpy (page_address (page), p, bytes); - -if (bio_add_pc_page (q, bio, page, bytes,0 ) < bytes) -break ; - - len -= bytes; - p += bytes; - } - -if (reading) { - bio->bi_end_io = bio_copy_kern_endio_read; - bio->bi_private = data; - }else { - bio->bi_end_io = bio_copy_kern_endio; - } - -return bio; - -cleanup: -bio_free_pages (bio); -bio_put (bio); -return ERR_PTR (-ENOMEM ); -} -/* * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions * for performing direct-IO in BIOs.
block/blk-map.c +508 -0
514 lines not on this page · raw message
block/blk.h +4 -0
@@ -484,4 +484,8 @@ static inline void part_nr_sects_write(struct hd_struct *part, sector_t size)struct request_queue *__blk_alloc_queue (int node_id); +int __bio_add_pc_page (struct request_queue *q,struct bio *bio, +struct page *page,unsigned int len,unsigned int offset, +bool *same_page); +#endif /* BLK_INTERNAL_H */
include/linux/bio.h +0 -14
@@ -441,14 +441,6 @@ void __bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int off);int bio_iov_iter_get_pages (struct bio *bio,struct iov_iter *iter);void bio_release_pages (struct bio *bio,bool mark_dirty); -struct rq_map_data ; -extern struct bio *bio_map_user_iov (struct request_queue *, -struct iov_iter *,gfp_t ); -extern void bio_unmap_user (struct bio *); -extern struct bio *bio_map_kern (struct request_queue *,void *,unsigned int , -gfp_t ); -extern struct bio *bio_copy_kern (struct request_queue *,void *,unsigned int , -gfp_t ,int );extern void bio_set_pages_dirty (struct bio *bio);extern void bio_check_pages_dirty (struct bio *bio);
@@ -463,12 +455,6 @@ extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,extern void bio_copy_data (struct bio *dst,struct bio *src);extern void bio_list_copy_data (struct bio *dst,struct bio *src);extern void bio_free_pages (struct bio *bio); - -extern struct bio *bio_copy_user_iov (struct request_queue *, -struct rq_map_data *, -struct iov_iter *, -gfp_t ); -extern int bio_uncopy_user (struct bio *);void zero_fill_bio_iter (struct bio *bio,struct bvec_iter iter);void bio_truncate (struct bio *bio,unsigned new_size);void guard_bio_eod (struct bio *bio);
Signature · 1 line
2.25.1