cleanup and refactor BLOCK_PC mapping helpers V2
Subsystems (3)
This series is based on the first two patches from the reent series that Dongsu Park recently sent, and refactors and cleanups up various aspect of our BLOCK_PC mapping helpers.
Changes since V1:
- keep a separate blk_rq_map_user for now, it contains magic for the sg and tape drivers that's still missing from blk_rq_map_user_iov
PATCH 1/7 block: simplify bio_map_kern
Just open code the trivial mapping from a kernel virtual address to a bio instead of going through the complex user address mapping machinery.
--- block/bio.c | 60 ++++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 16 deletions(-)
block/bio.c +44 -16
@@ -1563,9 +1563,8 @@ static void bio_copy_kern_endio(struct bio *bio, int err) {struct bio_vec *bvec;const int read =bio_data_dir (bio) ==READ ; -struct bio_map_data *bmd = bio->bi_private; +char *p = bio->bi_private;int i; -char *p = bmd->sgvecs[0 ].iov_base;bio_for_each_segment_all (bvec, bio, i) {char *addr =page_address (bvec->bv_page);
@@ -1577,7 +1576,6 @@ static void bio_copy_kern_endio(struct bio *bio, int err) p += bvec->bv_len; } -kfree (bmd);bio_put (bio); }
@@ -1595,28 +1593,58 @@ static void bio_copy_kern_endio(struct bio *bio, int err)struct bio *bio_copy_kern (struct request_queue *q,void *data,unsigned int len,gfp_t gfp_mask,int reading) { -struct bio *bio; +unsigned long kaddr = (unsigned long )data; +unsigned long end = (kaddr + len +PAGE_SIZE -1 ) >>PAGE_SHIFT ; +unsigned long start = kaddr >>PAGE_SHIFT ;struct bio_vec *bvec; -int i; +struct bio *bio; +void *p = data; +int nr_pages =0 , i; - bio =bio_copy_user (q,NULL , (unsigned long )data, len,1 , gfp_mask); -if (IS_ERR (bio)) -return bio; +/* +* Overflow, abort +*/ +if (end < start) +return ERR_PTR (-EINVAL ); -if (!reading) { -void *p = data; + nr_pages = end - start; + bio =bio_kmalloc (gfp_mask, nr_pages); +if (!bio) +return ERR_PTR (-ENOMEM ); -bio_for_each_segment_all (bvec, bio, i) { -char *addr =page_address (bvec->bv_page); +while (len) { +struct page *page; +unsigned int bytes =PAGE_SIZE ; -memcpy (addr, p, bvec->bv_len); - p += bvec->bv_len; - } +if (bytes > len) + bytes = len; + + page =alloc_page (q->bounce_gfp | gfp_mask); +if (!page) + goto cleanup; + +if (!reading) +memcpy (page_address (page), p, bytes); + +if (bio_add_pc_page (q, bio, page, bytes,0 ) < bytes) +break ; + + len -= bytes; + p += bytes; } - bio->bi_end_io = bio_copy_kern_endio; +if (!reading) + bio->bi_rw |=REQ_WRITE ; + bio->bi_private = data; + bio->bi_end_io = bio_copy_kern_endio;return bio; + +cleanup: +bio_for_each_segment_all (bvec, bio, i) +__free_page (bvec->bv_page); +bio_put (bio); +return ERR_PTR (-ENOMEM ); }EXPORT_SYMBOL (bio_copy_kern);
Signature · 1 line
1.9.1
PATCH 2/7 block: use blk_rq_map_user_iov to implement blk_rq_map_user
In reply to Christoph Hellwig--- block/bio.c | 53 +------------------- block/blk-map.c | 137 +++++----------------------------------------------- include/linux/bio.h | 4 -- 3 files changed, 14 insertions(+), 180 deletions(-)
block/bio.c +1 -52
@@ -1102,7 +1102,7 @@ static int __bio_copy_iov(struct bio *bio, const struct sg_iovec *iov, int iov_c* bio_uncopy_user - finish previously mapped bio * @bio: bio being terminated * -* Free pages allocated from bio_copy_user() and write back data +* Free pages allocated from bio_copy_user_iov() and write back data * to user space in case of a read. */ int bio_uncopy_user (struct bio *bio)
@@ -1256,32 +1256,6 @@ out_bmd:return ERR_PTR (ret); } -/** -* bio_copy_user - copy user data to bio -* @q: destination block queue -* @map_data: pointer to the rq_map_data holding pages (if necessary) -* @uaddr: start of user address -* @len: length in bytes -* @write_to_vm: bool indicating writing to pages or not -* @gfp_mask: memory allocation flags -* -* Prepares and returns a bio for indirect user io, bouncing data -* to/from kernel pages as necessary. Must be paired with -* call bio_uncopy_user() on io completion. -*/ -struct bio *bio_copy_user (struct request_queue *q,struct rq_map_data *map_data, -unsigned long uaddr,unsigned int len, -int write_to_vm,gfp_t gfp_mask) -{ -struct sg_iovec iov; - - iov.iov_base = (void __user *)uaddr; - iov.iov_len = len; - -return bio_copy_user_iov (q, map_data, &iov,1 , write_to_vm, gfp_mask); -} -EXPORT_SYMBOL (bio_copy_user); - static struct bio *__bio_map_user_iov(struct request_queue *q, struct block_device *bdev, const struct sg_iovec *iov, int iov_count,
@@ -1395,31 +1369,6 @@ static struct bio *__bio_map_user_iov(struct request_queue *q, }/** -* bio_map_user - map user address into bio -* @q: the struct request_queue for the bio -* @bdev: destination block device -* @uaddr: start of user address -* @len: length in bytes -* @write_to_vm: bool indicating writing to pages or not -* @gfp_mask: memory allocation flags -* -* Map the user space address into a bio suitable for io to a block -* device. Returns an error pointer in case of error. -*/ -struct bio *bio_map_user (struct request_queue *q,struct block_device *bdev, -unsigned long uaddr,unsigned int len,int write_to_vm, -gfp_t gfp_mask) -{ -struct sg_iovec iov; - - iov.iov_base = (void __user *)uaddr; - iov.iov_len = len; - -return bio_map_user_iov (q, bdev, &iov,1 , write_to_vm, gfp_mask); -} -EXPORT_SYMBOL (bio_map_user); - -/** * bio_map_user_iov - map user sg_iovec table into bio * @q: the struct request_queue for the bio * @bdev: destination block device
block/blk-map.c +13 -124
149 lines not on this page · raw message
include/linux/bio.h +0 -4
@@ -428,8 +428,6 @@ extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);extern int bio_add_pc_page (struct request_queue *,struct bio *,struct page *,unsigned int ,unsigned int );extern int bio_get_nr_vecs (struct block_device *); -extern struct bio *bio_map_user (struct request_queue *,struct block_device *, -unsigned long ,unsigned int ,int ,gfp_t );struct sg_iovec ;struct rq_map_data ; extern struct bio *bio_map_user_iov(struct request_queue *,
@@ -462,8 +460,6 @@ static inline void bio_flush_dcache_pages(struct bio *bi)extern void bio_copy_data (struct bio *dst,struct bio *src);extern int bio_alloc_pages (struct bio *bio,gfp_t gfp); -extern struct bio *bio_copy_user (struct request_queue *,struct rq_map_data *, -unsigned long ,unsigned int ,int ,gfp_t ); extern struct bio *bio_copy_user_iov(struct request_queue *, struct rq_map_data *, const struct sg_iovec *,
Signature · 1 line
1.9.1
PATCH 3/7 block: add a helper to free bio bounce buffer pages
In reply to Christoph HellwigThe code sniplet to walk all bio_vecs and free their pages is opencoded in way to many places, so factor it into a helper. Also convert the slightly more complex cases in bio_kern_endio and __bio_copy_iov where we break the freeing from an existing loop into a separate one.
--- block/bio.c | 65 +++++++++++++++++++++++++++++++------------------------------ 1 file changed, 33 insertions(+), 32 deletions(-)
block/bio.c +33 -32
@@ -1048,7 +1048,7 @@ static struct bio_map_data *bio_alloc_map_data(unsigned int iov_count, }static int __bio_copy_iov (struct bio *bio,const struct sg_iovec *iov,int iov_count, -int to_user,int from_user,int do_free_page) +int to_user,int from_user) {int ret =0 , i;struct bio_vec *bvec;
@@ -1090,14 +1090,20 @@ static int __bio_copy_iov(struct bio *bio, const struct sg_iovec *iov, int iov_c iov_off =0 ; } } - -if (do_free_page) -__free_page (bvec->bv_page); }return ret; } +static void bio_free_pages (struct bio *bio) +{ +struct bio_vec *bvec; +int i; + +bio_for_each_segment_all (bvec, bio, i) +__free_page (bvec->bv_page); +} +/** * bio_uncopy_user - finish previously mapped bio * @bio: bio being terminated
@@ -1108,8 +1114,7 @@ static int __bio_copy_iov(struct bio *bio, const struct sg_iovec *iov, int iov_cint bio_uncopy_user (struct bio *bio) {struct bio_map_data *bmd = bio->bi_private; -struct bio_vec *bvec; -int ret =0 , i; +int ret =0 ;if (!bio_flagged (bio,BIO_NULL_MAPPED )) {/*
@@ -1118,11 +1123,9 @@ int bio_uncopy_user(struct bio *bio)*/ if (current->mm) ret =__bio_copy_iov (bio, bmd->sgvecs, bmd->nr_sgvecs, -bio_data_dir (bio) ==READ , -0 , bmd->is_our_pages); -else if (bmd->is_our_pages) -bio_for_each_segment_all (bvec, bio, i) -__free_page (bvec->bv_page); +bio_data_dir (bio) ==READ ,0 ); +if (bmd->is_our_pages) +bio_free_pages (bio); }kfree (bmd);bio_put (bio);
@@ -1149,7 +1152,6 @@ struct bio *bio_copy_user_iov(struct request_queue *q, int write_to_vm, gfp_t gfp_mask) {struct bio_map_data *bmd; -struct bio_vec *bvec;struct page *page;struct bio *bio;int i, ret;
@@ -1238,7 +1240,7 @@ struct bio *bio_copy_user_iov(struct request_queue *q,*/ if ((!write_to_vm && (!map_data || !map_data->null_mapped)) || (map_data && map_data->from_user)) { - ret =__bio_copy_iov (bio, iov, iov_count,0 ,1 ,0 ); + ret =__bio_copy_iov (bio, iov, iov_count,0 ,1 );if (ret) goto cleanup; }
@@ -1247,9 +1249,7 @@ struct bio *bio_copy_user_iov(struct request_queue *q,return bio; cleanup:if (!map_data) -bio_for_each_segment_all (bvec, bio, i) -__free_page (bvec->bv_page); - +bio_free_pages (bio);bio_put (bio); out_bmd:kfree (bmd);
@@ -1510,22 +1510,22 @@ EXPORT_SYMBOL(bio_map_kern);static void bio_copy_kern_endio (struct bio *bio,int err) { -struct bio_vec *bvec; -const int read =bio_data_dir (bio) ==READ ; +bio_free_pages (bio); +bio_put (bio); +} + +static void bio_copy_kern_endio_read (struct bio *bio,int err) +{char *p = bio->bi_private; +struct bio_vec *bvec;int i;bio_for_each_segment_all (bvec, bio, i) { -char *addr =page_address (bvec->bv_page); - -if (read) -memcpy (p, addr, bvec->bv_len); - -__free_page (bvec->bv_page); +memcpy (p,page_address (bvec->bv_page), bvec->bv_len); p += bvec->bv_len; } -bio_put (bio); +bio_copy_kern_endio (bio, err); }/**
@@ -1545,10 +1545,9 @@ struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len,unsigned long kaddr = (unsigned long )data;unsigned long end = (kaddr + len +PAGE_SIZE -1 ) >>PAGE_SHIFT ;unsigned long start = kaddr >>PAGE_SHIFT ; -struct bio_vec *bvec;struct bio *bio;void *p = data; -int nr_pages =0 , i; +int nr_pages =0 ;/* * Overflow, abort
@@ -1582,16 +1581,18 @@ struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len, p += bytes; } -if (!reading) +if (reading) { + bio->bi_end_io = bio_copy_kern_endio_read; + bio->bi_private = data; + }else { + bio->bi_end_io = bio_copy_kern_endio; bio->bi_rw |=REQ_WRITE ; + } - bio->bi_private = data; - bio->bi_end_io = bio_copy_kern_endio;return bio; cleanup: -bio_for_each_segment_all (bvec, bio, i) -__free_page (bvec->bv_page); +bio_free_pages (bio);bio_put (bio);return ERR_PTR (-ENOMEM ); }
Signature · 1 line
1.9.1
PATCH 4/7 block: pass iov_iter to the BLOCK_PC mapping functions
In reply to Christoph HellwigFrom: Kent Overstreet <[email protected]>
Make use of a new interface provided by iov_iter, backed by scatter-gather list of iovec, instead of the old interface based on sg_iovec. Also use iov_iter_advance() instead of manual iteration.
This commit should contain only literal replacements, without functional changes.
[hch: fixed to do a deep clone of the iov_iter, and to properly use
the iov_iter direction]
--- block/bio.c | 146 +++++++++++++++++++++++-------------------------- block/blk-map.c | 38 ++++++------- block/scsi_ioctl.c | 17 ++---- drivers/scsi/sg.c | 15 ++--- include/linux/bio.h | 7 +-- include/linux/blkdev.h | 4 +- 6 files changed, 101 insertions(+), 126 deletions(-)
block/bio.c +67 -79
@@ -28,7 +28,6 @@#include <linux/mempool.h> #include <linux/workqueue.h> #include <linux/cgroup.h> -#include <scsi/sg.h> /* for struct sg_iovec */ #include <trace/events/block.h>
@@ -1022,21 +1021,11 @@ void bio_copy_data(struct bio *dst, struct bio *src)EXPORT_SYMBOL (bio_copy_data);struct bio_map_data { -int nr_sgvecs;int is_our_pages; -struct sg_iovec sgvecs[]; +struct iov_iter iter; +struct iovec iov[]; }; -static void bio_set_map_data (struct bio_map_data *bmd,struct bio *bio, -const struct sg_iovec *iov,int iov_count, -int is_our_pages) -{ -memcpy (bmd->sgvecs, iov,sizeof (struct sg_iovec ) * iov_count); - bmd->nr_sgvecs = iov_count; - bmd->is_our_pages = is_our_pages; - bio->bi_private = bmd; -} -static struct bio_map_data *bio_alloc_map_data (unsigned int iov_count,gfp_t gfp_mask) {
@@ -1044,36 +1033,33 @@ static struct bio_map_data *bio_alloc_map_data(unsigned int iov_count,return NULL ;return kmalloc (sizeof (struct bio_map_data ) + -sizeof (struct sg_iovec ) * iov_count, gfp_mask); +sizeof (struct iovec ) * iov_count, gfp_mask); } -static int __bio_copy_iov (struct bio *bio,const struct sg_iovec *iov,int iov_count, -int to_user,int from_user) +static int __bio_copy_iov (struct bio *bio,const struct iov_iter *iter, +int to_user,int from_user) {int ret =0 , i;struct bio_vec *bvec; -int iov_idx =0 ; -unsigned int iov_off =0 ; +struct iov_iter iov_iter = *iter;bio_for_each_segment_all (bvec, bio, i) {char *bv_addr =page_address (bvec->bv_page);unsigned int bv_len = bvec->bv_len; -while (bv_len && iov_idx < iov_count) { -unsigned int bytes; -char __user *iov_addr; - - bytes = min_t(unsigned int, - iov[iov_idx].iov_len - iov_off, bv_len); - iov_addr = iov[iov_idx].iov_base + iov_off; +while (bv_len && iov_iter.count) { +struct iovec iov =iov_iter_iovec (&iov_iter); + unsigned int bytes = min_t(unsigned int, bv_len, + iov.iov_len);if (!ret) {if (to_user) - ret =copy_to_user (iov_addr, bv_addr, - bytes); + ret =copy_to_user (iov.iov_base, + bv_addr, bytes);if (from_user) - ret =copy_from_user (bv_addr, iov_addr, + ret =copy_from_user (bv_addr, + iov.iov_base, bytes);if (ret)
@@ -1082,13 +1068,7 @@ static int __bio_copy_iov(struct bio *bio, const struct sg_iovec *iov, int iov_c bv_len -= bytes; bv_addr += bytes; - iov_addr += bytes; - iov_off += bytes; - -if (iov[iov_idx].iov_len == iov_off) { - iov_idx++; - iov_off =0 ; - } +iov_iter_advance (&iov_iter, bytes); } }
@@ -1122,7 +1102,7 @@ int bio_uncopy_user(struct bio *bio)* don't copy into a random user address space, just free. */ if (current->mm) - ret =__bio_copy_iov (bio, bmd->sgvecs, bmd->nr_sgvecs, + ret =__bio_copy_iov (bio, &bmd->iter,bio_data_dir (bio) ==READ ,0 );if (bmd->is_our_pages)bio_free_pages (bio);
@@ -1135,12 +1115,10 @@ EXPORT_SYMBOL(bio_uncopy_user);/** * bio_copy_user_iov - copy user data to bio -* @q: destination block queue -* @map_data: pointer to the rq_map_data holding pages (if necessary) -* @iov: the iovec. -* @iov_count: number of elements in the iovec -* @write_to_vm: bool indicating writing to pages or not -* @gfp_mask: memory allocation flags +* @q: destination block queue +* @map_data: pointer to the rq_map_data holding pages (if necessary) +* @iter: iovec iterator +* @gfp_mask: memory allocation flags * * Prepares and returns a bio for indirect user io, bouncing data * to/from kernel pages as necessary. Must be paired with
@@ -1148,24 +1126,25 @@ EXPORT_SYMBOL(bio_uncopy_user);*/ struct bio *bio_copy_user_iov (struct request_queue *q,struct rq_map_data *map_data, -const struct sg_iovec *iov,int iov_count, -int write_to_vm,gfp_t gfp_mask) +const struct iov_iter *iter, +gfp_t gfp_mask) {struct bio_map_data *bmd;struct page *page;struct bio *bio;int i, ret;int nr_pages =0 ; -unsigned int len =0 ; +unsigned int len = iter->count;unsigned int offset = map_data ? map_data->offset & ~PAGE_MASK :0 ; -for (i =0 ; i < iov_count; i++) { +for (i =0 ; i < iter->nr_segs; i++) {unsigned long uaddr;unsigned long end;unsigned long start; - uaddr = (unsigned long )iov[i].iov_base; - end = (uaddr + iov[i].iov_len +PAGE_SIZE -1 ) >>PAGE_SHIFT ; + uaddr = (unsigned long ) iter->iov[i].iov_base; + end = (uaddr + iter->iov[i].iov_len +PAGE_SIZE -1 ) + >>PAGE_SHIFT ; start = uaddr >>PAGE_SHIFT ;/*
@@ -1175,22 +1154,31 @@ struct bio *bio_copy_user_iov(struct request_queue *q,return ERR_PTR (-EINVAL ); nr_pages += end - start; - len += iov[i].iov_len; }if (offset) nr_pages++; - bmd =bio_alloc_map_data (iov_count, gfp_mask); + bmd =bio_alloc_map_data (iter->nr_segs, gfp_mask);if (!bmd)return ERR_PTR (-ENOMEM ); +/* +* We need to do a deep copy of the iov_iter including the iovecs. +* The caller provided iov might point to an on-stack or otherwise +* shortlived one. +*/ + bmd->is_our_pages = map_data ?0 :1 ; +memcpy (bmd->iov, iter->iov,sizeof (struct iovec ) * iter->nr_segs); +iov_iter_init (&bmd->iter, iter->type, bmd->iov, + iter->nr_segs, iter->count); + ret = -ENOMEM ; bio =bio_kmalloc (gfp_mask, nr_pages);if (!bio) goto out_bmd; -if (!write_to_vm) +if (iter->type &WRITE ) bio->bi_rw |=REQ_WRITE ; ret =0 ;
@@ -1238,14 +1226,14 @@ struct bio *bio_copy_user_iov(struct request_queue *q,/* * success */ -if ((!write_to_vm && (!map_data || !map_data->null_mapped)) || +if (((iter->type &WRITE ) && (!map_data || !map_data->null_mapped)) || (map_data && map_data->from_user)) { - ret =__bio_copy_iov (bio, iov, iov_count,0 ,1 ); + ret =__bio_copy_iov (bio, iter,0 ,1 );if (ret) goto cleanup; } -bio_set_map_data (bmd, bio, iov, iov_count, map_data ?0 :1 ); + bio->bi_private = bmd;return bio; cleanup:if (!map_data)
@@ -1258,19 +1246,21 @@ out_bmd:static struct bio *__bio_map_user_iov (struct request_queue *q,struct block_device *bdev, -const struct sg_iovec *iov,int iov_count, -int write_to_vm,gfp_t gfp_mask) +const struct iov_iter *iter, +gfp_t gfp_mask) { -int i, j; +int j;int nr_pages =0 ;struct page **pages;struct bio *bio;int cur_page =0 ;int ret, offset; +struct iov_iter i; +struct iovec iov; -for (i =0 ; i < iov_count; i++) { -unsigned long uaddr = (unsigned long )iov[i].iov_base; -unsigned long len = iov[i].iov_len; +iov_for_each (iov, i, *iter) { +unsigned long uaddr = (unsigned long ) iov.iov_base; +unsigned long len = iov.iov_len;unsigned long end = (uaddr + len +PAGE_SIZE -1 ) >>PAGE_SHIFT ;unsigned long start = uaddr >>PAGE_SHIFT ;
@@ -1300,16 +1290,17 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,if (!pages) goto out; -for (i =0 ; i < iov_count; i++) { -unsigned long uaddr = (unsigned long )iov[i].iov_base; -unsigned long len = iov[i].iov_len; +iov_for_each (iov, i, *iter) { +unsigned long uaddr = (unsigned long ) iov.iov_base; +unsigned long len = iov.iov_len;unsigned long end = (uaddr + len +PAGE_SIZE -1 ) >>PAGE_SHIFT ;unsigned long start = uaddr >>PAGE_SHIFT ;const int local_nr_pages = end - start;const int page_limit = cur_page + local_nr_pages; ret =get_user_pages_fast (uaddr, local_nr_pages, - write_to_vm, &pages[cur_page]); + (iter->type &WRITE ) !=WRITE , + &pages[cur_page]);if (ret < local_nr_pages) { ret = -EFAULT ; goto out_unmap;
@@ -1349,7 +1340,7 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,/* * set data direction, and check if mapped pages need bouncing */ -if (!write_to_vm) +if (iter->type &WRITE ) bio->bi_rw |=REQ_WRITE ; bio->bi_bdev = bdev;
@@ -1357,10 +1348,10 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,return bio; out_unmap: -for (i =0 ; i < nr_pages; i++) { -if (!pages[i]) +for (j =0 ; j < nr_pages; j++) { +if (!pages[j])break ; -page_cache_release (pages[i]); +page_cache_release (pages[j]); } out:kfree (pages);
@@ -1369,25 +1360,22 @@ static struct bio *__bio_map_user_iov(struct request_queue *q, }/** -* bio_map_user_iov - map user sg_iovec table into bio -* @q: the struct request_queue for the bio -* @bdev: destination block device -* @iov: the iovec. -* @iov_count: number of elements in the iovec -* @write_to_vm: bool indicating writing to pages or not -* @gfp_mask: memory allocation flags +* bio_map_user_iov - map user iovec into bio +* @q: the struct request_queue for the bio +* @bdev: destination block device +* @iter: iovec iterator +* @gfp_mask: memory allocation flags * * Map the user space address into a bio suitable for io to a block * device. Returns an error pointer in case of error. */ struct bio *bio_map_user_iov (struct request_queue *q,struct block_device *bdev, -const struct sg_iovec *iov,int iov_count, -int write_to_vm,gfp_t gfp_mask) +const struct iov_iter *iter, +gfp_t gfp_mask) {struct bio *bio; - bio =__bio_map_user_iov (q, bdev, iov, iov_count, write_to_vm, - gfp_mask); + bio =__bio_map_user_iov (q, bdev, iter, gfp_mask);if (IS_ERR (bio))return bio;
block/blk-map.c +19 -19
@@ -5,7 +5,7 @@#include <linux/module.h> #include <linux/bio.h> #include <linux/blkdev.h> -#include <scsi/sg.h> /* for struct sg_iovec */ +#include <linux/uio.h> #include "blk.h"
@@ -44,9 +44,7 @@ static int __blk_rq_unmap_user(struct bio *bio)* @q: request queue where request should be inserted * @rq: request to map data to * @map_data: pointer to the rq_map_data holding pages (if necessary) -* @iov: pointer to the iovec -* @iov_count: number of elements in the iovec -* @len: I/O byte count +* @iter: iovec iterator * @gfp_mask: memory allocation flags * * Description:
@@ -63,20 +61,21 @@ static int __blk_rq_unmap_user(struct bio *bio)* unmapping. */ int blk_rq_map_user_iov (struct request_queue *q,struct request *rq, -struct rq_map_data *map_data,const struct sg_iovec *iov, -int iov_count,unsigned int len,gfp_t gfp_mask) +struct rq_map_data *map_data, +const struct iov_iter *iter,gfp_t gfp_mask) {struct bio *bio; -int i, read =rq_data_dir (rq) ==READ ;int unaligned =0 ; +struct iov_iter i; +struct iovec iov; -if (!iov || iov_count <=0 ) +if (!iter || !iter->count)return -EINVAL ; -for (i =0 ; i < iov_count; i++) { -unsigned long uaddr = (unsigned long )iov[i].iov_base; +iov_for_each (iov, i, *iter) { +unsigned long uaddr = (unsigned long ) iov.iov_base; -if (!iov[i].iov_len) +if (!iov.iov_len)return -EINVAL ;/*
@@ -86,16 +85,15 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, unaligned =1 ; } -if (unaligned || (q->dma_pad_mask & len) || map_data) - bio =bio_copy_user_iov (q, map_data, iov, iov_count, read, - gfp_mask); +if (unaligned || (q->dma_pad_mask & iter->count) || map_data) + bio =bio_copy_user_iov (q, map_data, iter, gfp_mask);else - bio =bio_map_user_iov (q,NULL , iov, iov_count, read, gfp_mask); + bio =bio_map_user_iov (q,NULL , iter, gfp_mask);if (IS_ERR (bio))return PTR_ERR (bio); -if (bio->bi_iter.bi_size != len) { +if (bio->bi_iter.bi_size != iter->count) {/* * Grab an extra reference to this bio, as bio_unmap_user() * expects to be able to drop it twice as it happens on the
@@ -121,12 +119,14 @@ int blk_rq_map_user(struct request_queue *q, struct request *rq, struct rq_map_data *map_data, void __user *ubuf, unsigned long len, gfp_t gfp_mask) { - struct sg_iovec iov; + struct iovec iov; +struct iov_iter i; - iov.iov_base = (void __user *)ubuf; + iov.iov_base = ubuf; iov.iov_len = len; +iov_iter_init (&i,rq_data_dir (rq), &iov,1 , len); -return blk_rq_map_user_iov (q, rq, map_data, &iov,1 , len, gfp_mask); +return blk_rq_map_user_iov (q, rq, map_data, &i, gfp_mask); }EXPORT_SYMBOL (blk_rq_map_user);
block/scsi_ioctl.c +4 -13
@@ -332,7 +332,7 @@ static int sg_io(struct request_queue *q, struct gendisk *bd_disk, ret =0 ;if (hdr->iovec_count) { -size_t iov_data_len; +struct iov_iter i;struct iovec *iov =NULL ; ret = rw_copy_check_uvector(-1, hdr->dxferp, hdr->iovec_count,
@@ -342,20 +342,11 @@ static int sg_io(struct request_queue *q, struct gendisk *bd_disk, goto out_free_cdb; } - iov_data_len = ret; - ret =0 ; -/* SG_IO howto says that the shorter of the two wins */ -if (hdr->dxfer_len < iov_data_len) { - hdr->iovec_count =iov_shorten (iov, - hdr->iovec_count, - hdr->dxfer_len); - iov_data_len = hdr->dxfer_len; - } +iov_iter_init (&i,rq_data_dir (rq), iov, hdr->iovec_count, +min_t (unsigned, ret, hdr->dxfer_len)); - ret =blk_rq_map_user_iov (q, rq,NULL , (struct sg_iovec *) iov, - hdr->iovec_count, - iov_data_len,GFP_KERNEL ); + ret =blk_rq_map_user_iov (q, rq,NULL , &i,GFP_KERNEL );kfree (iov); }else if (hdr->dxfer_len) ret = blk_rq_map_user(q, rq, NULL, hdr->dxferp, hdr->dxfer_len,
drivers/scsi/sg.c +6 -9
@@ -1734,22 +1734,19 @@ sg_start_req(Sg_request *srp, unsigned char *cmd) }if (iov_count) { -int len, size =sizeof (struct sg_iovec ) * iov_count; +int size =sizeof (struct iovec ) * iov_count;struct iovec *iov; +struct iov_iter i; iov =memdup_user (hp->dxferp, size);if (IS_ERR (iov))return PTR_ERR (iov); - len =iov_length (iov, iov_count); -if (hp->dxfer_len < len) { - iov_count =iov_shorten (iov, iov_count, hp->dxfer_len); - len = hp->dxfer_len; - } +iov_iter_init (&i, rw, iov, iov_count, +min_t (size_t, hp->dxfer_len, +iov_length (iov, iov_count))); - res =blk_rq_map_user_iov (q, rq, md, (struct sg_iovec *)iov, - iov_count, - len,GFP_ATOMIC ); + res =blk_rq_map_user_iov (q, rq, md, &i,GFP_ATOMIC );kfree (iov); }else res = blk_rq_map_user(q, rq, md, hp->dxferp,
include/linux/bio.h +3 -4
@@ -428,11 +428,10 @@ extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);extern int bio_add_pc_page (struct request_queue *,struct bio *,struct page *,unsigned int ,unsigned int );extern int bio_get_nr_vecs (struct block_device *); -struct sg_iovec ;struct rq_map_data ;extern struct bio *bio_map_user_iov (struct request_queue *,struct block_device *, -const struct sg_iovec *,int ,int ,gfp_t ); +const struct iov_iter *,gfp_t );extern void bio_unmap_user (struct bio *);extern struct bio *bio_map_kern (struct request_queue *,void *,unsigned int ,gfp_t );
@@ -462,8 +461,8 @@ extern int bio_alloc_pages(struct bio *bio, gfp_t gfp);extern struct bio *bio_copy_user_iov (struct request_queue *,struct rq_map_data *, -const struct sg_iovec *, -int ,int ,gfp_t ); +const struct iov_iter *, +gfp_t );extern int bio_uncopy_user (struct bio *);void zero_fill_bio (struct bio *bio);extern struct bio_vec *bvec_alloc (gfp_t ,int ,unsigned long *,mempool_t *);
include/linux/blkdev.h +2 -2
@@ -850,8 +850,8 @@ extern int blk_rq_map_user(struct request_queue *, struct request *,extern int blk_rq_unmap_user (struct bio *);extern int blk_rq_map_kern (struct request_queue *,struct request *,void *,unsigned int ,gfp_t );extern int blk_rq_map_user_iov (struct request_queue *,struct request *, -struct rq_map_data *,const struct sg_iovec *, -int ,unsigned int ,gfp_t ); +struct rq_map_data *,const struct iov_iter *, +gfp_t );extern int blk_execute_rq (struct request_queue *,struct gendisk *,struct request *,int ); extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
Signature · 1 line
1.9.1
PATCH 5/7 block: merge __bio_map_kern into bio_map_kern
In reply to Christoph HellwigThis saves a little code, and allow to simplify the error handling.
--- block/bio.c | 50 +++++++++++++++++--------------------------------- 1 file changed, 17 insertions(+), 33 deletions(-)
block/bio.c +17 -33
@@ -1429,8 +1429,18 @@ static void bio_map_kern_endio(struct bio *bio, int err)bio_put (bio); } -static struct bio *__bio_map_kern (struct request_queue *q,void *data, -unsigned int len,gfp_t gfp_mask) +/** +* bio_map_kern - map kernel address into bio +* @q: the struct request_queue for the bio +* @data: pointer to buffer to map +* @len: length in bytes +* @gfp_mask: allocation flags for bio allocation +* +* Map the kernel address into a bio suitable for io to a block +* device. Returns an error pointer in case of error. +*/ +struct bio *bio_map_kern (struct request_queue *q,void *data,unsigned int len, +gfp_t gfp_mask) {unsigned long kaddr = (unsigned long )data;unsigned long end = (kaddr + len +PAGE_SIZE -1 ) >>PAGE_SHIFT ;
@@ -1454,8 +1464,11 @@ static struct bio *__bio_map_kern(struct request_queue *q, void *data, bytes = len;if (bio_add_pc_page (q, bio,virt_to_page (data), bytes, - offset) < bytes) -break ; + offset) < bytes) { +/* we don't support partial mappings */ +bio_put (bio); +return ERR_PTR (-EINVAL ); + } data += bytes; len -= bytes;
@@ -1465,35 +1478,6 @@ static struct bio *__bio_map_kern(struct request_queue *q, void *data, bio->bi_end_io = bio_map_kern_endio;return bio; } - -/** -* bio_map_kern - map kernel address into bio -* @q: the struct request_queue for the bio -* @data: pointer to buffer to map -* @len: length in bytes -* @gfp_mask: allocation flags for bio allocation -* -* Map the kernel address into a bio suitable for io to a block -* device. Returns an error pointer in case of error. -*/ -struct bio *bio_map_kern (struct request_queue *q,void *data,unsigned int len, -gfp_t gfp_mask) -{ -struct bio *bio; - - bio =__bio_map_kern (q, data, len, gfp_mask); -if (IS_ERR (bio)) -return bio; - -if (bio->bi_iter.bi_size == len) -return bio; - -/* -* Don't support partial mappings. -*/ -bio_put (bio); -return ERR_PTR (-EINVAL ); -}EXPORT_SYMBOL (bio_map_kern);static void bio_copy_kern_endio (struct bio *bio,int err)
Signature · 1 line
1.9.1
PATCH 6/7 block: merge __bio_map_user_iov into bio_map_user_iov
In reply to Christoph HellwigAnd also remove the unused bdev argument.
--- block/bio.c | 56 +++++++++++++++++++---------------------------------- block/blk-map.c | 2 +- include/linux/bio.h | 1 - 3 files changed, 21 insertions(+), 38 deletions(-)
block/bio.c +20 -36
@@ -1244,10 +1244,18 @@ out_bmd:return ERR_PTR (ret); } -static struct bio *__bio_map_user_iov (struct request_queue *q, -struct block_device *bdev, -const struct iov_iter *iter, -gfp_t gfp_mask) +/** +* bio_map_user_iov - map user iovec into bio +* @q: the struct request_queue for the bio +* @iter: iovec iterator +* @gfp_mask: memory allocation flags +* +* Map the user space address into a bio suitable for io to a block +* device. Returns an error pointer in case of error. +*/ +struct bio *bio_map_user_iov (struct request_queue *q, +const struct iov_iter *iter, +gfp_t gfp_mask) {int j;int nr_pages =0 ;
@@ -1343,8 +1351,15 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,if (iter->type &WRITE ) bio->bi_rw |=REQ_WRITE ; - bio->bi_bdev = bdev; bio->bi_flags |= (1 <<BIO_USER_MAPPED ); + +/* +* subtle -- if __bio_map_user() ended up bouncing a bio, +* it would normally disappear when its bi_end_io is run. +* however, we need it for the unmap, so grab an extra +* reference to it +*/ +bio_get (bio);return bio; out_unmap:
@@ -1359,37 +1374,6 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,return ERR_PTR (ret); } -/** -* bio_map_user_iov - map user iovec into bio -* @q: the struct request_queue for the bio -* @bdev: destination block device -* @iter: iovec iterator -* @gfp_mask: memory allocation flags -* -* Map the user space address into a bio suitable for io to a block -* device. Returns an error pointer in case of error. -*/ -struct bio *bio_map_user_iov (struct request_queue *q,struct block_device *bdev, -const struct iov_iter *iter, -gfp_t gfp_mask) -{ -struct bio *bio; - - bio =__bio_map_user_iov (q, bdev, iter, gfp_mask); -if (IS_ERR (bio)) -return bio; - -/* -* subtle -- if __bio_map_user() ended up bouncing a bio, -* it would normally disappear when its bi_end_io is run. -* however, we need it for the unmap, so grab an extra -* reference to it -*/ -bio_get (bio); - -return bio; -} -static void __bio_unmap_user (struct bio *bio) {struct bio_vec *bvec;
block/blk-map.c +1 -1
@@ -88,7 +88,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,if (unaligned || (q->dma_pad_mask & iter->count) || map_data) bio =bio_copy_user_iov (q, map_data, iter, gfp_mask);else - bio =bio_map_user_iov (q,NULL , iter, gfp_mask); + bio =bio_map_user_iov (q, iter, gfp_mask);if (IS_ERR (bio))return PTR_ERR (bio);
include/linux/bio.h +0 -1
@@ -430,7 +430,6 @@ extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,extern int bio_get_nr_vecs (struct block_device *);struct rq_map_data ;extern struct bio *bio_map_user_iov (struct request_queue *, -struct block_device *,const struct iov_iter *,gfp_t );extern void bio_unmap_user (struct bio *); extern struct bio *bio_map_kern(struct request_queue *, void *, unsigned int,
Signature · 1 line
1.9.1
PATCH 7/7 block: rewrite and split __bio_copy_iov()
In reply to Christoph HellwigFrom: Dongsu Park <[email protected]>
Rewrite __bio_copy_iov using the copy_page_{from,to}_iter helpers, and split it into two simpler functions.
This commit should contain only literal replacements, without functional changes.
--- block/bio.c | 90 ++++++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 56 insertions(+), 34 deletions(-)
block/bio.c +56 -34
@@ -1036,43 +1036,66 @@ static struct bio_map_data *bio_alloc_map_data(unsigned int iov_count, sizeof(struct iovec) * iov_count, gfp_mask); } -static int __bio_copy_iov (struct bio *bio,const struct iov_iter *iter, -int to_user,int from_user) +/** +* bio_copy_from_iter - copy all pages from iov_iter to bio +* @bio: The &struct bio which describes the I/O as destination +* @iter: iov_iter as source +* +* Copy all pages from iov_iter to bio. +* Returns 0 on success, or error on failure. +*/ +static int bio_copy_from_iter (struct bio *bio,struct iov_iter iter) { -int ret =0 , i; +int i;struct bio_vec *bvec; -struct iov_iter iov_iter = *iter;bio_for_each_segment_all (bvec, bio, i) { -char *bv_addr =page_address (bvec->bv_page); -unsigned int bv_len = bvec->bv_len; - -while (bv_len && iov_iter.count) { -struct iovec iov =iov_iter_iovec (&iov_iter); - unsigned int bytes = min_t(unsigned int, bv_len, - iov.iov_len); - -if (!ret) { -if (to_user) - ret =copy_to_user (iov.iov_base, - bv_addr, bytes); - -if (from_user) - ret =copy_from_user (bv_addr, - iov.iov_base, - bytes); - -if (ret) - ret = -EFAULT ; - } +ssize_t ret; - bv_len -= bytes; - bv_addr += bytes; -iov_iter_advance (&iov_iter, bytes); - } + ret =copy_page_from_iter (bvec->bv_page, + bvec->bv_offset, + bvec->bv_len, + &iter); + +if (!iov_iter_count (&iter)) +break ; + +if (ret < bvec->bv_len) +return -EFAULT ; } -return ret; +return 0 ; +} + +/** +* bio_copy_to_iter - copy all pages from bio to iov_iter +* @bio: The &struct bio which describes the I/O as source +* @iter: iov_iter as destination +* +* Copy all pages from bio to iov_iter. +* Returns 0 on success, or error on failure. +*/ +static int bio_copy_to_iter (struct bio *bio,struct iov_iter iter) +{ +int i; +struct bio_vec *bvec; + +bio_for_each_segment_all (bvec, bio, i) { +ssize_t ret; + + ret =copy_page_to_iter (bvec->bv_page, + bvec->bv_offset, + bvec->bv_len, + &iter); + +if (!iov_iter_count (&iter)) +break ; + +if (ret < bvec->bv_len) +return -EFAULT ; + } + +return 0 ; }static void bio_free_pages (struct bio *bio)
@@ -1101,9 +1124,8 @@ int bio_uncopy_user(struct bio *bio)* if we're in a workqueue, the request is orphaned, so * don't copy into a random user address space, just free. */ -if (current->mm) - ret =__bio_copy_iov (bio, &bmd->iter, -bio_data_dir (bio) ==READ ,0 ); +if (current->mm &&bio_data_dir (bio) ==READ ) + ret =bio_copy_to_iter (bio, bmd->iter);if (bmd->is_our_pages)bio_free_pages (bio); }
@@ -1228,7 +1250,7 @@ struct bio *bio_copy_user_iov(struct request_queue *q,*/ if (((iter->type &WRITE ) && (!map_data || !map_data->null_mapped)) || (map_data && map_data->from_user)) { - ret =__bio_copy_iov (bio, iter,0 ,1 ); + ret =bio_copy_from_iter (bio, *iter);if (ret) goto cleanup; }
Signature · 1 line
1.9.1
Ming Lei
Reviewed-byQuoted 8 lines · Christoph Hellwig
This series is based on the first two patches from the reent series that Dongsu Park recently sent, and refactors and cleanups up various aspect of our
BLOCK_PCmapping helpers.Changes since V1:
- keep a separateblk_rq_map_userfor now, it contains magic for the sg and tape drivers that's still missing fromblk_rq_map_user_iov
Looks all are nice cleanup:
Also patches passed xfstests(check -g auto) against v3.19-rc4_next-20150115.
Thanks,
Ming Lei
Quoted 5 lines · Ming Lei
Looks all are nice cleanup:
Reviewed-by Ming Lei <[email protected]>Also patches passed xfstests(check -g auto) against v3.19-rc4_next-20150115.
Jens, do these patches look fine to you? Any chance to get them into the tree for the 3.20 merge window?
Quoted 9 lines · Christoph Hellwig
Quoted 5 lines · Ming Lei
Looks all are nice cleanup:
Reviewed-by Ming Lei <[email protected]>Also patches passed xfstests(check -g auto) against v3.19-rc4_next-20150115.
Jens, do these patches look fine to you? Any chance to get them into the tree for the 3.20 merge window?
ping?
Jens Axboe
maintainerQuoted 9 lines · Christoph Hellwig
Quoted 5 lines · Ming Lei
Looks all are nice cleanup:
Reviewed-by Ming Lei <[email protected]>Also patches passed xfstests(check -g auto) against v3.19-rc4_next-20150115.
Jens, do these patches look fine to you? Any chance to get them into the tree for the 3.20 merge window?
Yes, I think they look fine. I'll throw them into the testing mix and merge them for 3.20.
Signature · 1 line
Jens Axboe
Quoted 6 lines · Jens Axboe
Quoted 2 lines · Christoph Hellwig
Jens, do these patches look fine to you? Any chance to get them into the tree for the 3.20 merge window?
Yes, I think they look fine. I'll throw them into the testing mix and merge them for 3.20.
Thanks a lot, and many thanks also to Christoph.
Dongsu
Quoted 3 lines
Signature · 1 line
Jens Axboe