block: fix page leak by merging to same page
Subsystems (3)
v3 cover block: fix page leak by merging to same page
Hi,
'pages' retrived by __bio_iov_iter_get_pages() may point to same page, and finally they can be merged to the same page in bio_add_page(), then page leak can be caused because bio_release_pages() only drops the page ref once.
Fixes this issue by dropping the extra page ref.
V3:
- avoid to call __bio_try_merge_page() two times
- add comment on the two new flags
V2:
- V1 breaks multi-page merge, and fix it and only put the page ref if the added page is really the 'same page'
Ming Lei (2):
block: introduce 'enum bvec_merge_flags' for __bio_try_merge_page
block: fix page leak in case of merging to same page
block/bio.c | 32 ++++++++++++++++++++++---------- fs/iomap.c | 3 ++- fs/xfs/xfs_aops.c | 3 ++- include/linux/bio.h | 22 +++++++++++++++++++++- 4 files changed, 47 insertions(+), 13 deletions(-)
Signature · 1 line
2.20.1
PATCH v3 1/2 block: introduce 'enum bvec_merge_flags' for __bio_try_merge_page
Introduce 'enum bvec_merge_flags' and pass it to __bio_try_merge_page, we have to deal with several cases related with page reference when merging same page to bio(bvec), such as:
1) only merge to same page without putting reference of the same page, such as iomap & xfs
2) merge to same page and putting reference of the same page, such as __bio_iov_iter_get_pages()
--- block/bio.c | 20 ++++++++++++-------- fs/iomap.c | 3 ++- fs/xfs/xfs_aops.c | 3 ++- include/linux/bio.h | 14 +++++++++++++- 4 files changed, 29 insertions(+), 11 deletions(-)
block/bio.c +12 -8
@@ -636,7 +636,7 @@ EXPORT_SYMBOL(bio_clone_fast);static inline bool page_is_mergeable (const struct bio_vec *bv,struct page *page,unsigned int len,unsigned int off, -bool same_page) +enum bvec_merge_flags flags) {phys_addr_t vec_end_addr =page_to_phys (bv->bv_page) + bv->bv_offset + bv->bv_len -1 ;
@@ -648,13 +648,14 @@ static inline bool page_is_mergeable(const struct bio_vec *bv,return false;if ((vec_end_addr &PAGE_MASK ) != page_addr) { -if (same_page) +if (flags &BVEC_MERGE_TO_SAME_PAGE )return false;if (pfn_to_page (PFN_DOWN (vec_end_addr)) +1 != page)return false; } -WARN_ON_ONCE (same_page && (len + off) >PAGE_SIZE ); +WARN_ON_ONCE ((flags &BVEC_MERGE_TO_SAME_PAGE ) && + (len + off) >PAGE_SIZE );return true; }
@@ -729,8 +730,9 @@ static int __bio_add_pc_page(struct request_queue *q, struct bio *bio,if (bvec_gap_to_prev (q, bvec, offset))return 0 ; -if (page_is_mergeable (bvec, page, len, offset, false) && -can_add_page_to_seg (q, bvec, page, len, offset)) { +if (page_is_mergeable (bvec, page, len, offset, +BVEC_MERGE_DEFAULT ) &&can_add_page_to_seg (q, bvec, + page, len, offset)) { bvec->bv_len += len; goto done; }
@@ -779,7 +781,8 @@ EXPORT_SYMBOL(bio_add_pc_page);* Return %true on success or %false on failure. */ bool __bio_try_merge_page (struct bio *bio,struct page *page, -unsigned int len,unsigned int off,bool same_page) +unsigned int len,unsigned int off, +enum bvec_merge_flags flags) {if (WARN_ON_ONCE (bio_flagged (bio,BIO_CLONED )))return false;
@@ -787,7 +790,7 @@ bool __bio_try_merge_page(struct bio *bio, struct page *page,if (bio->bi_vcnt >0 ) {struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt -1 ]; -if (page_is_mergeable (bv, page, len, off, same_page)) { +if (page_is_mergeable (bv, page, len, off, flags)) { bv->bv_len += len; bio->bi_iter.bi_size += len;return true;
@@ -837,7 +840,8 @@ EXPORT_SYMBOL_GPL(__bio_add_page);int bio_add_page (struct bio *bio,struct page *page,unsigned int len,unsigned int offset) { -if (!__bio_try_merge_page (bio, page, len, offset, false)) { +if (!__bio_try_merge_page (bio, page, len, offset, +BVEC_MERGE_DEFAULT )) {if (bio_full (bio))return 0 ;__bio_add_page (bio, page, len, offset);
fs/iomap.c +2 -1
@@ -316,7 +316,8 @@ iomap_readpage_actor(struct inode *inode, loff_t pos, loff_t length, void *data,*/ sector =iomap_sector (iomap, pos);if (ctx->bio &&bio_end_sector (ctx->bio) == sector) { -if (__bio_try_merge_page (ctx->bio, page, plen, poff, true)) +if (__bio_try_merge_page (ctx->bio, page, plen, poff, +BVEC_MERGE_TO_SAME_PAGE )) goto done; is_contig = true; }
fs/xfs/xfs_aops.c +2 -1
@@ -774,7 +774,8 @@ xfs_add_to_ioend( wpc->imap.br_state, offset, bdev, sector); } -if (!__bio_try_merge_page (wpc->ioend->io_bio, page, len, poff, true)) { +if (!__bio_try_merge_page (wpc->ioend->io_bio, page, len, poff, +BVEC_MERGE_TO_SAME_PAGE )) {if (iop)atomic_inc (&iop->write_count);if (bio_full (wpc->ioend->io_bio))
include/linux/bio.h +13 -1
@@ -419,11 +419,23 @@ extern void bio_uninit(struct bio *);extern void bio_reset (struct bio *);void bio_chain (struct bio *,struct bio *); +enum bvec_merge_flags { +BVEC_MERGE_DEFAULT , + +/* +* only merge if new page is same with bio's last page, this +* is exactly the behaviour before introducing multi-page +* bvec +*/ +BVEC_MERGE_TO_SAME_PAGE =BIT (0 ), +}; +extern int bio_add_page (struct bio *,struct page *,unsigned int ,unsigned int );extern int bio_add_pc_page (struct request_queue *,struct bio *,struct page *,unsigned int ,unsigned int );bool __bio_try_merge_page (struct bio *bio,struct page *page, -unsigned int len,unsigned int off,bool same_page); +unsigned int len,unsigned int off, +enum bvec_merge_flags flags);void __bio_add_page (struct bio *bio,struct page *page,unsigned int len,unsigned int off);int bio_iov_iter_get_pages (struct bio *bio,struct iov_iter *iter);
Signature · 1 line
2.20.1
PATCH v3 2/2 block: fix page leak in case of merging to same page
In reply to Ming LeiDifferent iovec may use one same page, then 'pages' array filled by iov_iter_get_pages() may get reference of the same page several times. If some elements in 'pages' can be merged to same page in one bvec by bio_add_page(), bio_release_pages() only drops the page's reference once.
This way causes page leak reported by David Gibson.
This issue can be triggered since 576ed913 ("block: use bio_add_page in bio_iov_iter_get_pages").
Fixes the issue by putting the page's ref if it is merged to same page.
--- block/bio.c | 12 ++++++++++-- include/linux/bio.h | 8 ++++++++ 2 files changed, 18 insertions(+), 2 deletions(-)
block/bio.c +10 -2
@@ -652,6 +652,9 @@ static inline bool page_is_mergeable(const struct bio_vec *bv,return false;if (pfn_to_page (PFN_DOWN (vec_end_addr)) +1 != page)return false; +/* drop page ref if the page has been added and user asks to do that */ + }else if (flags &BVEC_MERGE_PUT_SAME_PAGE ) { +put_page (page); } WARN_ON_ONCE((flags & BVEC_MERGE_TO_SAME_PAGE) &&
@@ -924,8 +927,13 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)struct page *page = pages[i]; len =min_t (size_t,PAGE_SIZE - offset, left); -if (WARN_ON_ONCE (bio_add_page (bio, page, len, offset) != len)) -return -EINVAL ; + +if (!__bio_try_merge_page (bio, page, len, offset, +BVEC_MERGE_PUT_SAME_PAGE )) { +if (WARN_ON_ONCE (bio_full (bio))) +return -EINVAL ; +__bio_add_page (bio, page, len, offset); + } offset =0 ; }
include/linux/bio.h +8 -0
@@ -428,6 +428,14 @@ enum bvec_merge_flags {* bvec */ BVEC_MERGE_TO_SAME_PAGE =BIT (0 ), + +/* +* put refcount of bio's last page if the start page to add is +* same with bio's last page. If user gets refcount of every +* page added to bio before calling bio_add_page, please consider +* to use this flag for avoiding page leak +*/ + BVEC_MERGE_PUT_SAME_PAGE = BIT(1), };extern int bio_add_page (struct bio *,struct page *,unsigned int ,unsigned int );
Signature · 1 line
2.20.1