vmalloc buffers POC
Hi Dave,
this is my old vmalloc buffers POC. I just dusted it off and did a quick rebased. It survives a 4k quick xfstests run but otherwise this should be treated as not very trustworth. Maybe it will be useful for your folio work.
Diffstat:
libxfs/xfs_ialloc.c | 2 libxfs/xfs_inode_buf.c | 2 scrub/inode_repair.c | 2 xfs_buf.c | 267 ++++++++----------------------------------------- xfs_buf.h | 22 ++-- xfs_buf_item_recover.c | 7 - xfs_inode.c | 2 7 files changed, 65 insertions(+), 239 deletions(-)
PATCH 1/4 xfs: kill XBF_UNMAPPED
Unmapped buffer access is a pain, so kill it. Note that this means we now pay the vmap overhead even for inode cluster buffers, but that will be fixed by switching to large folios.
--- fs/xfs/libxfs/xfs_ialloc.c | 2 +- fs/xfs/libxfs/xfs_inode_buf.c | 2 +- fs/xfs/scrub/inode_repair.c | 2 +- fs/xfs/xfs_buf.c | 45 +---------------------------------- fs/xfs/xfs_buf.h | 16 +++++++++---- fs/xfs/xfs_buf_item_recover.c | 7 +----- fs/xfs/xfs_inode.c | 2 +- 7 files changed, 17 insertions(+), 59 deletions(-)
fs/xfs/libxfs/xfs_ialloc.c +1 -1
@@ -358,7 +358,7 @@ xfs_ialloc_inode_init( (j * M_IGEO(mp)->blocks_per_cluster)); error =xfs_trans_get_buf (tp, mp->m_ddev_targp, d, mp->m_bsize *M_IGEO (mp)->blocks_per_cluster, -XBF_UNMAPPED , &fbuf); +0 , &fbuf);if (error)return error;
fs/xfs/libxfs/xfs_inode_buf.c +1 -1
@@ -133,7 +133,7 @@ xfs_imap_to_bp( struct xfs_buf **bpp) {return xfs_trans_read_buf (mp, tp, mp->m_ddev_targp, imap->im_blkno, - imap->im_len,XBF_UNMAPPED , bpp, + imap->im_len,0 , bpp, &xfs_inode_buf_ops); }
fs/xfs/scrub/inode_repair.c +1 -1
@@ -1095,7 +1095,7 @@ xrep_dinode_core(/* Read the inode cluster buffer. */ error =xfs_trans_read_buf (sc->mp, sc->tp, sc->mp->m_ddev_targp, - ri->imap.im_blkno, ri->imap.im_len,XBF_UNMAPPED , &bp, + ri->imap.im_blkno, ri->imap.im_len,0 , &bp,NULL );if (error)return error;
fs/xfs/xfs_buf.c +1 -44
63 lines not on this page · raw message
fs/xfs/xfs_buf.h +11 -5
@@ -51,7 +51,6 @@ struct xfs_buf;#define XBF_LIVESCAN (1u << 28)#define XBF_INCORE (1u << 29)/* lookup only, return if found in cache */ #define XBF_TRYLOCK (1u << 30)/* lock requested, but do not wait */ -#define XBF_UNMAPPED (1u << 31)/* do not map the buffer */ typedef unsigned int xfs_buf_flags_t ;
@@ -74,8 +73,7 @@ typedef unsigned int xfs_buf_flags_t;/* The following interface flags should never be set */ \ { XBF_LIVESCAN, "LIVESCAN" }, \ { XBF_INCORE, "INCORE" }, \ - { XBF_TRYLOCK, "TRYLOCK" }, \ - { XBF_UNMAPPED, "UNMAPPED" } + { XBF_TRYLOCK, "TRYLOCK" }/* * Internal state flags.
@@ -308,12 +306,20 @@ extern void __xfs_buf_ioerror(struct xfs_buf *bp, int error,#define xfs_buf_ioerror (bp, err) __xfs_buf_ioerror((bp), (err), __this_address)extern void xfs_buf_ioerror_alert (struct xfs_buf *bp,xfs_failaddr_t fa);void xfs_buf_ioend_fail (struct xfs_buf *); -void xfs_buf_zero (struct xfs_buf *bp,size_t boff,size_t bsize);void __xfs_buf_mark_corrupt (struct xfs_buf *bp,xfs_failaddr_t fa);#define xfs_buf_mark_corrupt (bp) __xfs_buf_mark_corrupt((bp), __this_address)/* Buffer Utility Routines */ -extern void *xfs_buf_offset (struct xfs_buf *,size_t ); +static inline void *xfs_buf_offset (struct xfs_buf *bp,size_t offset) +{ +return bp->b_addr + offset; +} + +static inline void xfs_buf_zero (struct xfs_buf *bp,size_t boff,size_t bsize) +{ +memset (bp->b_addr + boff,0 , bsize); +} +extern void xfs_buf_stale (struct xfs_buf *bp);/* Delayed Write Buffer Routines */
fs/xfs/xfs_buf_item_recover.c +1 -6
@@ -891,7 +891,6 @@ xlog_recover_buf_commit_pass2(struct xfs_mount *mp = log->l_mp;struct xfs_buf *bp;int error; -uint buf_flags;xfs_lsn_t lsn;/*
@@ -911,12 +910,8 @@ xlog_recover_buf_commit_pass2(trace_xfs_log_recover_buf_recover (log, buf_f); - buf_flags =0 ; -if (buf_f->blf_flags &XFS_BLF_INODE_BUF ) - buf_flags |=XBF_UNMAPPED ; - error =xfs_buf_read (mp->m_ddev_targp, buf_f->blf_blkno, buf_f->blf_len, - buf_flags, &bp,NULL ); +0 , &bp,NULL );if (error)return error;
fs/xfs/xfs_inode.c +1 -1
@@ -2301,7 +2301,7 @@ xfs_ifree_cluster(*/ error =xfs_trans_get_buf (tp, mp->m_ddev_targp, blkno, mp->m_bsize * igeo->blocks_per_cluster, -XBF_UNMAPPED , &bp); +0 , &bp);if (error)return error;
Signature · 1 line
2.39.2
PATCH 2/4 xfs: map buffers in xfs_buf_alloc_pages
In reply to Christoph HellwigWith the concept of unmapped buffer gone, there is no reason to not vmap the buffer pages directly in xfs_buf_alloc_pages.
--- fs/xfs/xfs_buf.c | 37 ++++++------------------------------- 1 file changed, 6 insertions(+), 31 deletions(-)
fs/xfs/xfs_buf.c +6 -31
@@ -406,18 +406,7 @@ xfs_buf_alloc_pages(XFS_STATS_INC (bp->b_mount, xb_page_retries);memalloc_retry_wait (gfp_mask); } -return 0 ; -} -/* -* Map buffer into kernel address-space if necessary. -*/ -STATIC int -_xfs_buf_map_pages ( -struct xfs_buf *bp, -xfs_buf_flags_t flags) -{ -ASSERT (bp->b_flags & _XBF_PAGES);if (bp->b_page_count ==1 ) {/* A single page buffer is always mappable */ bp->b_addr =page_address (bp->b_pages[0 ]);
@@ -443,8 +432,13 @@ _xfs_buf_map_pages( }while (retried++ <=1 );memalloc_nofs_restore (nofs_flag); -if (!bp->b_addr) +if (!bp->b_addr) { +xfs_warn_ratelimited (bp->b_target->bt_mount, +"%s: failed to map %u pages" , __func__, + bp->b_page_count); +xfs_buf_free_pages (bp);return -ENOMEM ; + } }return 0 ;
@@ -718,18 +712,6 @@ xfs_buf_get_map(xfs_perag_put (pag); } -/* We do not hold a perag reference anymore. */ -if (!bp->b_addr) { - error =_xfs_buf_map_pages (bp, flags); -if (unlikely (error)) { -xfs_warn_ratelimited (btp->bt_mount, -"%s: failed to map %u pages" , __func__, - bp->b_page_count); -xfs_buf_relse (bp); -return error; - } - } -/* * Clear b_error if this is a lookup from a caller that doesn't expect * valid data to be found in the buffer.
@@ -959,13 +941,6 @@ xfs_buf_get_uncached(if (error) goto fail_free_buf; - error =_xfs_buf_map_pages (bp,0 ); -if (unlikely (error)) { -xfs_warn (target->bt_mount, -"%s: failed to map pages" , __func__); - goto fail_free_buf; - } -trace_xfs_buf_get_uncached (bp, _RET_IP_); *bpp = bp;return 0 ;
Signature · 1 line
2.39.2
PATCH 3/4 xfs: walk b_addr for buffer I/O
In reply to Christoph HellwigInstead of walking the page array just walk the kernel virtual address in ->b_addr. This prepares for using vmalloc for buffers and removing the b_pages array.
--- fs/xfs/xfs_buf.c | 93 ++++++++++++++---------------------------------- fs/xfs/xfs_buf.h | 2 -- 2 files changed, 26 insertions(+), 69 deletions(-)
fs/xfs/xfs_buf.c +26 -67
@@ -343,7 +343,6 @@ xfs_buf_alloc_kmem( bp->b_addr =NULL ;return -ENOMEM ; } - bp->b_offset =offset_in_page (bp->b_addr); bp->b_pages = bp->b_page_array; bp->b_pages[0 ] =kmem_to_page (bp->b_addr); bp->b_page_count =1 ;
@@ -1426,79 +1425,47 @@ xfs_buf_bio_end_io(static void xfs_buf_ioapply_map (struct xfs_buf *bp, -int map, -int *buf_offset, -int *count, +unsigned int map, +unsigned int *buf_offset,blk_opf_t op) { -int page_index; -unsigned int total_nr_pages = bp->b_page_count; -int nr_pages;struct bio *bio; -sector_t sector = bp->b_maps[map].bm_bn;int size; -int offset; - -/* skip the pages in the buffer before the start offset */ - page_index =0 ; - offset = *buf_offset; -while (offset >=PAGE_SIZE ) { - page_index++; - offset -=PAGE_SIZE ; - }/* * Limit the IO size to the length of the current vector, and update the * remaining IO count for the next time around. */ - size =min_t (int,BBTOB (bp->b_maps[map].bm_len), *count); - *count -= size; - *buf_offset += size; + size = min_t(unsigned int, BBTOB(bp->b_maps[map].bm_len), +BBTOB (bp->b_length) - *buf_offset); +if (WARN_ON_ONCE (bp->b_page_count >BIO_MAX_VECS )) { +xfs_buf_ioerror (bp, -EIO ); +return ; + } -next_chunk:atomic_inc (&bp->b_io_remaining); - nr_pages =bio_max_segs (total_nr_pages); - bio =bio_alloc (bp->b_target->bt_bdev, nr_pages, op,GFP_NOIO ); - bio->bi_iter.bi_sector = sector; + bio =bio_alloc (bp->b_target->bt_bdev, bp->b_page_count, op,GFP_NOIO ); + bio->bi_iter.bi_sector = bp->b_maps[map].bm_bn; bio->bi_end_io = xfs_buf_bio_end_io; bio->bi_private = bp; -for (; size && nr_pages; nr_pages--, page_index++) { -int rbytes, nbytes =PAGE_SIZE - offset; +do { +void *data = bp->b_addr + *buf_offset; +struct page *page =kmem_to_page (data); +unsigned int off =offset_in_page (data); +unsigned int len =min_t (unsigned, size,PAGE_SIZE - off); -if (nbytes > size) - nbytes = size; +__bio_add_page (bio, page, len, off); + size -= len; + *buf_offset += len; + }while (size); - rbytes =bio_add_page (bio, bp->b_pages[page_index], nbytes, - offset); -if (rbytes < nbytes) -break ; - - offset =0 ; - sector +=BTOBB (nbytes); - size -= nbytes; - total_nr_pages--; +if (xfs_buf_is_vmapped (bp)) { +flush_kernel_vmap_range (bp->b_addr, +xfs_buf_vmap_len (bp)); } - -if (likely (bio->bi_iter.bi_size)) { -if (xfs_buf_is_vmapped (bp)) { -flush_kernel_vmap_range (bp->b_addr, -xfs_buf_vmap_len (bp)); - } -submit_bio (bio); -if (size) - goto next_chunk; - }else { -/* -* This is guaranteed not to be the last io reference count -* because the caller (xfs_buf_submit) holds a count itself. -*/ -atomic_dec (&bp->b_io_remaining); -xfs_buf_ioerror (bp, -EIO ); -bio_put (bio); - } - +submit_bio (bio); }STATIC void
@@ -1507,8 +1474,7 @@ _xfs_buf_ioapply( {struct blk_plug plug;blk_opf_t op; -int offset; -int size; +unsigned int offset =0 ;int i;/*
@@ -1564,16 +1530,9 @@ _xfs_buf_ioapply(* _xfs_buf_ioapply_vec() will modify them appropriately for each * subsequent call. */ - offset = bp->b_offset; - size =BBTOB (bp->b_length);blk_start_plug (&plug); -for (i =0 ; i < bp->b_map_count; i++) { -xfs_buf_ioapply_map (bp, i, &offset, &size, op); -if (bp->b_error) -break ; -if (size <=0 ) -break ;/* all done */ - } +for (i =0 ; i < bp->b_map_count; i++) +xfs_buf_ioapply_map (bp, i, &offset, op);blk_finish_plug (&plug); }
fs/xfs/xfs_buf.h +0 -2
@@ -186,8 +186,6 @@ struct xfs_buf {atomic_t b_pin_count;/* pin count */ atomic_t b_io_remaining;/* #outstanding I/O requests */ unsigned int b_page_count;/* size of page array */ -unsigned int b_offset;/* page offset of b_addr, -only for _XBF_KMEM buffers */ int b_error;/* error code on I/O */ /*
Signature · 1 line
2.39.2
PATCH 4/4 xfs: use vmalloc for multi-page buffers
In reply to Christoph HellwigInstead of allocating the pages manually using the bulk page allocator and then using vm_map_page just use vmalloc to allocate the entire buffer - vmalloc will use the bulk allocator internally if it fits.
With this the b_pages array can go away as well as nothing uses it.
--- fs/xfs/xfs_buf.c | 96 +++++++++--------------------------------------- fs/xfs/xfs_buf.h | 4 -- 2 files changed, 18 insertions(+), 82 deletions(-)
fs/xfs/xfs_buf.c +18 -78
@@ -197,7 +197,7 @@ xfs_buf_get_maps( }/* -* Frees b_pages if it was allocated. +* Frees b_maps if it was allocated. */ static void xfs_buf_free_maps(
@@ -271,29 +271,6 @@ _xfs_buf_alloc(return 0 ; } -static void -xfs_buf_free_pages ( -struct xfs_buf *bp) -{ -uint i; - -ASSERT (bp->b_flags & _XBF_PAGES); - -if (xfs_buf_is_vmapped (bp)) -vm_unmap_ram (bp->b_addr, bp->b_page_count); - -for (i =0 ; i < bp->b_page_count; i++) { -if (bp->b_pages[i]) -__free_page (bp->b_pages[i]); - } -mm_account_reclaimed_pages (bp->b_page_count); - -if (bp->b_pages != bp->b_page_array) -kmem_free (bp->b_pages); - bp->b_pages =NULL ; - bp->b_flags &= ~_XBF_PAGES; -} -static void xfs_buf_free_callback (struct callback_head *cb)
@@ -312,10 +289,15 @@ xfs_buf_free(ASSERT (list_empty (&bp->b_lru)); +if (!(bp->b_flags & _XBF_KMEM)) +mm_account_reclaimed_pages (bp->b_page_count); +if (bp->b_flags & _XBF_PAGES) -xfs_buf_free_pages (bp); -else if (bp->b_flags & _XBF_KMEM) -kmem_free (bp->b_addr); +put_page (virt_to_page (bp->b_addr)); +else +kvfree (bp->b_addr); + + bp->b_flags &= _XBF_KMEM | _XBF_PAGES;call_rcu (&bp->b_rcu, xfs_buf_free_callback); }
@@ -343,8 +325,6 @@ xfs_buf_alloc_kmem( bp->b_addr =NULL ;return -ENOMEM ; } - bp->b_pages = bp->b_page_array; - bp->b_pages[0 ] =kmem_to_page (bp->b_addr); bp->b_page_count =1 ; bp->b_flags |= _XBF_KMEM;return 0 ;
@@ -356,7 +336,6 @@ xfs_buf_alloc_pages( xfs_buf_flags_t flags) {gfp_t gfp_mask = __GFP_NOWARN; -long filled =0 ;if (flags &XBF_READ_AHEAD ) gfp_mask |= __GFP_NORETRY;
@@ -365,56 +344,24 @@ xfs_buf_alloc_pages(/* Make sure that we have a page list */ bp->b_page_count =DIV_ROUND_UP (BBTOB (bp->b_length),PAGE_SIZE ); -if (bp->b_page_count <=XB_PAGES ) { - bp->b_pages = bp->b_page_array; - }else { - bp->b_pages =kzalloc (sizeof (struct page *) * bp->b_page_count, - gfp_mask); -if (!bp->b_pages) -return -ENOMEM ; - } - bp->b_flags |= _XBF_PAGES;/* Assure zeroed buffer for non-read cases. */ if (!(flags &XBF_READ )) gfp_mask |= __GFP_ZERO; -/* -* Bulk filling of pages can take multiple calls. Not filling the entire -* array is not an allocation failure, so don't back off if we get at -* least one extra page. -*/ -for (;;) { -long last = filled; - - filled =alloc_pages_bulk_array (gfp_mask, bp->b_page_count, - bp->b_pages); -if (filled == bp->b_page_count) { -XFS_STATS_INC (bp->b_mount, xb_page_found); -break ; - } - -if (filled != last) -continue ; +if (bp->b_page_count ==1 ) { +struct page *page; -if (flags &XBF_READ_AHEAD ) { -xfs_buf_free_pages (bp); + page =alloc_page (gfp_mask); +if (!page)return -ENOMEM ; - } - -XFS_STATS_INC (bp->b_mount, xb_page_retries); -memalloc_retry_wait (gfp_mask); - } - -if (bp->b_page_count ==1 ) { -/* A single page buffer is always mappable */ - bp->b_addr =page_address (bp->b_pages[0 ]); + bp->b_addr =page_address (page); + bp->b_flags |= _XBF_PAGES; }else { -int retried =0 ;unsigned nofs_flag;/* -* vm_map_ram() will allocate auxiliary structures (e.g. +* vmalloc() will allocate auxiliary structures (e.g. * pagetables) with GFP_KERNEL, yet we are likely to be under * GFP_NOFS context here. Hence we need to tell memory reclaim * that we are in such a context via PF_MEMALLOC_NOFS to prevent
@@ -422,20 +369,13 @@ xfs_buf_alloc_pages(* potentially deadlocking. */ nofs_flag =memalloc_nofs_save (); -do { - bp->b_addr =vm_map_ram (bp->b_pages, bp->b_page_count, --1 ); -if (bp->b_addr) -break ; -vm_unmap_aliases (); - }while (retried++ <=1 ); + bp->b_addr =__vmalloc (BBTOB (bp->b_length), gfp_mask);memalloc_nofs_restore (nofs_flag);if (!bp->b_addr) {xfs_warn_ratelimited (bp->b_target->bt_mount, -"%s: failed to map %u pages" , __func__, +"%s: failed to allocate %u pages" , __func__, bp->b_page_count); -xfs_buf_free_pages (bp);return -ENOMEM ; } }
fs/xfs/xfs_buf.h +0 -4
@@ -114,8 +114,6 @@ typedef struct xfs_buftarg {struct ratelimit_state bt_ioerror_rl; } xfs_buftarg_t; -#define XB_PAGES 2 -struct xfs_buf_map {xfs_daddr_t bm_bn;/* block number for I/O */ int bm_len;/* size of I/O */
@@ -178,8 +176,6 @@ struct xfs_buf {struct xfs_buf_log_item *b_log_item;struct list_head b_li_list;/* Log items list head */ struct xfs_trans *b_transp; -struct page **b_pages;/* array of page pointers */ -struct page *b_page_array[XB_PAGES ];/* inline pages */ struct xfs_buf_map *b_maps;/* compound buffer map */ struct xfs_buf_map __b_map;/* inline compound buffer map */ int b_map_count;
Signature · 1 line
2.39.2