RFC: buffer cache backing page allocation cleanup
Hi all,
while reviewing the patch from Dave to use alloc_pages_bulk_array I stumbled over all kinds of lose end in the buffer cache, and I started cleaning them up while procrastinating and avoiding more urgent work, and rebased the alloc_pages_bulk_array patch on top.
Only run through a quick group xfstests run on a 4k file system so far.
Diffstat:
libxfs/xfs_ag.c | 1 xfs_buf.c | 256 +++++++++++++++++++++----------------------------------- xfs_buf.h | 3 3 files changed, 99 insertions(+), 161 deletions(-)
PATCH 1/11 xfs: cleanup error handling in xfs_buf_get_map
Use a single goto label for freeing the buffer and returning an error.
--- fs/xfs/xfs_buf.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-)
fs/xfs/xfs_buf.c +7 -8
@@ -721,16 +721,12 @@ xfs_buf_get_map(return error; error =xfs_buf_allocate_memory (new_bp, flags); -if (error) { -xfs_buf_free (new_bp); -return error; - } +if (error) + goto out_free_buf; error =xfs_buf_find (target, map, nmaps, flags, new_bp, &bp); -if (error) { -xfs_buf_free (new_bp); -return error; - } +if (error) + goto out_free_buf;if (bp != new_bp)xfs_buf_free (new_bp);
@@ -758,6 +754,9 @@ xfs_buf_get_map(trace_xfs_buf_get (bp, flags, _RET_IP_); *bpp = bp;return 0 ; +out_free_buf: +xfs_buf_free (new_bp); +return error; }int
Signature · 1 line
2.30.2
Re: [PATCH 01/11] xfs: cleanup error handling in xfs_buf_get_map
Quoted 3 lines · Christoph Hellwig
Use a single goto label for freeing the buffer and returning an error.
Signed-off-by Christoph Hellwig <[email protected]>
Looks ok,
--D
Quoted 42 lines
--- fs/xfs/xfs_buf.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-)fs/xfs/xfs_buf.c +7
-8@@ -721,16 +721,12 @@ xfs_buf_get_map(return error; error =xfs_buf_allocate_memory (new_bp, flags); -if (error) { -xfs_buf_free (new_bp); -return error; - } +if (error) + goto out_free_buf; error =xfs_buf_find (target, map, nmaps, flags, new_bp, &bp); -if (error) { -xfs_buf_free (new_bp); -return error; - } +if (error) + goto out_free_buf;if (bp != new_bp)xfs_buf_free (new_bp);@@ -758,6 +754,9 @@ xfs_buf_get_map(trace_xfs_buf_get (bp, flags, _RET_IP_); *bpp = bp;return 0 ; +out_free_buf: +xfs_buf_free (new_bp); +return error; }int Signature · 1 line
2.30.2
PATCH 2/11 xfs: split xfs_buf_allocate_memory
In reply to Christoph HellwigSplit xfs_buf_allocate_memory into one helper that allocates from slab and one that allocates using the page allocator.
--- fs/xfs/xfs_buf.c | 83 +++++++++++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 39 deletions(-)
fs/xfs/xfs_buf.c +44 -39
@@ -347,11 +347,41 @@ xfs_buf_free(kmem_cache_free (xfs_buf_zone, bp); } +static int +xfs_buf_alloc_slab ( +struct xfs_buf *bp, +unsigned int flags) +{ +struct xfs_buftarg *btp = bp->b_target; +int align =xfs_buftarg_dma_alignment (btp); +size_t size =BBTOB (bp->b_length); +xfs_km_flags_t km_flags =KM_ZERO ; + +if (!(flags &XBF_READ )) + km_flags |=KM_ZERO ; + bp->b_addr =kmem_alloc_io (size, align, km_flags); +if (!bp->b_addr) +return -ENOMEM ; +if (((unsigned long )(bp->b_addr + size -1 ) &PAGE_MASK ) != + ((unsigned long )bp->b_addr &PAGE_MASK )) { +/* b_addr spans two pages - use alloc_page instead */ +kmem_free (bp->b_addr); + bp->b_addr =NULL ; +return -ENOMEM ; + } + bp->b_offset =offset_in_page (bp->b_addr); + bp->b_pages = bp->b_page_array; + bp->b_pages[0 ] =kmem_to_page (bp->b_addr); + bp->b_page_count =1 ; + bp->b_flags |= _XBF_KMEM; +return 0 ; +} +/* * Allocates all the pages for buffer in question and builds it's page list. */ -STATIC int -xfs_buf_allocate_memory ( +static int +xfs_buf_alloc_pages (struct xfs_buf *bp,uint flags) {
@@ -361,47 +391,14 @@ xfs_buf_allocate_memory(unsigned short page_count, i;xfs_off_t start, end;int error; -xfs_km_flags_t kmflag_mask =0 ;/* * assure zeroed buffer for non-read cases. */ -if (!(flags &XBF_READ )) { - kmflag_mask |=KM_ZERO ; +if (!(flags &XBF_READ )) gfp_mask |= __GFP_ZERO; - } -/* -* for buffers that are contained within a single page, just allocate -* the memory from the heap - there's no need for the complexity of -* page arrays to keep allocation down to order 0. -*/ size =BBTOB (bp->b_length); -if (size <PAGE_SIZE ) { -int align_mask =xfs_buftarg_dma_alignment (bp->b_target); - bp->b_addr =kmem_alloc_io (size, align_mask, -KM_NOFS | kmflag_mask); -if (!bp->b_addr) { -/* low memory - use alloc_page loop instead */ - goto use_alloc_page; - } - -if (((unsigned long )(bp->b_addr + size -1 ) &PAGE_MASK ) != - ((unsigned long )bp->b_addr &PAGE_MASK )) { -/* b_addr spans two pages - use alloc_page instead */ -kmem_free (bp->b_addr); - bp->b_addr =NULL ; - goto use_alloc_page; - } - bp->b_offset =offset_in_page (bp->b_addr); - bp->b_pages = bp->b_page_array; - bp->b_pages[0 ] =kmem_to_page (bp->b_addr); - bp->b_page_count =1 ; - bp->b_flags |= _XBF_KMEM; -return 0 ; - } - -use_alloc_page: start =BBTOB (bp->b_maps[0 ].bm_bn) >>PAGE_SHIFT ; end = (BBTOB (bp->b_maps[0 ].bm_bn + bp->b_length) +PAGE_SIZE -1 ) >>PAGE_SHIFT ;
@@ -720,9 +717,17 @@ xfs_buf_get_map(if (error)return error; - error =xfs_buf_allocate_memory (new_bp, flags); -if (error) - goto out_free_buf; +/* +* For buffers that are contained within a single page, just allocate +* the memory from the heap - there's no need for the complexity of +* page arrays to keep allocation down to order 0. +*/ +if (BBTOB (new_bp->b_length) >=PAGE_SIZE || +xfs_buf_alloc_slab (new_bp, flags) <0 ) { + error =xfs_buf_alloc_pages (new_bp, flags); +if (error) + goto out_free_buf; + } error =xfs_buf_find (target, map, nmaps, flags, new_bp, &bp);if (error)
Signature · 1 line
2.30.2
Re: [PATCH 02/11] xfs: split xfs_buf_allocate_memory
Quoted 4 lines · Christoph Hellwig
Split
xfs_buf_allocate_memoryinto one helper that allocates from slab and one that allocates using the page allocator.Signed-off-by Christoph Hellwig <[email protected]>
....
fs/xfs/xfs_buf.c xfs_buf_free( · line 354
Quoted 5 lines
fs/xfs/xfs_buf.c +5
-0+static int +xfs_buf_alloc_slab ( +struct xfs_buf *bp, +unsigned int flags) +{
xfs_buf_alloc_kmem() or xfs_buf_alloc_heap() would be better, I think, because it matches the flag used to indicate how the memory associated with the buffer was allocated.
Quoted 8 lines
fs/xfs/xfs_buf.c +1
-3@@ -720,9 +717,17 @@ xfs_buf_get_map(if (error)return error; - error =xfs_buf_allocate_memory (new_bp, flags); -if (error) - goto out_free_buf; +/*
fs/xfs/xfs_buf.c xfs_buf_get_map( · line 730
Quoted 10 lines
fs/xfs/xfs_buf.c +10
-0+* For buffers that are contained within a single page, just allocate +* the memory from the heap - there's no need for the complexity of +* page arrays to keep allocation down to order 0. +*/ +if (BBTOB (new_bp->b_length) >=PAGE_SIZE || +xfs_buf_alloc_slab (new_bp, flags) <0 ) { + error =xfs_buf_alloc_pages (new_bp, flags); +if (error) + goto out_free_buf; + }
Took me a moment to grok the logic pattern here, then I realised the comment didn't help as it makes no indication that the heap allocation is best effort and will fall back to pages. A small tweak like:
/*
* For buffers that fit entirely within a single page, first
* attempt to allocate the memory from the heap to minimise
* memory usage. If we can't get heap memory for these small
* buffers, we fall back to using the page allocator. */
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
PATCH 3/11 xfs: remove ->b_offset handling for page backed buffers
In reply to Christoph Hellwig->b_offset can only be non-zero for SLAB backed buffers, so remove all code dealing with it for page backed buffers.
--- fs/xfs/xfs_buf.c | 15 +++++---------- fs/xfs/xfs_buf.h | 3 ++- 2 files changed, 7 insertions(+), 11 deletions(-)
fs/xfs/xfs_buf.c +5 -10
@@ -79,7 +79,7 @@ static inline intxfs_buf_vmap_len ( struct xfs_buf *bp) { -return (bp->b_page_count *PAGE_SIZE ) - bp->b_offset; +return (bp->b_page_count *PAGE_SIZE ); }/*
@@ -329,8 +329,7 @@ xfs_buf_free(uint i;if (xfs_buf_is_vmapped (bp)) -vm_unmap_ram (bp->b_addr - bp->b_offset, - bp->b_page_count); +vm_unmap_ram (bp->b_addr, bp->b_page_count);for (i =0 ; i < bp->b_page_count; i++) {struct page *page = bp->b_pages[i];
@@ -386,7 +385,7 @@ xfs_buf_alloc_pages( uint flags) {size_t size; -size_t nbytes, offset; +size_t nbytes;gfp_t gfp_mask =xb_to_gfp (flags);unsigned short page_count, i;xfs_off_t start, end;
@@ -407,7 +406,6 @@ xfs_buf_alloc_pages(if (unlikely (error))return error; - offset = bp->b_offset; bp->b_flags |= _XBF_PAGES;for (i =0 ; i < bp->b_page_count; i++) {
@@ -441,10 +439,9 @@ xfs_buf_alloc_pages(XFS_STATS_INC (bp->b_mount, xb_page_found); - nbytes =min_t (size_t, size,PAGE_SIZE - offset); + nbytes =min_t (size_t, size,PAGE_SIZE ); size -= nbytes; bp->b_pages[i] = page; - offset =0 ; }return 0 ;
@@ -466,7 +463,7 @@ _xfs_buf_map_pages(ASSERT (bp->b_flags & _XBF_PAGES);if (bp->b_page_count ==1 ) {/* A single page buffer is always mappable */ - bp->b_addr =page_address (bp->b_pages[0 ]) + bp->b_offset; + bp->b_addr =page_address (bp->b_pages[0 ]); }else if (flags &XBF_UNMAPPED ) { bp->b_addr =NULL ; }else {
@@ -493,7 +490,6 @@ _xfs_buf_map_pages(if (!bp->b_addr)return -ENOMEM ; - bp->b_addr += bp->b_offset; }return 0 ;
@@ -1726,7 +1722,6 @@ xfs_buf_offset(if (bp->b_addr)return bp->b_addr + offset; - offset += bp->b_offset; page = bp->b_pages[offset >>PAGE_SHIFT ];return page_address (page) + (offset & (PAGE_SIZE -1 )); }
fs/xfs/xfs_buf.h +2 -1
@@ -167,7 +167,8 @@ struct xfs_buf {atomic_t b_pin_count;/* pin count */ atomic_t b_io_remaining;/* #outstanding I/O requests */ unsigned int b_page_count;/* size of page array */ -unsigned int b_offset;/* page offset in first page */ +unsigned int b_offset;/* page offset in first page, +only used for SLAB buffers */ int b_error;/* error code on I/O */ /*
Signature · 1 line
2.30.2
Re: [PATCH 03/11] xfs: remove ->b_offset handling for page backed buffers
Quoted 2 lines · Christoph Hellwig
->b_offset can only be non-zero for SLAB backed buffers, so remove all code dealing with it for page backed buffers.
Can you refer to these as _XBF_KMEM buffers, not "SLAB backed"? That way there is no confusion as to what type of buffer needs to pay attention to b_offset...
fs/xfs/xfs_buf.h struct xfs_buf { · line 171
Quoted 11 lines
fs/xfs/xfs_buf.h +2
-1@@ -167,7 +167,8 @@ struct xfs_buf {atomic_t b_pin_count;/* pin count */ atomic_t b_io_remaining;/* #outstanding I/O requests */ unsigned int b_page_count;/* size of page array */ -unsigned int b_offset;/* page offset in first page */ +unsigned int b_offset;/* page offset in first page, +only used for SLAB buffers */
Here too.
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
PATCH 4/11 xfs: cleanup _xfs_buf_get_pages
In reply to Christoph HellwigRemove the check for an existing b_pages array as this function is always called right after allocating a buffer, so this can't happen. Also use kmem_zalloc to allocate the page array instead of doing a manual memset gіven that the inline array is already pre-zeroed as part of the freshly allocated buffer anyway.
--- fs/xfs/xfs_buf.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-)
fs/xfs/xfs_buf.c +11 -12
@@ -281,19 +281,18 @@ _xfs_buf_get_pages( struct xfs_buf *bp, int page_count) { -/* Make sure that we have a page list */ -if (bp->b_pages ==NULL ) { - bp->b_page_count = page_count; -if (page_count <=XB_PAGES ) { - bp->b_pages = bp->b_page_array; - }else { - bp->b_pages =kmem_alloc (sizeof (struct page *) * - page_count,KM_NOFS ); -if (bp->b_pages ==NULL ) -return -ENOMEM ; - } -memset (bp->b_pages,0 ,sizeof (struct page *) * page_count); +ASSERT (bp->b_pages ==NULL ); + + bp->b_page_count = page_count; +if (page_count >XB_PAGES ) { + bp->b_pages =kmem_zalloc (sizeof (struct page *) * page_count, +KM_NOFS ); +if (!bp->b_pages) +return -ENOMEM ; + }else { + bp->b_pages = bp->b_page_array; } +return 0 ; }
Signature · 1 line
2.30.2
Re: [PATCH 04/11] xfs: cleanup _xfs_buf_get_pages
Quoted 35 lines · Christoph Hellwig
Remove the check for an existing
b_pagesarray as this function is always called right after allocating a buffer, so this can't happen. Also usekmem_zallocto allocate the page array instead of doing a manual memset gіven that the inline array is already pre-zeroed as part of the freshly allocated buffer anyway.Signed-off-by Christoph Hellwig <[email protected]>--- fs/xfs/xfs_buf.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-)fs/xfs/xfs_buf.c +4
-12@@ -281,19 +281,18 @@ _xfs_buf_get_pages( struct xfs_buf *bp, int page_count) { -/* Make sure that we have a page list */ -if (bp->b_pages ==NULL ) { - bp->b_page_count = page_count; -if (page_count <=XB_PAGES ) { - bp->b_pages = bp->b_page_array; - }else { - bp->b_pages =kmem_alloc (sizeof (struct page *) * - page_count,KM_NOFS ); -if (bp->b_pages ==NULL ) -return -ENOMEM ; - } -memset (bp->b_pages,0 ,sizeof (struct page *) * page_count); +ASSERT (bp->b_pages ==NULL ); + + bp->b_page_count = page_count; +if (page_count >XB_PAGES ) {
fs/xfs/xfs_buf.c _xfs_buf_get_pages( · line 297
Quoted 10 lines
fs/xfs/xfs_buf.c +7
-0+ bp->b_pages =kmem_zalloc (sizeof (struct page *) * page_count, +KM_NOFS ); +if (!bp->b_pages) +return -ENOMEM ; + }else { + bp->b_pages = bp->b_page_array; } +return 0 ; }
This will not apply (and break) the bulk alloc patch I sent out - we have to ensure that the b_pages array is always zeroed before we call the bulk alloc function, hence I moved the memset() in this function to be unconditional. I almost cleaned up this function in that patchset....
Just doing this:
bp->b_page_count = page_count;
if (page_count > XB_PAGES) {
bp->b_pages = kmem_alloc(sizeof(struct page *) * page_count,
KM_NOFS);
if (!bp->b_pages)
return -ENOMEM;
} else {
bp->b_pages = bp->b_page_array;
}
memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
return 0;will make it work fine with bulk alloc.
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
Re: [PATCH 04/11] xfs: cleanup _xfs_buf_get_pages
Quoted 5 lines · Dave Chinner
This will not apply (and break) the bulk alloc patch I sent out - we have to ensure that the
b_pagesarray is always zeroed before we call the bulk alloc function, hence I moved thememset()in this function to be unconditional. I almost cleaned up this function in that patchset....
The buffer is freshly allocated here using kmem_cache_zalloc, so b_pages can't be set, b_page_array is already zeroed from kmem_cache_zalloc, and the separate b_pages allocation is swithced to use kmem_zalloc. I thought the commit log covers this, but maybe I need to improve it?
Re: [PATCH 04/11] xfs: cleanup _xfs_buf_get_pages
Quoted 7 lines · Christoph Hellwig
Quoted 5 lines · Dave Chinner
This will not apply (and break) the bulk alloc patch I sent out - we have to ensure that the
b_pagesarray is always zeroed before we call the bulk alloc function, hence I moved thememset()in this function to be unconditional. I almost cleaned up this function in that patchset....
Quoted 5 lines
The buffer is freshly allocated here using
kmem_cache_zalloc, sob_pagescan't be set,b_page_arrayis already zeroed fromkmem_cache_zalloc, and the separateb_pagesallocation is swithced to usekmem_zalloc. I thought the commit log covers this, but maybe I need to improve it?
I think I'm still living in the past a bit, where the page array in an active uncached buffer could change via the old "associate memory" interface. We still actually have that interface in userspace, but we don't have anything in the kernel that uses it any more.
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
PATCH 5/11 xfs: remove the xb_page_found stat counter in xfs_buf_alloc_pages
In reply to Christoph HellwigWe did not find any page, we're allocating them all from the page allocator.
--- fs/xfs/xfs_buf.c | 2 -- 1 file changed, 2 deletions(-)
fs/xfs/xfs_buf.c +0 -2
@@ -436,8 +436,6 @@ xfs_buf_alloc_pages( goto retry; } -XFS_STATS_INC (bp->b_mount, xb_page_found); - nbytes =min_t (size_t, size,PAGE_SIZE ); size -= nbytes; bp->b_pages[i] = page;
Signature · 1 line
2.30.2
Re: [PATCH 05/11] xfs: remove the xb_page_found stat counter in xfs_buf_alloc_pages
Quoted 13 lines · Christoph Hellwig
We did not find any page, we're allocating them all from the page allocator.
Signed-off-by Christoph Hellwig <[email protected]>--- fs/xfs/xfs_buf.c | 2 -- 1 file changed, 2 deletions(-)fs/xfs/xfs_buf.c +0
-0
fs/xfs/xfs_buf.c xfs_buf_alloc_pages( · line 441
Quoted 8 lines
fs/xfs/xfs_buf.c +0
-2@@ -436,8 +436,6 @@ xfs_buf_alloc_pages( goto retry; } -XFS_STATS_INC (bp->b_mount, xb_page_found); - nbytes =min_t (size_t, size,PAGE_SIZE ); size -= nbytes; bp->b_pages[i] = page;
NACK. This is actually telling us that a page was allocated successfully. I just used this very stat in combination with the page allocate failure stat (xb_page_retries) to determine that the bulk alloc code was failing to allocate all the pages asked for at least 20% of the time.
$ xfs_stats.pl
.....
xs_ig_dup............ 0 Buf Statistics
xs_ig_reclaims....... 38377759 pb_get................ 432887411
xs_ig_attrchg........ 1 pb_create............. 1839653
Log Operations pb_get_locked......... 431047794
xs_log_writes........ 71572 pb_get_locked_waited.. 346
xs_log_blocks........ 36644864 pb_busy_locked........ 13615
xs_log_noiclogs...... 265 pb_miss_locked........ 1839651
xs_log_force......... 521 pb_page_retries....... 488537
xs_log_force_sleep... 495 pb_page_found......... 1839431
pb_get_read........... 577See the pb_miss_locked, pb_page_found and pb_page_retries numbers? Almost all cache misses required page (rather than heap) allocation, and 25% of them bulk allocation failed to allocate all pages in a single call.
So, yeah, the buffer cache stats are useful diagnostic information that I use a lot...
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
PATCH 6/11 xfs: remove the size and nbytes variables in xfs_buf_alloc_pages
In reply to Christoph HellwigThese variables are not used for anything but recursively updating each other.
--- fs/xfs/xfs_buf.c | 5 ----- 1 file changed, 5 deletions(-)
fs/xfs/xfs_buf.c +0 -5
@@ -383,8 +383,6 @@ xfs_buf_alloc_pages( struct xfs_buf *bp, uint flags) { - size_t size; -size_t nbytes;gfp_t gfp_mask =xb_to_gfp (flags);unsigned short page_count, i;xfs_off_t start, end;
@@ -396,7 +394,6 @@ xfs_buf_alloc_pages(if (!(flags &XBF_READ )) gfp_mask |= __GFP_ZERO; - size =BBTOB (bp->b_length); start =BBTOB (bp->b_maps[0 ].bm_bn) >>PAGE_SHIFT ; end = (BBTOB (bp->b_maps[0 ].bm_bn + bp->b_length) +PAGE_SIZE -1 ) >>PAGE_SHIFT ;
@@ -436,8 +433,6 @@ xfs_buf_alloc_pages( goto retry; } - nbytes =min_t (size_t, size,PAGE_SIZE ); - size -= nbytes; bp->b_pages[i] = page; }return 0 ;
Signature · 1 line
2.30.2
Re: [PATCH 06/11] xfs: remove the size and nbytes variables in xfs_buf_alloc_pages
Quoted 30 lines · Christoph Hellwig
These variables are not used for anything but recursively updating each other.
Signed-off-by Christoph Hellwig <[email protected]>--- fs/xfs/xfs_buf.c | 5 ----- 1 file changed, 5 deletions(-)fs/xfs/xfs_buf.c +0
-3@@ -383,8 +383,6 @@ xfs_buf_alloc_pages( struct xfs_buf *bp, uint flags) { - size_t size; -size_t nbytes;gfp_t gfp_mask =xb_to_gfp (flags);unsigned short page_count, i;xfs_off_t start, end;@@ -396,7 +394,6 @@ xfs_buf_alloc_pages(if (!(flags &XBF_READ )) gfp_mask |= __GFP_ZERO; - size =BBTOB (bp->b_length); start =BBTOB (bp->b_maps[0 ].bm_bn) >>PAGE_SHIFT ; end = (BBTOB (bp->b_maps[0 ].bm_bn + bp->b_length) +PAGE_SIZE -1 ) >>PAGE_SHIFT ;
fs/xfs/xfs_buf.c xfs_buf_alloc_pages( · line 438
Quoted 8 lines
fs/xfs/xfs_buf.c +0
-2@@ -436,8 +433,6 @@ xfs_buf_alloc_pages( goto retry; } - nbytes =min_t (size_t, size,PAGE_SIZE ); - size -= nbytes; bp->b_pages[i] = page; }return 0 ;
These have already gone away with the bulk allocation patch. I think you should rebase this series on top of that...
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
PATCH 7/11 xfs: simplify the b_page_count calculation
In reply to Christoph HellwigEver since we stopped using the Linux page cache to back XFS buffes there is no need to take the start sector into account for calculating the number of pages in a buffer, as the data always start from the beginning of the buffer.
--- fs/xfs/xfs_buf.c | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-)
fs/xfs/xfs_buf.c +9 -17
@@ -278,15 +278,14 @@ _xfs_buf_alloc(*/ STATIC int_xfs_buf_get_pages ( -struct xfs_buf *bp, -int page_count) +struct xfs_buf *bp) {ASSERT (bp->b_pages ==NULL ); - bp->b_page_count = page_count; -if (page_count >XB_PAGES ) { - bp->b_pages =kmem_zalloc (sizeof (struct page *) * page_count, -KM_NOFS ); + bp->b_page_count =DIV_ROUND_UP (BBTOB (bp->b_length),PAGE_SIZE ); +if (bp->b_page_count >XB_PAGES ) { + bp->b_pages =kmem_zalloc (sizeof (struct page *) * + bp->b_page_count,KM_NOFS );if (!bp->b_pages)return -ENOMEM ; }else {
@@ -384,8 +383,7 @@ xfs_buf_alloc_pages( uint flags) {gfp_t gfp_mask =xb_to_gfp (flags); -unsigned short page_count, i; -xfs_off_t start, end; +unsigned short i;int error;/*
@@ -394,11 +392,7 @@ xfs_buf_alloc_pages(if (!(flags &XBF_READ )) gfp_mask |= __GFP_ZERO; - start =BBTOB (bp->b_maps[0 ].bm_bn) >>PAGE_SHIFT ; - end = (BBTOB (bp->b_maps[0 ].bm_bn + bp->b_length) +PAGE_SIZE -1 ) - >>PAGE_SHIFT ; - page_count = end - start; - error =_xfs_buf_get_pages (bp, page_count); + error =_xfs_buf_get_pages (bp);if (unlikely (error))return error;
@@ -942,7 +936,6 @@ xfs_buf_get_uncached(int flags, struct xfs_buf **bpp) { - unsigned long page_count; int error, i;struct xfs_buf *bp;DEFINE_SINGLE_BUF_MAP (map,XFS_BUF_DADDR_NULL , numblks);
@@ -954,12 +947,11 @@ xfs_buf_get_uncached(if (error) goto fail; - page_count =PAGE_ALIGN (numblks <<BBSHIFT ) >>PAGE_SHIFT ; - error =_xfs_buf_get_pages (bp, page_count); + error =_xfs_buf_get_pages (bp);if (error) goto fail_free_buf; -for (i =0 ; i < page_count; i++) { +for (i =0 ; i < bp->b_page_count; i++) { bp->b_pages[i] =alloc_page (xb_to_gfp (flags));if (!bp->b_pages[i]) { error = -ENOMEM ;
Signature · 1 line
2.30.2
PATCH 8/11 xfs: centralize page allocation and freeing for buffers
In reply to Christoph HellwigFactor out two helpers that do everything needed for allocating and freeing pages that back a buffer, and remove the duplication between the different interfaces.
--- fs/xfs/xfs_buf.c | 110 ++++++++++++++++------------------------------- 1 file changed, 37 insertions(+), 73 deletions(-)
fs/xfs/xfs_buf.c +37 -73
@@ -273,35 +273,17 @@ _xfs_buf_alloc( }/* -* Allocate a page array capable of holding a specified number -* of pages, and point the page buf at it. +* Free all pages allocated to the buffer including the page map. */ -STATIC int -_xfs_buf_get_pages ( -struct xfs_buf *bp) +static void +xfs_buf_free_pages ( +struct xfs_buf *bp) { -ASSERT (bp->b_pages ==NULL ); - - bp->b_page_count =DIV_ROUND_UP (BBTOB (bp->b_length),PAGE_SIZE ); -if (bp->b_page_count >XB_PAGES ) { - bp->b_pages =kmem_zalloc (sizeof (struct page *) * - bp->b_page_count,KM_NOFS ); -if (!bp->b_pages) -return -ENOMEM ; - }else { - bp->b_pages = bp->b_page_array; - } +unsigned int i; -return 0 ; -} +for (i =0 ; i < bp->b_page_count; i++) +__free_page (bp->b_pages[i]); -/* -* Frees b_pages if it was allocated. -*/ -STATIC void -_xfs_buf_free_pages ( -struct xfs_buf *bp) -{if (bp->b_pages != bp->b_page_array) {kmem_free (bp->b_pages); bp->b_pages =NULL ;
@@ -324,22 +306,14 @@ xfs_buf_free(ASSERT (list_empty (&bp->b_lru));if (bp->b_flags & _XBF_PAGES) { -uint i; -if (xfs_buf_is_vmapped (bp))vm_unmap_ram (bp->b_addr, bp->b_page_count); - -for (i =0 ; i < bp->b_page_count; i++) { -struct page *page = bp->b_pages[i]; - -__free_page (page); - } +xfs_buf_free_pages (bp);if (current->reclaim_state) current->reclaim_state->reclaimed_slab += bp->b_page_count; }else if (bp->b_flags & _XBF_KMEM)kmem_free (bp->b_addr); -_xfs_buf_free_pages (bp);xfs_buf_free_maps (bp);kmem_cache_free (xfs_buf_zone, bp); }
@@ -380,34 +354,33 @@ xfs_buf_alloc_slab(static int xfs_buf_alloc_pages (struct xfs_buf *bp, -uint flags) +gfp_t gfp_mask, +bool fail_fast) { -gfp_t gfp_mask =xb_to_gfp (flags); -unsigned short i; -int error; - -/* -* assure zeroed buffer for non-read cases. -*/ -if (!(flags &XBF_READ )) - gfp_mask |= __GFP_ZERO; +int i; - error =_xfs_buf_get_pages (bp); -if (unlikely (error)) -return error; +ASSERT (bp->b_pages ==NULL ); - bp->b_flags |= _XBF_PAGES; + bp->b_page_count =DIV_ROUND_UP (BBTOB (bp->b_length),PAGE_SIZE ); +if (bp->b_page_count >XB_PAGES ) { + bp->b_pages =kmem_zalloc (sizeof (struct page *) * + bp->b_page_count,KM_NOFS ); +if (!bp->b_pages) +return -ENOMEM ; + }else { + bp->b_pages = bp->b_page_array; + }for (i =0 ; i < bp->b_page_count; i++) {struct page *page;uint retries =0 ; retry: page =alloc_page (gfp_mask); -if (unlikely (page ==NULL )) { -if (flags &XBF_READ_AHEAD ) { +if (unlikely (!page)) { +if (fail_fast) { bp->b_page_count = i; - error = -ENOMEM ; - goto out_free_pages; +xfs_buf_free_pages (bp); +return -ENOMEM ; }/*
@@ -429,13 +402,9 @@ xfs_buf_alloc_pages( bp->b_pages[i] = page; } -return 0 ; -out_free_pages: -for (i =0 ; i < bp->b_page_count; i++) -__free_page (bp->b_pages[i]); - bp->b_flags &= ~_XBF_PAGES; -return error; + bp->b_flags |= _XBF_PAGES; +return 0 ; }/*
@@ -706,7 +675,13 @@ xfs_buf_get_map(*/ if (BBTOB (new_bp->b_length) >=PAGE_SIZE ||xfs_buf_alloc_slab (new_bp, flags) <0 ) { - error =xfs_buf_alloc_pages (new_bp, flags); +gfp_t gfp_mask =xb_to_gfp (flags); + +/* assure a zeroed buffer for non-read cases */ +if (!(flags &XBF_READ )) + gfp_mask |= __GFP_ZERO; + error =xfs_buf_alloc_pages (new_bp, gfp_mask, + flags &XBF_READ_AHEAD );if (error) goto out_free_buf; }
@@ -936,7 +911,7 @@ xfs_buf_get_uncached(int flags, struct xfs_buf **bpp) { - int error, i; + int error;struct xfs_buf *bp;DEFINE_SINGLE_BUF_MAP (map,XFS_BUF_DADDR_NULL , numblks);
@@ -947,19 +922,10 @@ xfs_buf_get_uncached(if (error) goto fail; - error =_xfs_buf_get_pages (bp); + error =xfs_buf_alloc_pages (bp,xb_to_gfp (flags), true);if (error) goto fail_free_buf; -for (i =0 ; i < bp->b_page_count; i++) { - bp->b_pages[i] =alloc_page (xb_to_gfp (flags)); -if (!bp->b_pages[i]) { - error = -ENOMEM ; - goto fail_free_mem; - } - } - bp->b_flags |= _XBF_PAGES; - error =_xfs_buf_map_pages (bp,0 );if (unlikely (error)) { xfs_warn(target->bt_mount,
@@ -972,9 +938,7 @@ xfs_buf_get_uncached(return 0 ; fail_free_mem: -while (--i >=0 ) -__free_page (bp->b_pages[i]); -_xfs_buf_free_pages (bp); +xfs_buf_free_pages (bp); fail_free_buf:xfs_buf_free_maps (bp);kmem_cache_free (xfs_buf_zone, bp);
Signature · 1 line
2.30.2
Re: [PATCH 08/11] xfs: centralize page allocation and freeing for buffers
Quoted 5 lines · Christoph Hellwig
Factor out two helpers that do everything needed for allocating and freeing pages that back a buffer, and remove the duplication between the different interfaces.
Signed-off-by Christoph Hellwig <[email protected]>
This seems really confused.
Up until this point in the patch set you are pulling code out of xfs_buf_alloc_pages() into helpers. Now you are getting rid of the helpers and putting the slightly modified code back into xfs_buf_alloc_pages(). This doesn't make any sense at all.
The freeing helper now requires the buffer state to be manipulated on allocation failure so that the free function doesn't run off the end of the bp->b_pages array. That's a bit of a landmine, and it doesn't really clean anything up much at all.
And on the allocation side there is new "fail fast" behaviour because you've lifted the readahead out of xfs_buf_alloc_pages. You also lifted the zeroing checks, which I note that you immediately put back inside xfs_buf_alloc_pages() in the next patch.
The stuff up to this point in the series makes sense. From this patch onwards it seems to me that you're just undoing the factoring and cleanups from the first few patches...
I mean, like the factoring of xfs_buf_alloc_slab(), you could have just factored out xfs_buf_alloc_pages(bp, page_count) from xfs_buf_allocate_memory() and used that directly in xfs_buf_get_uncached() and avoided a bunch of this factoring, make a slight logic modification and recombine churn. And it would be trivial to do on top of the bulk allocation patch which already converts both of these functions to use bulk allocation....
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
Re: [PATCH 08/11] xfs: centralize page allocation and freeing for buffers
Quoted 4 lines · Dave Chinner
Up until this point in the patch set you are pulling code out of
xfs_buf_alloc_pages()into helpers. Now you are getting rid of the helpers and putting the slightly modified code back intoxfs_buf_alloc_pages(). This doesn't make any sense at all.
It makes a whole lot of sense, but it seems you don't like the structure :)
As stated in the commit log we now have one helper that sets a _XBF_PAGES backing with pages and the map, and one helper to
tear it down. I think it makes a whole lot of sense this way.
Quoted 4 lines
The freeing helper now requires the buffer state to be manipulated on allocation failure so that the free function doesn't run off the end of the
bp->b_pagesarray. That's a bit of a landmine, and it doesn't really clean anything up much at all.
It is something we also do elsewhere in the kernel. Another alternative would be to do a NULL check on the page, or to just pointlessly duplicate the freeing loop.
Quoted 4 lines
And on the allocation side there is new "fail fast" behaviour because you've lifted the readahead out of
xfs_buf_alloc_pages. You also lifted the zeroing checks, which I note that you immediately put back insidexfs_buf_alloc_pages()in the next patch.
This is to clearly split code consolidatation from behavior changes. I could move both earlier at the downside of adding a lot of new code first that later gets removed.
Quoted 7 lines
I mean, like the factoring of
xfs_buf_alloc_slab(), you could have just factored outxfs_buf_alloc_pages(bp, page_count)fromxfs_buf_allocate_memory()and used that directly inxfs_buf_get_uncached()and avoided a bunch of this factoring, make a slight logic modification and recombine churn. And it would be trivial to do on top of the bulk allocation patch which already converts both of these functions to use bulk allocation....
As mentioned in the cover letter: the bulk allocation review is what trigger this as it tripped me following various lose ends. And as usual I'd rather have that kind of change at the end where the surrounding code makes sense, so the rebased version is now is patch 11 of this series.
Re: [PATCH 08/11] xfs: centralize page allocation and freeing for buffers
Quoted 6 lines · Christoph Hellwig
Quoted 4 lines · Dave Chinner
Up until this point in the patch set you are pulling code out of
xfs_buf_alloc_pages()into helpers. Now you are getting rid of the helpers and putting the slightly modified code back intoxfs_buf_alloc_pages(). This doesn't make any sense at all.
Quoted 6 lines
It makes a whole lot of sense, but it seems you don't like the structure :)
As stated in the commit log we now have one helper that sets a
_XBF_PAGESbacking with pages and the map, and one helper to
tear it down. I think it makes a whole lot of sense this way.
I don't like the way the patchset is built. It creates temporary infrastructure, then tears it down again to return the code to almost exactly the same structure that it originally had. In doing this, you change the semantics of functions and helpers multiple times yet, eventually, we end up with the same semantics as we started with.
It's much more obvious to factor out the end helpers first, with the exact semantics that the current have and will end up with, and then just convert and clean up the code in and around those helpers. It's much easier to follow and very correct if the function call semnatics and behaviour don't keep changing...
Quoted 8 lines
Quoted 4 lines
The freeing helper now requires the buffer state to be manipulated on allocation failure so that the free function doesn't run off the end of the
bp->b_pagesarray. That's a bit of a landmine, and it doesn't really clean anything up much at all.It is something we also do elsewhere in the kernel. Another alternative would be to do a NULL check on the page, or to just pointlessly duplicate the freeing loop.
A null check in the freeing code is much simpler to understand at a glance. It's easy to miss that the error handling only works because callers have a single extra line of code that makes error handling work correctly. This is a bad pattern because it's easy for new code to get it wrong and have nobody notice that it's wrong.
Quoted 8 lines
Quoted 4 lines
And on the allocation side there is new "fail fast" behaviour because you've lifted the readahead out of
xfs_buf_alloc_pages. You also lifted the zeroing checks, which I note that you immediately put back insidexfs_buf_alloc_pages()in the next patch.This is to clearly split code consolidatation from behavior changes. I could move both earlier at the downside of adding a lot of new code first that later gets removed.
Ah, what new code? factoring out the _alloc_pages() code at the same time as the alloc_kmem() code is the only "new" code that is necessary. Everything else is then consolidation, and this doesn't require repeatedly changing behaviour and moving code out and back into helpers....
Quoted 8 lines
Quoted 7 lines
I mean, like the factoring of
xfs_buf_alloc_slab(), you could have just factored outxfs_buf_alloc_pages(bp, page_count)fromxfs_buf_allocate_memory()and used that directly inxfs_buf_get_uncached()and avoided a bunch of this factoring, make a slight logic modification and recombine churn. And it would be trivial to do on top of the bulk allocation patch which already converts both of these functions to use bulk allocation....
Quoted 5 lines
As mentioned in the cover letter: the bulk allocation review is what trigger this as it tripped me following various lose ends. And as usual I'd rather have that kind of change at the end where the surrounding code makes sense, so the rebased version is now is patch 11 of this series.
I've re-written my patch based on this cleanup series. It largely does all the same things, and ends up largely in the same place, but does things in an order that doesn't keep changing behaviour and repeatedly moving the same code around. I'll post it once I've QA'd it.
Cheers,
Dave.
Signature · 2 lines
Dave Chinner [email protected]
PATCH 9/11 xfs: lift the buffer zeroing logic into xfs_buf_alloc_pages
In reply to Christoph HellwigLift the buffer zeroing logic from xfs_buf_get_map into so that it also covers uncached buffers, and remove the now obsolete manual zeroing in the only direct caller of xfs_buf_get_uncached.
--- fs/xfs/libxfs/xfs_ag.c | 1 - fs/xfs/xfs_buf.c | 24 +++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-)
fs/xfs/libxfs/xfs_ag.c +0 -1
@@ -43,7 +43,6 @@ xfs_get_aghdr_buf(if (error)return error; -xfs_buf_zero (bp,0 ,BBTOB (bp->b_length)); bp->b_bn = blkno; bp->b_maps[0 ].bm_bn = blkno; bp->b_ops = ops;
fs/xfs/xfs_buf.c +13 -11
@@ -22,9 +22,6 @@static kmem_zone_t *xfs_buf_zone; -#define xb_to_gfp (flags) \ - ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : GFP_NOFS) | __GFP_NOWARN) -/* * Locking orders *
@@ -354,11 +351,21 @@ xfs_buf_alloc_slab(static int xfs_buf_alloc_pages (struct xfs_buf *bp, -gfp_t gfp_mask, +xfs_buf_flags_t flags,bool fail_fast) { +gfp_t gfp_mask = __GFP_NOWARN;int i; +if (flags &XBF_READ_AHEAD ) + gfp_mask |= __GFP_NORETRY; +else + gfp_mask |=GFP_NOFS ; + +/* assure a zeroed buffer for non-read cases */ +if (!(flags &XBF_READ )) + gfp_mask |= __GFP_ZERO; +ASSERT (bp->b_pages ==NULL ); bp->b_page_count =DIV_ROUND_UP (BBTOB (bp->b_length),PAGE_SIZE );
@@ -675,12 +682,7 @@ xfs_buf_get_map(*/ if (BBTOB (new_bp->b_length) >=PAGE_SIZE ||xfs_buf_alloc_slab (new_bp, flags) <0 ) { -gfp_t gfp_mask =xb_to_gfp (flags); - -/* assure a zeroed buffer for non-read cases */ -if (!(flags &XBF_READ )) - gfp_mask |= __GFP_ZERO; - error =xfs_buf_alloc_pages (new_bp, gfp_mask, + error =xfs_buf_alloc_pages (new_bp, flags, flags &XBF_READ_AHEAD );if (error) goto out_free_buf;
@@ -922,7 +924,7 @@ xfs_buf_get_uncached(if (error) goto fail; - error =xfs_buf_alloc_pages (bp,xb_to_gfp (flags), true); + error =xfs_buf_alloc_pages (bp, flags, true);if (error) goto fail_free_buf;
Signature · 1 line
2.30.2
PATCH 10/11 xfs: retry allocations from xfs_buf_get_uncached as well
In reply to Christoph HellwigThere is no good reason why xfs_buf_get_uncached should fail on the first allocation failure, so make it behave the same as the normal xfs_buf_get_map path.
--- fs/xfs/xfs_buf.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-)
fs/xfs/xfs_buf.c +4 -6
@@ -351,8 +351,7 @@ xfs_buf_alloc_slab(static int xfs_buf_alloc_pages (struct xfs_buf *bp, -xfs_buf_flags_t flags, -bool fail_fast) +xfs_buf_flags_t flags) {gfp_t gfp_mask = __GFP_NOWARN;int i;
@@ -384,7 +383,7 @@ xfs_buf_alloc_pages( retry: page =alloc_page (gfp_mask);if (unlikely (!page)) { -if (fail_fast) { +if (flags &XBF_READ_AHEAD ) { bp->b_page_count = i;xfs_buf_free_pages (bp);return -ENOMEM ;
@@ -682,8 +681,7 @@ xfs_buf_get_map(*/ if (BBTOB (new_bp->b_length) >=PAGE_SIZE ||xfs_buf_alloc_slab (new_bp, flags) <0 ) { - error =xfs_buf_alloc_pages (new_bp, flags, - flags &XBF_READ_AHEAD ); + error =xfs_buf_alloc_pages (new_bp, flags);if (error) goto out_free_buf; }
@@ -924,7 +922,7 @@ xfs_buf_get_uncached(if (error) goto fail; - error =xfs_buf_alloc_pages (bp, flags, true); + error =xfs_buf_alloc_pages (bp, flags);if (error) goto fail_free_buf;
Signature · 1 line
2.30.2
PATCH 11/11 xfs: use alloc_pages_bulk_array() for buffers
In reply to Christoph HellwigFrom: Dave Chinner <[email protected]>
Because it's more efficient than allocating pages one at a time in a loop.
--- fs/xfs/xfs_buf.c | 39 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 24 deletions(-)
fs/xfs/xfs_buf.c +15 -24
@@ -354,7 +354,7 @@ xfs_buf_alloc_pages( xfs_buf_flags_t flags) {gfp_t gfp_mask = __GFP_NOWARN; -int i; +unsigned long filled =0 ;if (flags &XBF_READ_AHEAD ) gfp_mask |= __GFP_NORETRY;
@@ -377,36 +377,27 @@ xfs_buf_alloc_pages( bp->b_pages = bp->b_page_array; } -for (i =0 ; i < bp->b_page_count; i++) { -struct page *page; -uint retries =0 ; -retry: - page =alloc_page (gfp_mask); -if (unlikely (!page)) { +/* +* Bulk filling of pages can take multiple calls. Not filling the entire +* array is not an allocation failure, so don't back off if we get at +* least one extra page. +*/ +for (;;) { +unsigned long last = filled; + + filled =alloc_pages_bulk_array (gfp_mask, bp->b_page_count, + bp->b_pages); +if (filled == bp->b_page_count) +break ; +if (filled == last) {if (flags &XBF_READ_AHEAD ) { - bp->b_page_count = i; + bp->b_page_count = filled;xfs_buf_free_pages (bp);return -ENOMEM ; } - -/* -* This could deadlock. -* -* But until all the XFS lowlevel code is revamped to -* handle buffer allocation failures we can't do much. -*/ -if (!(++retries %100 )) -xfs_err (NULL , -"%s(%u) possible memory allocation deadlock in %s (mode:0x%x)" , - current->comm, current->pid, - __func__, gfp_mask); -XFS_STATS_INC (bp->b_mount, xb_page_retries);congestion_wait (BLK_RW_ASYNC ,HZ /50 ); - goto retry; } - - bp->b_pages[i] = page; } bp->b_flags |= _XBF_PAGES;
Signature · 1 line
2.30.2