allocate extent_buffer GFP_NOFAIL with unlocked retry
cover allocate extent_buffer GFP_NOFAIL with unlocked retry
From sampled fleet data measuring lock holders that go into direct reclaim with a waiter present when they eventually unlock, we have observed ~15% of those are btrfs extent_buffer allocations in btrfs_search_slot() done while holding btree locks. This is the single largest category. Additionally, a large source of hung_task timeouts is both btree waiters and direct reclaiming btree allocations, which further motivates the desire to drive down this source of stalls and contention.
The aim of this series is to allow us to allocate the extent_buffer, btrfs_folio_state, and the extent_buffer folios with GFP_NOWAIT then fallback with EAGAIN to outside the critical section to retry with GFP_NOFS | GFP_NOFAIL without any locks held.
This is analogous to how we must drop locks to read an extent_buffer and then EAGAIN.
The series does not manage to completely eliminate allocations from this lock holding path, as we also allocate inside xarray functions for the extent_buffer xarray and the btree_inode mapping xarray, the latter of which is done via filemap_add_folio() with no reserve type API. Luckily, those particular allocations are small cached slab allocations and have nearly no contribution to the production reclaim fueled contention.
Boris Burkov (5):
btrfs: factor init_extent_buffer from __alloc_extent_buffer
btrfs: add struct btrfs_eb_prealloc
btrfs: enable unlocked NOFAIL retry for eb allocations
btrfs: probe with GFP_NOWAIT for tree block readahead
btrfs: use GFP_NOWAIT when inhibiting eb writeback
fs/btrfs/ctree.c | 36 ++++++- fs/btrfs/disk-io.c | 6 +- fs/btrfs/disk-io.h | 2 + fs/btrfs/extent-tree.c | 6 +- fs/btrfs/extent_io.c | 224 ++++++++++++++++++++++++++++------------- fs/btrfs/extent_io.h | 23 +++++ fs/btrfs/subpage.c | 7 +- fs/btrfs/subpage.h | 3 +- fs/btrfs/tree-log.c | 3 +- 9 files changed, 226 insertions(+), 84 deletions(-)
Signature · 1 line
2.54.0
PATCH 1/5 btrfs: factor init_extent_buffer from __alloc_extent_buffer
In preparation for preallocating extent_buffer data, factor eb initialization away from specifically allocating it. This allows us to allocate the eb, bfs, folios, etc. together in the main search_slot code paths, but still share initialization code with the dummy/test/clone allocation paths.
--- fs/btrfs/extent_io.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-)
fs/btrfs/extent_io.c +12 -8
@@ -3054,12 +3054,9 @@ void btrfs_uninhibit_all_eb_writeback(struct btrfs_trans_handle *trans)xa_destroy (&trans->writeback_inhibited_ebs); } -static struct extent_buffer *__alloc_extent_buffer (struct btrfs_fs_info *fs_info, -u64 start) +static void init_extent_buffer (struct btrfs_fs_info *fs_info, +struct extent_buffer *eb,u64 start) { -struct extent_buffer *eb =NULL ; - - eb =kmem_cache_zalloc (extent_buffer_cache,GFP_NOFS |__GFP_NOFAIL); eb->start = start; eb->len = fs_info->nodesize; eb->fs_info = fs_info;
@@ -3072,7 +3069,15 @@ static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_inforefcount_set (&eb->refs,1 );ASSERT (eb->len <=BTRFS_MAX_METADATA_BLOCKSIZE ); +} + +static struct extent_buffer *__alloc_extent_buffer (struct btrfs_fs_info *fs_info, +u64 start) +{ +struct extent_buffer *eb; + eb =kmem_cache_zalloc (extent_buffer_cache,GFP_NOFS | __GFP_NOFAIL); +init_extent_buffer (fs_info, eb, start);return eb; }
@@ -3476,9 +3481,8 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,if (eb)return eb; - eb =__alloc_extent_buffer (fs_info, start); -if (!eb) -return ERR_PTR (-ENOMEM ); + eb =kmem_cache_zalloc (extent_buffer_cache,GFP_NOFS | __GFP_NOFAIL); +init_extent_buffer (fs_info, eb, start);/* * The reloc trees are just snapshots, so we need them to appear to be
Signature · 1 line
2.54.0
PATCH 2/5 btrfs: add struct btrfs_eb_prealloc
In reply to Boris BurkovIn further preparation for supporting NOFAIL allocations with retries outside the critical section, add a struct to carry the extent_buffer and btrfs_folio_state we need to allocate.
Refactor the allocation pathways to use the new struct but with no functional change. Wire empty prealloc structs in from callers.
--- fs/btrfs/ctree.c | 21 +++-- fs/btrfs/disk-io.c | 6 +- fs/btrfs/disk-io.h | 2 + fs/btrfs/extent-tree.c | 6 +- fs/btrfs/extent_io.c | 179 ++++++++++++++++++++++++++++------------- fs/btrfs/extent_io.h | 19 +++++ fs/btrfs/tree-log.c | 3 +- 7 files changed, 168 insertions(+), 68 deletions(-)
fs/btrfs/ctree.c +16 -5
@@ -1460,6 +1460,7 @@ static noinline void unlock_up(struct btrfs_path *path, int level,*/ static int read_block_for_search (struct btrfs_root *root,struct btrfs_path *p, +struct btrfs_eb_prealloc *pa,struct extent_buffer **eb_ret,int slot,const struct btrfs_key *key) {
@@ -1546,7 +1547,8 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,if (p->reada !=READA_NONE )reada_for_search (fs_info, p, parent_level, slot, key->objectid); - tmp =btrfs_find_create_tree_block (fs_info, blocknr, check.owner_root, check.level); + tmp =btrfs_find_create_tree_block (fs_info, pa, blocknr, + check.owner_root, check.level);if (IS_ERR (tmp)) { ret =PTR_ERR (tmp); tmp =NULL ;
@@ -2004,6 +2006,7 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,u8 lowest_level =0 ;int min_write_lock_level;int prev_cmp; +struct btrfs_eb_prealloc pa = {0 };if (!root)return -EINVAL ;
@@ -2187,7 +2190,7 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, goto done; } - ret2 =read_block_for_search (root, p, &b, slot, key); + ret2 =read_block_for_search (root, p, &pa, &b, slot, key);if (ret2 == -EAGAIN && !p->nowait) {trace_btrfs_search_slot_restart (root, level,"read_block" ); goto again;
@@ -2234,6 +2237,8 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, ret = ret2; } +btrfs_free_eb_prealloc (&pa); +return ret; }ALLOW_ERROR_INJECTION (btrfs_search_slot,ERRNO );
@@ -2259,6 +2264,7 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key,int level;int lowest_unlock =1 ;u8 lowest_level =0 ; +struct btrfs_eb_prealloc pa = {0 }; lowest_level = p->lowest_level;WARN_ON (p->nodes[0 ] !=NULL );
@@ -2316,7 +2322,7 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, goto done; } - ret2 =read_block_for_search (root, p, &b, slot, key); + ret2 =read_block_for_search (root, p, &pa, &b, slot, key);if (ret2 == -EAGAIN && !p->nowait) goto again;if (ret2) {
@@ -2339,6 +2345,8 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key,if (ret <0 )btrfs_release_path (p); +btrfs_free_eb_prealloc (&pa); +return ret; }
@@ -4780,6 +4788,7 @@ int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path,struct extent_buffer *next;struct btrfs_fs_info *fs_info = root->fs_info;struct btrfs_key key; +struct btrfs_eb_prealloc pa = {0 };bool need_commit_sem = false;u32 nritems;int ret;
@@ -4880,7 +4889,7 @@ int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, } next = c; - ret =read_block_for_search (root, path, &next, slot, &key); + ret =read_block_for_search (root, path, &pa, &next, slot, &key);if (ret == -EAGAIN && !path->nowait) goto again;
@@ -4923,7 +4932,7 @@ int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path,if (!level)break ; - ret =read_block_for_search (root, path, &next,0 , &key); + ret =read_block_for_search (root, path, &pa, &next,0 , &key);if (ret == -EAGAIN && !path->nowait) goto again;
@@ -4956,6 +4965,8 @@ int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, ret = ret2; } +btrfs_free_eb_prealloc (&pa); +return ret; }
fs/btrfs/disk-io.c +4 -2
@@ -590,12 +590,13 @@ static const struct address_space_operations btree_aops = {struct extent_buffer *btrfs_find_create_tree_block (struct btrfs_fs_info *fs_info, +struct btrfs_eb_prealloc *pa,u64 bytenr,u64 owner_root,int level) {if (btrfs_is_testing (fs_info))return alloc_test_extent_buffer (fs_info, bytenr); -return alloc_extent_buffer (fs_info, bytenr, owner_root, level); +return alloc_extent_buffer (fs_info, pa, bytenr, owner_root, level); }/*
@@ -608,12 +609,13 @@ struct extent_buffer *btrfs_find_create_tree_block(struct extent_buffer *read_tree_block (struct btrfs_fs_info *fs_info,u64 bytenr,struct btrfs_tree_parent_check *check) { +struct btrfs_eb_prealloc pa = {0 };struct extent_buffer *buf =NULL ;int ret;ASSERT (check); - buf =btrfs_find_create_tree_block (fs_info, bytenr, check->owner_root, + buf =btrfs_find_create_tree_block (fs_info, &pa, bytenr, check->owner_root, check->level);if (IS_ERR (buf))return buf;
fs/btrfs/disk-io.h +2 -0
@@ -15,6 +15,7 @@struct block_device ;struct super_block ;struct extent_buffer ; +struct btrfs_eb_prealloc ;struct btrfs_device ;struct btrfs_fs_devices ;struct btrfs_fs_info ;
@@ -48,6 +49,7 @@ struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr, struct btrfs_tree_parent_check *check);struct extent_buffer *btrfs_find_create_tree_block (struct btrfs_fs_info *fs_info, +struct btrfs_eb_prealloc *pa,u64 bytenr,u64 owner_root,int level);int btrfs_start_pre_rw_mount (struct btrfs_fs_info *fs_info);
fs/btrfs/extent-tree.c +4 -2
@@ -5260,10 +5260,11 @@ btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root, enum btrfs_lock_nesting nest) {struct btrfs_fs_info *fs_info = root->fs_info; +struct btrfs_eb_prealloc pa = {0 };struct extent_buffer *buf;u64 lockdep_owner = owner; - buf =btrfs_find_create_tree_block (fs_info, bytenr, owner, level); + buf =btrfs_find_create_tree_block (fs_info, &pa, bytenr, owner, level);if (IS_ERR (buf))return buf;
@@ -5917,6 +5918,7 @@ static noinline int do_walk_down(struct btrfs_trans_handle *trans, struct walk_control *wc) {struct btrfs_fs_info *fs_info = root->fs_info; +struct btrfs_eb_prealloc pa = {0 };u64 bytenr;u64 generation;u64 owner_root =0 ;
@@ -5939,7 +5941,7 @@ static noinline int do_walk_down(struct btrfs_trans_handle *trans, bytenr =btrfs_node_blockptr (path->nodes[level], path->slots[level]); - next =btrfs_find_create_tree_block (fs_info, bytenr,btrfs_root_id (root), + next =btrfs_find_create_tree_block (fs_info, &pa, bytenr,btrfs_root_id (root), level -1 );if (IS_ERR (next))return PTR_ERR (next);
fs/btrfs/extent_io.c +121 -58
258 lines not on this page · raw message
fs/btrfs/extent_io.h +19 -0
@@ -119,6 +119,21 @@ struct extent_buffer {#endif }; +/* +* Wrapper struct for managing preallocating an extent_buffer, its folios and a +* btrfs_folio_state if needed. +* +* Only used to mediate allocation, do not refer to the eb directly if not +* returned from a successful eb allocating API. +* +* The eb folios and bfs should generally not be fully attached, except briefly +* before they are NULLed in the struct after successful attachment. +*/ +struct btrfs_eb_prealloc { +struct extent_buffer *eb; +struct btrfs_folio_state *bfs; +}; +struct btrfs_eb_write_context {struct writeback_control *wbc;struct extent_buffer *eb;
@@ -266,7 +281,11 @@ int set_folio_extent_mapped(struct folio *folio);void clear_folio_extent_mapped (struct folio *folio);struct extent_buffer *alloc_extent_buffer (struct btrfs_fs_info *fs_info, +struct btrfs_eb_prealloc *pa,u64 start,u64 owner_root,int level); +int btrfs_init_eb_prealloc (struct btrfs_fs_info *fs_info, +struct btrfs_eb_prealloc *pa); +void btrfs_free_eb_prealloc (struct btrfs_eb_prealloc *pa);struct extent_buffer *alloc_dummy_extent_buffer (struct btrfs_fs_info *fs_info,u64 start);struct extent_buffer *btrfs_clone_extent_buffer (const struct extent_buffer *src);
fs/btrfs/tree-log.c +2 -1
@@ -2986,6 +2986,7 @@ static noinline int walk_down_log_tree(struct btrfs_path *path, int *level, {struct btrfs_trans_handle *trans = wc->trans;struct btrfs_fs_info *fs_info = wc->log->fs_info; +struct btrfs_eb_prealloc pa = {0 };u64 bytenr;u64 ptr_gen;struct extent_buffer *next;
@@ -3010,7 +3011,7 @@ static noinline int walk_down_log_tree(struct btrfs_path *path, int *level, check.has_first_key = true;btrfs_node_key_to_cpu (cur, &check.first_key, path->slots[*level]); - next =btrfs_find_create_tree_block (fs_info, bytenr, + next =btrfs_find_create_tree_block (fs_info, &pa, bytenr,btrfs_header_owner (cur), *level -1 );if (IS_ERR (next)) {
Signature · 1 line
2.54.0
PATCH 3/5 btrfs: enable unlocked NOFAIL retry for eb allocations
In reply to Boris BurkovNow that we have the btrfs_eb_prealloc struct to carry the allocation and the "needs prealloc" signal, wire that up between the various search_slot style callers down into alloc_extent_buffer.
If the prealloc struct indicates that it supports a nowait try, then alloc_extent_buffer tries to allocate NOWAIT. If that succeeds, great. Otherwise, we return EAGAIN and signal via the struct that preallocation is required. The caller then does the allocation and tries again with the eb, bfs, and folios wired through in the prealloc struct.
If unlock-and-allocate retries are not supported then we just use the normal gfp flags like before.
Note that there are still two GFP_NOFS allocations, as far as I know, that happen under the lock and cannot be preallocated:
- the __xa_cmpxchg to insert the eb into the eb xarray
- the xarray allocations for filemap_add_folio to add the folios to the btree_inode mapping.
The former we could wire up with xa_reserve if we signaled the "prealloc start" back up to the retry point. However, since there is no concept of reservation in the filemap xarray, it seemed relatively unhelpful to bother. These allocations are relatively small cached slab allocations, so hopefully we can move the needle on reclaim stalls without reserving them.
--- fs/btrfs/ctree.c | 21 ++++++++++++++++++--- fs/btrfs/extent_io.c | 27 +++++++++++++++++++++------ fs/btrfs/extent_io.h | 6 +++++- fs/btrfs/subpage.c | 7 ++++--- fs/btrfs/subpage.h | 3 ++- 5 files changed, 50 insertions(+), 14 deletions(-)
fs/btrfs/ctree.c +18 -3
@@ -2006,7 +2006,7 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,u8 lowest_level =0 ;int min_write_lock_level;int prev_cmp; -struct btrfs_eb_prealloc pa = {0 }; +struct btrfs_eb_prealloc pa = { .supports_nowait = true };if (!root)return -EINVAL ;
@@ -2061,6 +2061,11 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, } again: +if (pa.needs_prealloc) { + ret =btrfs_init_eb_prealloc (fs_info, &pa, false); +if (ret) + goto done; + } prev_cmp =-1 ; b =btrfs_search_slot_get_root (root, p, write_lock_level);if (IS_ERR (b)) {
@@ -2264,7 +2269,7 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key,int level;int lowest_unlock =1 ;u8 lowest_level =0 ; -struct btrfs_eb_prealloc pa = {0 }; +struct btrfs_eb_prealloc pa = { .supports_nowait = true }; lowest_level = p->lowest_level;WARN_ON (p->nodes[0 ] !=NULL );
@@ -2276,6 +2281,11 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, } again: +if (pa.needs_prealloc) { + ret =btrfs_init_eb_prealloc (fs_info, &pa, false); +if (ret) + goto done; + } b =btrfs_get_old_root (root, time_seq);if (unlikely (!b)) { ret = -EIO ;
@@ -4788,7 +4798,7 @@ int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path,struct extent_buffer *next;struct btrfs_fs_info *fs_info = root->fs_info;struct btrfs_key key; -struct btrfs_eb_prealloc pa = {0 }; +struct btrfs_eb_prealloc pa = { .supports_nowait = true };bool need_commit_sem = false;u32 nritems;int ret;
@@ -4807,6 +4817,11 @@ int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path,btrfs_item_key_to_cpu (path->nodes[0 ], &key, nritems -1 ); again: +if (pa.needs_prealloc) { + ret =btrfs_init_eb_prealloc (fs_info, &pa, false); +if (ret) + goto done; + } level =1 ; next =NULL ;btrfs_release_path (path);
fs/btrfs/extent_io.c +21 -6
@@ -3477,18 +3477,28 @@ static int attach_eb_folio_to_filemap(struct extent_buffer *eb, int i,/* * Allocate the extent_buffer, its folios, and btrfs_folio_state, if needed. * +* @pa: The holder struct to do the allocation in. +* @nowait: Whether to do a speculative GFP_NOWAIT allocation while holding locks. +* * Return 0 on success and a negative errno otherwise. On failure, pa->eb/bfs -* will be NULL. +* will be NULL. If @nowait=true, then on ENOMEM, mark @pa->needs_prealloc and +* return -EAGAIN to signal the caller to unlock and retry. */ int btrfs_init_eb_prealloc (struct btrfs_fs_info *fs_info, -struct btrfs_eb_prealloc *pa) +struct btrfs_eb_prealloc *pa,bool nowait) { +gfp_t gfp = nowait ?GFP_NOWAIT :GFP_NOFS | __GFP_NOFAIL;int ret;ASSERT (!pa->eb,"unexpected non-null eb: %p" , pa->eb);ASSERT (!pa->bfs,"unexpected non-null bfs: %p" , pa->bfs); + pa->needs_prealloc = false; - pa->eb =kmem_cache_zalloc (extent_buffer_cache,GFP_NOFS | __GFP_NOFAIL); + pa->eb =kmem_cache_zalloc (extent_buffer_cache, gfp); +if (!pa->eb) { + ret = -ENOMEM ; + goto out; + }/* alloc_eb_folio_array() needs len; init_extent_buffer() sets it again later. */ pa->eb->len = fs_info->nodesize;
@@ -3501,7 +3511,7 @@ int btrfs_init_eb_prealloc(struct btrfs_fs_info *fs_info,*/ if (btrfs_meta_is_subpage (fs_info)) { pa->bfs =btrfs_alloc_folio_state (fs_info,PAGE_SIZE , -BTRFS_SUBPAGE_METADATA ); +BTRFS_SUBPAGE_METADATA , gfp);if (IS_ERR (pa->bfs)) { ret =PTR_ERR (pa->bfs); pa->bfs =NULL ;
@@ -3514,7 +3524,7 @@ int btrfs_init_eb_prealloc(struct btrfs_fs_info *fs_info,* below (added to LRU, served by btree_migrate_folio), so request * __GFP_MOVABLE so the page allocator places them in MOVABLE pageblocks. */ - ret =alloc_eb_folio_array (pa->eb,GFP_NOFS | __GFP_NOFAIL | __GFP_MOVABLE); + ret =alloc_eb_folio_array (pa->eb, gfp | __GFP_MOVABLE);if (ret <0 ) goto free_bfs;
@@ -3526,6 +3536,11 @@ int btrfs_init_eb_prealloc(struct btrfs_fs_info *fs_info, free_eb:kmem_cache_free (extent_buffer_cache, pa->eb); pa->eb =NULL ; +out: +if (nowait && ret == -ENOMEM ) { + pa->needs_prealloc = true; + ret = -EAGAIN ; + }return ret; }
@@ -3582,7 +3597,7 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,return eb;if (!pa->eb) { - ret =btrfs_init_eb_prealloc (fs_info, pa); + ret =btrfs_init_eb_prealloc (fs_info, pa, pa->supports_nowait);if (ret)return ERR_PTR (ret); }
fs/btrfs/extent_io.h +5 -1
@@ -132,6 +132,10 @@ struct extent_buffer {struct btrfs_eb_prealloc {struct extent_buffer *eb;struct btrfs_folio_state *bfs; +/* eb alloc may use GFP_NOWAIT; caller can drop locks and retry. */ +bool supports_nowait; +/* GFP_NOWAIT eb alloc failed; preallocate again and retry. */ +bool needs_prealloc; };struct btrfs_eb_write_context {
@@ -284,7 +288,7 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,struct btrfs_eb_prealloc *pa, u64 start, u64 owner_root, int level);int btrfs_init_eb_prealloc (struct btrfs_fs_info *fs_info, -struct btrfs_eb_prealloc *pa); +struct btrfs_eb_prealloc *pa,bool nowait);void btrfs_free_eb_prealloc (struct btrfs_eb_prealloc *pa);struct extent_buffer *alloc_dummy_extent_buffer (struct btrfs_fs_info *fs_info,u64 start);
fs/btrfs/subpage.c +4 -3
@@ -59,7 +59,7 @@ int btrfs_attach_folio_state(const struct btrfs_fs_info *fs_info,if (type ==BTRFS_SUBPAGE_DATA && !btrfs_is_subpage (fs_info, folio))return 0 ; - bfs =btrfs_alloc_folio_state (fs_info,folio_size (folio), type); + bfs =btrfs_alloc_folio_state (fs_info,folio_size (folio), type,GFP_NOFS );if (IS_ERR (bfs))return PTR_ERR (bfs);
@@ -86,7 +86,8 @@ void btrfs_detach_folio_state(const struct btrfs_fs_info *fs_info, struct folio }struct btrfs_folio_state *btrfs_alloc_folio_state (const struct btrfs_fs_info *fs_info, -size_t fsize,enum btrfs_folio_type type) +size_t fsize,enum btrfs_folio_type type, +gfp_t gfp) {struct btrfs_folio_state *ret;unsigned int real_size;
@@ -96,7 +97,7 @@ struct btrfs_folio_state *btrfs_alloc_folio_state(const struct btrfs_fs_info *fs real_size =struct_size (ret, bitmaps,BITS_TO_LONGS (btrfs_bitmap_nr_max * (fsize >> fs_info->sectorsize_bits))); - ret =kzalloc (real_size,GFP_NOFS ); + ret =kzalloc (real_size, gfp);if (!ret)return ERR_PTR (-ENOMEM );
fs/btrfs/subpage.h +2 -1
@@ -102,7 +102,8 @@ void btrfs_detach_folio_state(const struct btrfs_fs_info *fs_info, struct folio/* Allocate additional data where page represents more than one sector */ struct btrfs_folio_state *btrfs_alloc_folio_state (const struct btrfs_fs_info *fs_info, -size_t fsize,enum btrfs_folio_type type); +size_t fsize,enum btrfs_folio_type type, +gfp_t gfp);static inline void btrfs_free_folio_state (struct btrfs_folio_state *bfs) {kfree (bfs);
Signature · 1 line
2.54.0
PATCH 4/5 btrfs: probe with GFP_NOWAIT for tree block readahead
In reply to Boris Burkovextent_buffer readahead should not be able to painfully stall a search_slot and hog tree locks by getting stuck in direct reclaim.
If the allocation fails, that is fine, we simply fail to do the readahead in that case.
--- fs/btrfs/extent_io.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
fs/btrfs/extent_io.c +2 -1
@@ -4737,7 +4737,8 @@ void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info, .level = level, .transid = gen }; -struct btrfs_eb_prealloc pa = {0 }; +/* Readahead is best effort so prefer to fail rather than block in reclaim. */ +struct btrfs_eb_prealloc pa = { .supports_nowait = true };struct extent_buffer *eb;int ret;
Signature · 1 line
2.54.0
PATCH 5/5 btrfs: use GFP_NOWAIT when inhibiting eb writeback
In reply to Boris BurkovInhibition allocation is already best effort and done while holding locks so switch it to NOWAIT.
--- fs/btrfs/extent_io.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-)
fs/btrfs/extent_io.c +6 -5
@@ -3002,10 +3002,11 @@ static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)* @trans: transaction handle that will own the inhibitor * @eb: extent buffer to inhibit writeback on * -* Attempt to track this extent buffer in the transaction's inhibited set. If -* memory allocation fails, the buffer is simply not tracked. It may be written -* back and need re-COW, which is the original behavior. This is acceptable -* since inhibiting writeback is an optimization. +* Attempt to track this extent buffer in the transaction's inhibited set. We +* are called under eb->lock, so try with GFP_NOWAIT rather than entering +* reclaim under the lock. If the allocation fails, the buffer is simply not +* tracked. It may be written back and need re-COW, which is the original +* behavior. This is acceptable since inhibiting writeback is an optimization. */ void btrfs_inhibit_eb_writeback (struct btrfs_trans_handle *trans,struct extent_buffer *eb) {
@@ -3021,7 +3022,7 @@ void btrfs_inhibit_eb_writeback(struct btrfs_trans_handle *trans, struct extent_/* Take reference for the xarray entry. */ refcount_inc (&eb->refs); - old =xa_store (&trans->writeback_inhibited_ebs, index, eb,GFP_NOFS ); + old =xa_store (&trans->writeback_inhibited_ebs, index, eb,GFP_NOWAIT );if (xa_is_err (old)) {/* Allocation failed, just skip inhibiting this buffer. */ free_extent_buffer (eb);
Signature · 1 line
2.54.0
Re: [PATCH 1/5] btrfs: factor init_extent_buffer from __alloc_extent_buffer
In reply to Boris BurkovQuoted 48 lines · Boris Burkov
In preparation for preallocating
extent_bufferdata, factor eb initialization away from specifically allocating it. This allows us to allocate the eb, bfs, folios, etc. together in the mainsearch_slotcode paths, but still share initialization code with the dummy/test/clone allocation paths.Signed-off-by Boris Burkov <[email protected]>--- fs/btrfs/extent_io.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-)fs/btrfs/extent_io.c +10
-5@@ -3054,12 +3054,9 @@ void btrfs_uninhibit_all_eb_writeback(struct btrfs_trans_handle *trans)xa_destroy (&trans->writeback_inhibited_ebs); } -static struct extent_buffer *__alloc_extent_buffer (struct btrfs_fs_info *fs_info, -u64 start) +static void init_extent_buffer (struct btrfs_fs_info *fs_info, +struct extent_buffer *eb,u64 start) { -struct extent_buffer *eb =NULL ; - - eb =kmem_cache_zalloc (extent_buffer_cache,GFP_NOFS |__GFP_NOFAIL); eb->start = start; eb->len = fs_info->nodesize; eb->fs_info = fs_info;@@ -3072,7 +3069,15 @@ static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_inforefcount_set (&eb->refs,1 );ASSERT (eb->len <=BTRFS_MAX_METADATA_BLOCKSIZE ); +} + +static struct extent_buffer *__alloc_extent_buffer (struct btrfs_fs_info *fs_info, +u64 start) +{ +struct extent_buffer *eb; + eb =kmem_cache_zalloc (extent_buffer_cache,GFP_NOFS | __GFP_NOFAIL); +init_extent_buffer (fs_info, eb, start);return eb; }@@ -3476,9 +3481,8 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,if (eb)
fs/btrfs/extent_io.c struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, · line 3488
Quoted 10 lines
fs/btrfs/extent_io.c +2
-3return eb; - eb =__alloc_extent_buffer (fs_info, start); -if (!eb) -return ERR_PTR (-ENOMEM ); + eb =kmem_cache_zalloc (extent_buffer_cache,GFP_NOFS | __GFP_NOFAIL); +init_extent_buffer (fs_info, eb, start);/* * The reloc trees are just snapshots, so we need them to appear to be
This looks wrong to me. Not the split out code, but how it is used in alloc_extent_buffer(). Either you keep calling __alloc_extent_buffer() in alloc_extent_buffer() OR don't call init_extent_buffer() in __alloc_extent_buffer(). I see this is an intermediate step (and I haven't looked at the rest of the series yet) but it looks wrong.
Re: [PATCH 1/5] btrfs: factor init_extent_buffer from __alloc_extent_buffer
Quoted 51 lines · Johannes Thumshirn
Quoted 50 lines · Boris Burkov
In preparation for preallocating
extent_bufferdata, factor eb initialization away from specifically allocating it. This allows us to allocate the eb, bfs, folios, etc. together in the mainsearch_slotcode paths, but still share initialization code with the dummy/test/clone allocation paths.Signed-off-by Boris Burkov <[email protected]>--- fs/btrfs/extent_io.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-)fs/btrfs/extent_io.c +12
-8@@ -3054,12 +3054,9 @@ void btrfs_uninhibit_all_eb_writeback(struct btrfs_trans_handle *trans)xa_destroy (&trans->writeback_inhibited_ebs); } -static struct extent_buffer *__alloc_extent_buffer (struct btrfs_fs_info *fs_info, -u64 start) +static void init_extent_buffer (struct btrfs_fs_info *fs_info, +struct extent_buffer *eb,u64 start) { -struct extent_buffer *eb =NULL ; - - eb =kmem_cache_zalloc (extent_buffer_cache,GFP_NOFS |__GFP_NOFAIL); eb->start = start; eb->len = fs_info->nodesize; eb->fs_info = fs_info;@@ -3072,7 +3069,15 @@ static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_inforefcount_set (&eb->refs,1 );ASSERT (eb->len <=BTRFS_MAX_METADATA_BLOCKSIZE ); +} + +static struct extent_buffer *__alloc_extent_buffer (struct btrfs_fs_info *fs_info, +u64 start) +{ +struct extent_buffer *eb; + eb =kmem_cache_zalloc (extent_buffer_cache,GFP_NOFS | __GFP_NOFAIL); +init_extent_buffer (fs_info, eb, start);return eb; }@@ -3476,9 +3481,8 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,if (eb)return eb; - eb =__alloc_extent_buffer (fs_info, start); -if (!eb) -return ERR_PTR (-ENOMEM ); + eb =kmem_cache_zalloc (extent_buffer_cache,GFP_NOFS | __GFP_NOFAIL); +init_extent_buffer (fs_info, eb, start);
Quoted 9 lines
fs/btrfs/extent_io.c struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, · line 3488
Quoted 2 lines
fs/btrfs/extent_io.c +0
-0/* * The reloc trees are just snapshots, so we need them to appear to be This looks wrong to me. Not the split out code, but how it is used in
alloc_extent_buffer(). Either you keep calling__alloc_extent_buffer()inalloc_extent_buffer()OR don't callinit_extent_buffer()in__alloc_extent_buffer(). I see this is an intermediate step (and I haven't looked at the rest of the series yet) but it looks wrong.
The idea is that you eventually (optionally) have the allocation happen elsewhere, so alloc_extent_buffer should not call __alloc_extent_buffer any more. That should be clearer in the second patch, I think.
I left __alloc_extent_buffer for btrfs_clone_extent_buffer() and alloc_dummy_extent_buffer(). Would you prefer me to delete __alloc_extent_buffer and open code it in those two callers?
Thanks,
Boris
Re: [PATCH 1/5] btrfs: factor init_extent_buffer from __alloc_extent_buffer
Quoted 60 lines · Boris Burkov
Quoted 50 lines · Johannes Thumshirn
Quoted 49 lines · Boris Burkov
In preparation for preallocating
extent_bufferdata, factor eb initialization away from specifically allocating it. This allows us to allocate the eb, bfs, folios, etc. together in the mainsearch_slotcode paths, but still share initialization code with the dummy/test/clone allocation paths.Signed-off-by Boris Burkov <[email protected]>--- fs/btrfs/extent_io.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-)fs/btrfs/extent_io.c +11
-8@@ -3054,12 +3054,9 @@ void btrfs_uninhibit_all_eb_writeback(struct btrfs_trans_handle *trans)xa_destroy (&trans->writeback_inhibited_ebs); } -static struct extent_buffer *__alloc_extent_buffer (struct btrfs_fs_info *fs_info, -u64 start) +static void init_extent_buffer (struct btrfs_fs_info *fs_info, +struct extent_buffer *eb,u64 start) { -struct extent_buffer *eb =NULL ; - - eb =kmem_cache_zalloc (extent_buffer_cache,GFP_NOFS |__GFP_NOFAIL); eb->start = start; eb->len = fs_info->nodesize; eb->fs_info = fs_info;@@ -3072,7 +3069,15 @@ static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_inforefcount_set (&eb->refs,1 );ASSERT (eb->len <=BTRFS_MAX_METADATA_BLOCKSIZE ); +} + +static struct extent_buffer *__alloc_extent_buffer (struct btrfs_fs_info *fs_info, +u64 start) +{ +struct extent_buffer *eb; + eb =kmem_cache_zalloc (extent_buffer_cache,GFP_NOFS | __GFP_NOFAIL); +init_extent_buffer (fs_info, eb, start);return eb; }@@ -3476,9 +3481,8 @@ struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,if (eb)return eb; - eb =__alloc_extent_buffer (fs_info, start); -if (!eb) -return ERR_PTR (-ENOMEM ); + eb =kmem_cache_zalloc (extent_buffer_cache,GFP_NOFS | __GFP_NOFAIL);Quoted 9 lines
fs/btrfs/extent_io.c struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, · line 3488
Quoted 3 lines
fs/btrfs/extent_io.c +1
-0+init_extent_buffer (fs_info, eb, start);/* * The reloc trees are just snapshots, so we need them to appear to be This looks wrong to me. Not the split out code, but how it is used in
alloc_extent_buffer(). Either you keep calling__alloc_extent_buffer()inalloc_extent_buffer()OR don't callinit_extent_buffer()in__alloc_extent_buffer(). I see this is an intermediate step (and I haven't looked at the rest of the series yet) but it looks wrong.
Quoted 7 lines
The idea is that you eventually (optionally) have the allocation happen elsewhere, so
alloc_extent_buffershould not call__alloc_extent_bufferany more. That should be clearer in the second patch, I think.I left
__alloc_extent_bufferforbtrfs_clone_extent_buffer()andalloc_dummy_extent_buffer(). Would you prefer me to delete__alloc_extent_bufferand open code it in those two callers?
Yes looking at it again with the complete series applied it make more sense. Sorry for the noise.
Quoted 31 lines · Boris Burkov
From sampled fleet data measuring lock holders that go into direct reclaim with a waiter present when they eventually unlock, we have observed ~15% of those are btrfs
extent_bufferallocations inbtrfs_search_slot()done while holding btree locks. This is the single largest category. Additionally, a large source ofhung_tasktimeouts is both btree waiters and direct reclaiming btree allocations, which further motivates the desire to drive down this source of stalls and contention.The aim of this series is to allow us to allocate the
extent_buffer,btrfs_folio_state, and theextent_bufferfolios withGFP_NOWAITthen fallback with EAGAIN to outside the critical section to retry withGFP_NOFS|GFP_NOFAILwithout any locks held.This is analogous to how we must drop locks to read an
extent_bufferand then EAGAIN.The series does not manage to completely eliminate allocations from this lock holding path, as we also allocate inside xarray functions for the
extent_bufferxarray and thebtree_inodemapping xarray, the latter of which is done viafilemap_add_folio()with no reserve type API. Luckily, those particular allocations are small cached slab allocations and have nearly no contribution to the production reclaim fueled contention.Boris Burkov (5):
btrfs: factorinit_extent_bufferfrom__alloc_extent_buffer
btrfs: add structbtrfs_eb_prealloc
btrfs: enable unlocked NOFAIL retry for eb allocations
btrfs: probe withGFP_NOWAITfor tree block readahead
btrfs: useGFP_NOWAITwhen inhibiting eb writeback
Quoted 10 lines
fs/btrfs/ctree.c | 36 ++++++- fs/btrfs/disk-io.c | 6 +- fs/btrfs/disk-io.h | 2 + fs/btrfs/extent-tree.c | 6 +- fs/btrfs/extent_io.c | 224 ++++++++++++++++++++++++++++------------- fs/btrfs/extent_io.h | 23 +++++ fs/btrfs/subpage.c | 7 +- fs/btrfs/subpage.h | 3 +- fs/btrfs/tree-log.c | 3 +- 9 files changed, 226 insertions(+), 84 deletions(-)
Nice work, Boris! This all looks good to me. You can add:
Note that LLM review noted a couple of minor nits you might want to address if/when you post a v2:
- [N1] Dead code: commit 3 repoints the wrapper's only caller to the _prealloc variant, so btrfs_insert_delayed_dir_index() (kept "to preserve behavior" in commit 2) ends the series with zero callers in-tree. Suggest removing it + its prototype in commit 3 or a follow-up.
- [N2] Latent (unreachable) leak: in btrfs_insert_dir_item, a non-NULL prealloc with need_delayed_index == false is neither committed nor freed. Only btrfs_rename passes a prealloc and its root is never tree_root, so it can't fire today — a defensive else if (prealloc) free would harden it.