btrfs: avoid some block group rbtree lock contention
cover btrfs: avoid some block group rbtree lock contention
From: Filipe Manana <[email protected]>
This patchset allows for better concurrency when accessing the red black tree of block groups, which is used very frequently and most accesses are read-only, as well as avoid some unnecessary searches in the tree during NOCOW writes. Details in the changelogs.
Filipe Manana (5):
btrfs: remove search start argument from first_logical_byte()
btrfs: use rbtree with leftmost node cached for tracking lowest block group
btrfs: use a read/write lock for protecting the block groups tree
btrfs: return block group directly at btrfs_next_block_group()
btrfs: avoid double search for block group during NOCOW writes
fs/btrfs/block-group.c | 130 ++++++++++++++++++++---------------- fs/btrfs/block-group.h | 5 +- fs/btrfs/ctree.h | 5 +- fs/btrfs/disk-io.c | 5 +- fs/btrfs/extent-tree.c | 29 ++++---- fs/btrfs/free-space-cache.c | 2 +- fs/btrfs/free-space-tree.c | 2 +- fs/btrfs/inode.c | 26 +++++--- fs/btrfs/transaction.c | 4 +- 9 files changed, 114 insertions(+), 94 deletions(-)
Signature · 1 line
2.35.1
PATCH 2/5 btrfs: use rbtree with leftmost node cached for tracking lowest block group
From: Filipe Manana <[email protected]>
We keep track of the start offset of the block group with the lowest start offset at fs_info->first_logical_byte. This requires explicitly updating that field everytime we add, delete or lookup a block group to/from the red black tree at fs_info->block_group_cache_tree.
Since the block group with the lowest start address happens to always be the one that is the leftmost node of the tree, we can use a red black tree that caches the left most node. Then when we need the start address of that block group, we can just quickly get the leftmost node in the tree and extract the start offset of that node's block group. This avoids the need to explicitly keep track of that address in the dedicated member fs_info->first_logical_byte, and it also allows the next patch in the series to switch the lock that protects the red black tree from a spin lock to a read/write lock - without this change it would be tricky because block group searches also update fs_info->first_logical_byte.
--- fs/btrfs/block-group.c | 30 ++++++++++++------------------ fs/btrfs/ctree.h | 3 +-- fs/btrfs/disk-io.c | 3 +-- fs/btrfs/extent-tree.c | 22 +++++++++------------- fs/btrfs/free-space-cache.c | 2 +- fs/btrfs/free-space-tree.c | 2 +- 6 files changed, 25 insertions(+), 37 deletions(-)
fs/btrfs/block-group.c +12 -18
@@ -168,11 +168,12 @@ static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,struct rb_node **p;struct rb_node *parent =NULL ;struct btrfs_block_group *cache; +bool leftmost = true;ASSERT (block_group->length !=0 );spin_lock (&info->block_group_cache_lock); - p = &info->block_group_cache_tree.rb_node; + p = &info->block_group_cache_tree.rb_root.rb_node;while (*p) { parent = *p;
@@ -181,6 +182,7 @@ static int btrfs_add_block_group_cache(struct btrfs_fs_info *info, p = &(*p)->rb_left; }else if (block_group->start > cache->start) { p = &(*p)->rb_right; + leftmost = false; }else {spin_unlock (&info->block_group_cache_lock);return -EEXIST ;
@@ -188,11 +190,8 @@ static int btrfs_add_block_group_cache(struct btrfs_fs_info *info, }rb_link_node (&block_group->cache_node, parent, p); -rb_insert_color (&block_group->cache_node, - &info->block_group_cache_tree); - -if (info->first_logical_byte > block_group->start) - info->first_logical_byte = block_group->start; +rb_insert_color_cached (&block_group->cache_node, + &info->block_group_cache_tree, leftmost);spin_unlock (&info->block_group_cache_lock);
@@ -211,7 +210,7 @@ static struct btrfs_block_group *block_group_cache_tree_search(u64 end, start;spin_lock (&info->block_group_cache_lock); - n = info->block_group_cache_tree.rb_node; + n = info->block_group_cache_tree.rb_root.rb_node;while (n) { cache = rb_entry(n, struct btrfs_block_group, cache_node);
@@ -233,11 +232,8 @@ static struct btrfs_block_group *block_group_cache_tree_search(break ; } } -if (ret) { +if (ret)btrfs_get_block_group (ret); -if (bytenr ==0 && info->first_logical_byte > ret->start) - info->first_logical_byte = ret->start; - }spin_unlock (&info->block_group_cache_lock);return ret;
@@ -958,15 +954,13 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, goto out;spin_lock (&fs_info->block_group_cache_lock); -rb_erase (&block_group->cache_node, - &fs_info->block_group_cache_tree); +rb_erase_cached (&block_group->cache_node, + &fs_info->block_group_cache_tree);RB_CLEAR_NODE (&block_group->cache_node);/* Once for the block groups rbtree */ btrfs_put_block_group (block_group); -if (fs_info->first_logical_byte == block_group->start) - fs_info->first_logical_byte = (u64)-1 ;spin_unlock (&fs_info->block_group_cache_lock);down_write (&block_group->space_info->groups_sem);
@@ -4014,11 +4008,11 @@ int btrfs_free_block_groups(struct btrfs_fs_info *info)spin_unlock (&info->zone_active_bgs_lock);spin_lock (&info->block_group_cache_lock); -while ((n =rb_last (&info->block_group_cache_tree)) !=NULL ) { +while ((n =rb_last (&info->block_group_cache_tree.rb_root)) !=NULL ) { block_group = rb_entry(n, struct btrfs_block_group, cache_node); -rb_erase (&block_group->cache_node, - &info->block_group_cache_tree); +rb_erase_cached (&block_group->cache_node, + &info->block_group_cache_tree);RB_CLEAR_NODE (&block_group->cache_node);spin_unlock (&info->block_group_cache_lock);
fs/btrfs/ctree.h +1 -2
@@ -680,8 +680,7 @@ struct btrfs_fs_info {/* block group cache stuff */ spinlock_t block_group_cache_lock; -u64 first_logical_byte; -struct rb_root block_group_cache_tree; +struct rb_root_cached block_group_cache_tree;/* keep track of unallocated space */ atomic64_t free_chunk_space;
fs/btrfs/disk-io.c +1 -2
@@ -3231,8 +3231,7 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info)btrfs_init_async_reclaim_work (fs_info);spin_lock_init (&fs_info->block_group_cache_lock); - fs_info->block_group_cache_tree =RB_ROOT ; - fs_info->first_logical_byte = (u64)-1 ; + fs_info->block_group_cache_tree =RB_ROOT_CACHED ;extent_io_tree_init (fs_info, &fs_info->excluded_extents,IO_TREE_FS_EXCLUDED_EXTENTS ,NULL );
fs/btrfs/extent-tree.c +9 -13
@@ -2494,23 +2494,19 @@ static u64 get_alloc_profile_by_root(struct btrfs_root *root, int data)static u64 first_logical_byte (struct btrfs_fs_info *fs_info) { -struct btrfs_block_group *cache; -u64 bytenr; +struct rb_node *leftmost; +u64 bytenr =0 ;spin_lock (&fs_info->block_group_cache_lock); - bytenr = fs_info->first_logical_byte; -spin_unlock (&fs_info->block_group_cache_lock); - -if (bytenr < (u64)-1 ) -return bytenr; -/* Get the block group with the lowest logical start address. */ - cache =btrfs_lookup_first_block_group (fs_info,0 ); -if (!cache) -return 0 ; + leftmost =rb_first_cached (&fs_info->block_group_cache_tree); +if (leftmost) { +struct btrfs_block_group *bg; - bytenr = cache->start; -btrfs_put_block_group (cache); + bg = rb_entry(leftmost, struct btrfs_block_group, cache_node); + bytenr = bg->start; + } +spin_unlock (&fs_info->block_group_cache_lock);return bytenr; }
fs/btrfs/free-space-cache.c +1 -1
@@ -4072,7 +4072,7 @@ static int cleanup_free_space_cache_v1(struct btrfs_fs_info *fs_info,btrfs_info (fs_info,"cleaning free space cache v1" ); - node =rb_first (&fs_info->block_group_cache_tree); + node =rb_first_cached (&fs_info->block_group_cache_tree);while (node) { block_group = rb_entry(node, struct btrfs_block_group, cache_node); ret =btrfs_remove_free_space_inode (trans,NULL , block_group);
fs/btrfs/free-space-tree.c +1 -1
@@ -1178,7 +1178,7 @@ int btrfs_create_free_space_tree(struct btrfs_fs_info *fs_info) goto abort; } - node =rb_first (&fs_info->block_group_cache_tree); + node =rb_first_cached (&fs_info->block_group_cache_tree);while (node) { block_group = rb_entry(node, struct btrfs_block_group, cache_node);
Signature · 1 line
2.35.1
PATCH 3/5 btrfs: use a read/write lock for protecting the block groups tree
In reply to [email protected]From: Filipe Manana <[email protected]>
Currently we use a spin lock to protect the red black tree that we use to track block groups. Most accesses to that tree are actually read only and for large filesystems, with thousands of block groups, it actually has a bad impact on performance, as concurrent read only searches on the tree are serialized.
Read only searches on the tree are very frequent and done when:
1) Pinning and unpinning extents, as we need to lookup the respective block group from the tree;
2) Freeing the last reference of a tree block, regardless if we pin the underlying extent or add it back to free space cache/tree;
3) During NOCOW writes, both buffered IO and direct IO, we need to check if the block group that contains an extent is read only or not and to increment the number of NOCOW writers in the block group. For those operations we need to search for the block group in the tree. Similarly, after creating the ordered extent for the NOCOW write, we need to decrement the number of NOCOW writers from the same block group, which requires searching for it in the tree;
4) Decreasing the number of extent revervations in a block group;
5) When allocating extents and freeing reserved extents;
6) Adding and removing free space to the free space tree;
7) When releasing delalloc bytes during ordered extent completion;
8) When relocating a block group;
9) During fitrim, to iterate over the block groups;
10) etc;
Write accesses to the tree, to add or remove block groups, are much less frequent as they happen only when allocating a new block group or when deleting a block group.
We also use the same spin lock to protect the list of currently caching block groups. Additions to this list are made when we need to cache a block group, because we don't have a free space cache for it (or we have but it's invalid), and removals from this list are done when caching of the block group's free space finishes. These cases are also not very common, but when they happen, they happen only once when the filesystem is mounted.
So switch the lock that protects the tree of block groups from a spinning lock to a read/write lock.
--- fs/btrfs/block-group.c | 40 ++++++++++++++++++++-------------------- fs/btrfs/ctree.h | 2 +- fs/btrfs/disk-io.c | 2 +- fs/btrfs/extent-tree.c | 4 ++-- fs/btrfs/transaction.c | 4 ++-- 5 files changed, 26 insertions(+), 26 deletions(-)
fs/btrfs/block-group.c +20 -20
@@ -172,7 +172,7 @@ static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,ASSERT (block_group->length !=0 ); -spin_lock (&info->block_group_cache_lock); +write_lock (&info->block_group_cache_lock); p = &info->block_group_cache_tree.rb_root.rb_node;while (*p) {
@@ -184,7 +184,7 @@ static int btrfs_add_block_group_cache(struct btrfs_fs_info *info, p = &(*p)->rb_right; leftmost = false; }else { -spin_unlock (&info->block_group_cache_lock); +write_unlock (&info->block_group_cache_lock);return -EEXIST ; } }
@@ -193,7 +193,7 @@ static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,rb_insert_color_cached (&block_group->cache_node, &info->block_group_cache_tree, leftmost); -spin_unlock (&info->block_group_cache_lock); +write_unlock (&info->block_group_cache_lock);return 0 ; }
@@ -209,7 +209,7 @@ static struct btrfs_block_group *block_group_cache_tree_search(struct rb_node *n;u64 end, start; -spin_lock (&info->block_group_cache_lock); +read_lock (&info->block_group_cache_lock); n = info->block_group_cache_tree.rb_root.rb_node;while (n) {
@@ -234,7 +234,7 @@ static struct btrfs_block_group *block_group_cache_tree_search( }if (ret)btrfs_get_block_group (ret); -spin_unlock (&info->block_group_cache_lock); +read_unlock (&info->block_group_cache_lock);return ret; }
@@ -263,13 +263,13 @@ struct btrfs_block_group *btrfs_next_block_group(struct btrfs_fs_info *fs_info = cache->fs_info;struct rb_node *node; -spin_lock (&fs_info->block_group_cache_lock); +read_lock (&fs_info->block_group_cache_lock);/* If our block group was removed, we need a full search. */ if (RB_EMPTY_NODE (&cache->cache_node)) {const u64 next_bytenr = cache->start + cache->length; -spin_unlock (&fs_info->block_group_cache_lock); +read_unlock (&fs_info->block_group_cache_lock);btrfs_put_block_group (cache); cache =btrfs_lookup_first_block_group (fs_info, next_bytenr);return cache; }
@@ -280,7 +280,7 @@ struct btrfs_block_group *btrfs_next_block_group(btrfs_get_block_group (cache); }else cache =NULL ; -spin_unlock (&fs_info->block_group_cache_lock); +read_unlock (&fs_info->block_group_cache_lock);return cache; }
@@ -768,10 +768,10 @@ int btrfs_cache_block_group(struct btrfs_block_group *cache, int load_cache_only cache->has_caching_ctl =1 ;spin_unlock (&cache->lock); -spin_lock (&fs_info->block_group_cache_lock); +write_lock (&fs_info->block_group_cache_lock);refcount_inc (&caching_ctl->count);list_add_tail (&caching_ctl->list, &fs_info->caching_block_groups); -spin_unlock (&fs_info->block_group_cache_lock); +write_unlock (&fs_info->block_group_cache_lock);btrfs_get_block_group (cache);
@@ -953,7 +953,7 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans,if (ret) goto out; -spin_lock (&fs_info->block_group_cache_lock); +write_lock (&fs_info->block_group_cache_lock);rb_erase_cached (&block_group->cache_node, &fs_info->block_group_cache_tree);RB_CLEAR_NODE (&block_group->cache_node);
@@ -961,7 +961,7 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans,/* Once for the block groups rbtree */ btrfs_put_block_group (block_group); -spin_unlock (&fs_info->block_group_cache_lock); +write_unlock (&fs_info->block_group_cache_lock);down_write (&block_group->space_info->groups_sem);/*
@@ -986,7 +986,7 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans,if (block_group->cached ==BTRFS_CACHE_STARTED )btrfs_wait_block_group_cache_done (block_group);if (block_group->has_caching_ctl) { -spin_lock (&fs_info->block_group_cache_lock); +write_lock (&fs_info->block_group_cache_lock);if (!caching_ctl) {struct btrfs_caching_control *ctl;
@@ -1000,7 +1000,7 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, }if (caching_ctl)list_del_init (&caching_ctl->list); -spin_unlock (&fs_info->block_group_cache_lock); +write_unlock (&fs_info->block_group_cache_lock);if (caching_ctl) {/* Once for the caching bgs list and once for us. */ btrfs_put_caching_control (caching_ctl);
@@ -3970,14 +3970,14 @@ int btrfs_free_block_groups(struct btrfs_fs_info *info)struct btrfs_caching_control *caching_ctl;struct rb_node *n; -spin_lock (&info->block_group_cache_lock); +write_lock (&info->block_group_cache_lock);while (!list_empty (&info->caching_block_groups)) { caching_ctl =list_entry (info->caching_block_groups.next, struct btrfs_caching_control, list);list_del (&caching_ctl->list);btrfs_put_caching_control (caching_ctl); } -spin_unlock (&info->block_group_cache_lock); +write_unlock (&info->block_group_cache_lock);spin_lock (&info->unused_bgs_lock);while (!list_empty (&info->unused_bgs)) {
@@ -4007,14 +4007,14 @@ int btrfs_free_block_groups(struct btrfs_fs_info *info) }spin_unlock (&info->zone_active_bgs_lock); -spin_lock (&info->block_group_cache_lock); +write_lock (&info->block_group_cache_lock);while ((n =rb_last (&info->block_group_cache_tree.rb_root)) !=NULL ) { block_group = rb_entry(n, struct btrfs_block_group, cache_node);rb_erase_cached (&block_group->cache_node, &info->block_group_cache_tree);RB_CLEAR_NODE (&block_group->cache_node); -spin_unlock (&info->block_group_cache_lock); +write_unlock (&info->block_group_cache_lock);down_write (&block_group->space_info->groups_sem);list_del (&block_group->list);
@@ -4037,9 +4037,9 @@ int btrfs_free_block_groups(struct btrfs_fs_info *info)ASSERT (block_group->swap_extents ==0 );btrfs_put_block_group (block_group); -spin_lock (&info->block_group_cache_lock); +write_lock (&info->block_group_cache_lock); } -spin_unlock (&info->block_group_cache_lock); +write_unlock (&info->block_group_cache_lock);btrfs_release_global_block_rsv (info);
fs/btrfs/ctree.h +1 -1
@@ -679,7 +679,7 @@ struct btrfs_fs_info {struct radix_tree_root fs_roots_radix;/* block group cache stuff */ -spinlock_t block_group_cache_lock; +rwlock_t block_group_cache_lock;struct rb_root_cached block_group_cache_tree;/* keep track of unallocated space */
fs/btrfs/disk-io.c +1 -1
@@ -3230,7 +3230,7 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info)btrfs_init_balance (fs_info);btrfs_init_async_reclaim_work (fs_info); -spin_lock_init (&fs_info->block_group_cache_lock); +rwlock_init (&fs_info->block_group_cache_lock); fs_info->block_group_cache_tree =RB_ROOT_CACHED ; extent_io_tree_init(fs_info, &fs_info->excluded_extents,
fs/btrfs/extent-tree.c +2 -2
@@ -2497,7 +2497,7 @@ static u64 first_logical_byte(struct btrfs_fs_info *fs_info)struct rb_node *leftmost;u64 bytenr =0 ; -spin_lock (&fs_info->block_group_cache_lock); +read_lock (&fs_info->block_group_cache_lock);/* Get the block group with the lowest logical start address. */ leftmost =rb_first_cached (&fs_info->block_group_cache_tree);if (leftmost) {
@@ -2506,7 +2506,7 @@ static u64 first_logical_byte(struct btrfs_fs_info *fs_info) bg = rb_entry(leftmost, struct btrfs_block_group, cache_node); bytenr = bg->start; } -spin_unlock (&fs_info->block_group_cache_lock); +read_unlock (&fs_info->block_group_cache_lock);return bytenr; }
fs/btrfs/transaction.c +2 -2
@@ -221,7 +221,7 @@ static noinline void switch_commit_roots(struct btrfs_trans_handle *trans)* the caching thread will re-start it's search from 3, and thus find * the hole from [4,6) to add to the free space cache. */ -spin_lock (&fs_info->block_group_cache_lock); +write_lock (&fs_info->block_group_cache_lock);list_for_each_entry_safe (caching_ctl, next, &fs_info->caching_block_groups, list) {struct btrfs_block_group *cache = caching_ctl->block_group;
@@ -234,7 +234,7 @@ static noinline void switch_commit_roots(struct btrfs_trans_handle *trans) cache->last_byte_to_unpin = caching_ctl->progress; } } -spin_unlock (&fs_info->block_group_cache_lock); +write_unlock (&fs_info->block_group_cache_lock);up_write (&fs_info->commit_root_sem); }
Signature · 1 line
2.35.1
PATCH 1/5 btrfs: remove search start argument from first_logical_byte()
In reply to [email protected]From: Filipe Manana <[email protected]>
The search start argument passed to first_logical_byte() is always 0, as we always want to get the logical start address of the block group with the lowest logical start address. So remove it, as not only it is not necessary, it also makes the following patches that change the lock that protects the red black tree of block groups from a spin lock to a read/write lock.
--- fs/btrfs/extent-tree.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-)
fs/btrfs/extent-tree.c +4 -3
@@ -2492,7 +2492,7 @@ static u64 get_alloc_profile_by_root(struct btrfs_root *root, int data)return ret; } -static u64 first_logical_byte (struct btrfs_fs_info *fs_info,u64 search_start) +static u64 first_logical_byte (struct btrfs_fs_info *fs_info) {struct btrfs_block_group *cache;u64 bytenr;
@@ -2504,7 +2504,8 @@ static u64 first_logical_byte(struct btrfs_fs_info *fs_info, u64 search_start)if (bytenr < (u64)-1 )return bytenr; - cache =btrfs_lookup_first_block_group (fs_info, search_start); +/* Get the block group with the lowest logical start address. */ + cache =btrfs_lookup_first_block_group (fs_info,0 );if (!cache)return 0 ;
@@ -4267,7 +4268,7 @@ static noinline int find_free_extent(struct btrfs_root *root,return ret; ffe_ctl->search_start =max (ffe_ctl->search_start, -first_logical_byte (fs_info,0 )); +first_logical_byte (fs_info)); ffe_ctl->search_start =max (ffe_ctl->search_start, ffe_ctl->hint_byte);if (ffe_ctl->search_start == ffe_ctl->hint_byte) { block_group = btrfs_lookup_block_group(fs_info,
Signature · 1 line
2.35.1
PATCH 4/5 btrfs: return block group directly at btrfs_next_block_group()
In reply to [email protected]From: Filipe Manana <[email protected]>
At btrfs_next_block_group(), we have this long line with two statements:
cache = btrfs_lookup_first_block_group(...); return cache;
This makes it a bit harder to read due to two statements on the same line, so change that to directly return the result of the call to btrfs_lookup_first_block_group().
--- fs/btrfs/block-group.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
fs/btrfs/block-group.c +1 -1
@@ -271,7 +271,7 @@ struct btrfs_block_group *btrfs_next_block_group(read_unlock (&fs_info->block_group_cache_lock);btrfs_put_block_group (cache); - cache =btrfs_lookup_first_block_group (fs_info, next_bytenr);return cache; +return btrfs_lookup_first_block_group (fs_info, next_bytenr); } node =rb_next (&cache->cache_node);btrfs_put_block_group (cache);
Signature · 1 line
2.35.1
PATCH 5/5 btrfs: avoid double search for block group during NOCOW writes
In reply to [email protected]From: Filipe Manana <[email protected]>
When doing a NOCOW write, either through direct IO or buffered IO, we do two lookups for the block group that contains the target extent: once when we call btrfs_inc_nocow_writers() and then later again when we call btrfs_dec_nocow_writers() after creating the ordered extent.
The lookups require taking a lock and navigating the red black tree used to track all block groups, which can take a non-negligible amount of time for a large filesystem with thousands of block groups, as well as lock contention and cache line bouncing.
Improve on this by having a single block group search: making btrfs_inc_nocow_writers() return the block group to its caller and then have the caller pass that block group to btrfs_dec_nocow_writers().
This is part of a patchset comprised of the following patches:
btrfs: remove search start argument from first_logical_byte()
btrfs: use rbtree with leftmost node cached for tracking lowest block group
btrfs: use a read/write lock for protecting the block groups tree
btrfs: return block group directly at btrfs_next_block_group()
btrfs: avoid double search for block group during NOCOW writes
The following test was used to test these changes from a performance perspective:
$ cat test.sh #!/bin/bash
modprobe null_blk nr_devices=0
NULL_DEV_PATH=/sys/kernel/config/nullb/nullb0
mkdir $NULL_DEV_PATH
if [ $? -ne 0 ]; then
echo "Failed to create nullb0 directory."
exit 1
fi
echo 2 > $NULL_DEV_PATH/submit_queues
echo 16384 > $NULL_DEV_PATH/size # 16G
echo 1 > $NULL_DEV_PATH/memory_backed
echo 1 > $NULL_DEV_PATH/power
DEV=/dev/nullb0
MNT=/mnt/nullb0
LOOP_MNT="$MNT/loop"
MOUNT_OPTIONS="-o ssd -o nodatacow"
MKFS_OPTIONS="-R free-space-tree -O no-holes"
cat <<EOF > /tmp/fio-job.ini
[io_uring_writes]
rw=randwrite
fsync=0
fallocate=posix
group_reporting=1
direct=1
ioengine=io_uring
iodepth=64
bs=64k
filesize=1g
runtime=300
time_based
directory=$LOOP_MNT
numjobs=8
thread
EOF
echo performance | \
tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor echo
echo "Using config:"
echo
cat /tmp/fio-job.ini
echo
umount $MNT &> /dev/null
mkfs.btrfs -f $MKFS_OPTIONS $DEV &> /dev/null
mount $MOUNT_OPTIONS $DEV $MNT
mkdir $LOOP_MNT
truncate -s 4T $MNT/loopfile
mkfs.btrfs -f $MKFS_OPTIONS $MNT/loopfile &> /dev/null mount $MOUNT_OPTIONS $MNT/loopfile $LOOP_MNT
# Trigger the allocation of about 3500 data block groups, without
# actually consuming space on underlying filesystem, just to make # the tree of block group large. fallocate -l 3500G $LOOP_MNT/filler
fio /tmp/fio-job.ini
umount $LOOP_MNT
umount $MNT
echo 0 > $NULL_DEV_PATH/power
rmdir $NULL_DEV_PATH
The test was run on a non-debug kernel (Debian't default kernel config), the result were the following.
Before patchset:
WRITE: bw=1455MiB/s (1526MB/s), 1455MiB/s-1455MiB/s (1526MB/s-1526MB/s), io=426GiB (458GB), run=300006-300006msec
After patchset:
WRITE: bw=1503MiB/s (1577MB/s), 1503MiB/s-1503MiB/s (1577MB/s-1577MB/s), io=440GiB (473GB), run=300006-300006msec
+3.3% write throughput and +3.3% IO done in the same time period.
The test has somewhat limited coverage scope, as with only NOCOW writes we get less contention on the red black tree of block groups, since we don't have the extra contention caused by COW writes, namely when allocating data extents, pinning and unpinning data extents, but on the hand there's access to tree in the NOCOW path, when incrementing a block group's number of NOCOW writers.
--- fs/btrfs/block-group.c | 58 +++++++++++++++++++++++++++++------------- fs/btrfs/block-group.h | 5 ++-- fs/btrfs/inode.c | 26 +++++++++++-------- 3 files changed, 60 insertions(+), 29 deletions(-)
fs/btrfs/block-group.c +41 -17
@@ -284,42 +284,66 @@ struct btrfs_block_group *btrfs_next_block_group(return cache; } -bool btrfs_inc_nocow_writers (struct btrfs_fs_info *fs_info,u64 bytenr) +/* +* Check if we can do a NOCOW write for a given extent. +* +* @fs_info: The filesystem information object. +* @bytenr: Logical start address of the extent. +* +* Check if we can do a NOCOW write for the given extent, and increments the +* number of NOCOW writers in the block group that contains the extent, as long +* as the block group exists and it's currently not in read-only mode. +* +* Returns: A non-NULL block group pointer if we can do a NOCOW write, the caller +* is responsible for calling btrfs_dec_nocow_writers() later. +* +* Or NULL if we can not do a NOCOW write +*/ +struct btrfs_block_group *btrfs_inc_nocow_writers (struct btrfs_fs_info *fs_info, +u64 bytenr) {struct btrfs_block_group *bg; -bool ret = true; +bool can_nocow = true; bg =btrfs_lookup_block_group (fs_info, bytenr);if (!bg) -return false; +return NULL ;spin_lock (&bg->lock);if (bg->ro) - ret = false; + can_nocow = false;else atomic_inc (&bg->nocow_writers);spin_unlock (&bg->lock); -/* No put on block group, done by btrfs_dec_nocow_writers */ -if (!ret) +if (!can_nocow) {btrfs_put_block_group (bg); +return NULL ; + } -return ret; +/* No put on block group, done by btrfs_dec_nocow_writers(). */ +return bg; } -void btrfs_dec_nocow_writers (struct btrfs_fs_info *fs_info,u64 bytenr) +/* +* Decrement the number of NOCOW writers in a block group. +* +* @bg: The block group. +* +* This is meant to be called after a previous call to btrfs_inc_nocow_writers(), +* and on the block group returned by that call. Typically this is called after +* creating an ordered extent for a NOCOW write, to prevent races with scrub and +* relocation. +* +* After this call, the caller should not use the block group anymore. It it wants +* to use it, then it should get a reference on it before calling this function. +*/ +void btrfs_dec_nocow_writers (struct btrfs_block_group *bg) { -struct btrfs_block_group *bg; - - bg =btrfs_lookup_block_group (fs_info, bytenr); -ASSERT (bg);if (atomic_dec_and_test (&bg->nocow_writers))wake_up_var (&bg->nocow_writers); -/* -* Once for our lookup and once for the lookup done by a previous call -* to btrfs_inc_nocow_writers() -*/ -btrfs_put_block_group (bg); + +/* For the lookup done by a previous call to btrfs_inc_nocow_writers(). */ btrfs_put_block_group (bg); }
fs/btrfs/block-group.h +3 -2
@@ -254,8 +254,9 @@ void btrfs_put_block_group(struct btrfs_block_group *cache);void btrfs_dec_block_group_reservations (struct btrfs_fs_info *fs_info,const u64 start);void btrfs_wait_block_group_reservations (struct btrfs_block_group *bg); -bool btrfs_inc_nocow_writers (struct btrfs_fs_info *fs_info,u64 bytenr); -void btrfs_dec_nocow_writers (struct btrfs_fs_info *fs_info,u64 bytenr); +struct btrfs_block_group *btrfs_inc_nocow_writers (struct btrfs_fs_info *fs_info, +u64 bytenr); +void btrfs_dec_nocow_writers (struct btrfs_block_group *bg);void btrfs_wait_nocow_writers (struct btrfs_block_group *bg);void btrfs_wait_block_group_cache_progress (struct btrfs_block_group *cache,u64 num_bytes);
fs/btrfs/inode.c +16 -10
@@ -1784,6 +1784,7 @@ static noinline int run_delalloc_nocow(struct btrfs_inode *inode,int ret;bool check_prev = true;u64 ino =btrfs_ino (inode); +struct btrfs_block_group *bg;bool nocow = false;struct can_nocow_file_extent_args nocow_args = {0 };
@@ -1912,7 +1913,8 @@ static noinline int run_delalloc_nocow(struct btrfs_inode *inode, } ret =0 ; -if (btrfs_inc_nocow_writers (fs_info, nocow_args.disk_bytenr)) + bg =btrfs_inc_nocow_writers (fs_info, nocow_args.disk_bytenr); +if (bg) nocow = true; out_check:/*
@@ -1988,9 +1990,10 @@ static noinline int run_delalloc_nocow(struct btrfs_inode *inode, goto error; } -if (nocow) -btrfs_dec_nocow_writers (fs_info, nocow_args.disk_bytenr); - nocow = false; +if (nocow) { +btrfs_dec_nocow_writers (bg); + nocow = false; + }if (btrfs_is_data_reloc_root (root))/*
@@ -2034,7 +2037,7 @@ static noinline int run_delalloc_nocow(struct btrfs_inode *inode, error:if (nocow) -btrfs_dec_nocow_writers (fs_info, nocow_args.disk_bytenr); +btrfs_dec_nocow_writers (bg);if (ret && cur_offset < end) extent_clear_unlock_delalloc(inode, cur_offset, end,
@@ -7428,6 +7431,7 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map,struct extent_map *em = *map;int type;u64 block_start, orig_start, orig_block_len, ram_bytes; +struct btrfs_block_group *bg;bool can_nocow = false;bool space_reserved = false;u64 prev_len;
@@ -7453,9 +7457,11 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, block_start = em->block_start + (start - em->start);if (can_nocow_extent (inode, start, &len, &orig_start, - &orig_block_len, &ram_bytes, false) ==1 && -btrfs_inc_nocow_writers (fs_info, block_start)) - can_nocow = true; + &orig_block_len, &ram_bytes, false) ==1 ) { + bg =btrfs_inc_nocow_writers (fs_info, block_start); +if (bg) + can_nocow = true; + } } prev_len = len;
@@ -7469,7 +7475,7 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map,/* Our caller expects us to free the input extent map. */ free_extent_map (em); *map =NULL ; -btrfs_dec_nocow_writers (fs_info, block_start); +btrfs_dec_nocow_writers (bg);if (nowait && (ret == -ENOSPC || ret == -EDQUOT )) ret = -EAGAIN ; goto out;
@@ -7480,7 +7486,7 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, orig_start, block_start, len, orig_block_len, ram_bytes, type); -btrfs_dec_nocow_writers (fs_info, block_start); +btrfs_dec_nocow_writers (bg);if (type ==BTRFS_ORDERED_PREALLOC ) {free_extent_map (em); *map = em = em2;
Signature · 1 line
2.35.1
Quoted 14 lines · On 13.04.22 г. 18:20 ч., [email protected] wrote:
From: Filipe Manana <[email protected]>
This patchset allows for better concurrency when accessing the red black tree of block groups, which is used very frequently and most accesses are read-only, as well as avoid some unnecessary searches in the tree during NOCOW writes. Details in the changelogs.
Filipe Manana (5):
btrfs: remove search start argument fromfirst_logical_byte()
btrfs: use rbtree with leftmost node cached for tracking lowest block group
btrfs: use a read/write lock for protecting the block groups tree
btrfs: return block group directly atbtrfs_next_block_group()
btrfs: avoid double search for block group during NOCOW writes
Quoted 11 lines
fs/btrfs/block-group.c | 130 ++++++++++++++++++++---------------- fs/btrfs/block-group.h | 5 +- fs/btrfs/ctree.h | 5 +- fs/btrfs/disk-io.c | 5 +- fs/btrfs/extent-tree.c | 29 ++++---- fs/btrfs/free-space-cache.c | 2 +- fs/btrfs/free-space-tree.c | 2 +- fs/btrfs/inode.c | 26 +++++--- fs/btrfs/transaction.c | 4 +- 9 files changed, 114 insertions(+), 94 deletions(-)
Quoted 7 lines · On Wed, Apr 13, 2022 at 04:20:38PM +0100, [email protected] wrote:
From: Filipe Manana <[email protected]>
This patchset allows for better concurrency when accessing the red black tree of block groups, which is used very frequently and most accesses are read-only, as well as avoid some unnecessary searches in the tree during NOCOW writes. Details in the changelogs.
Quoted 6 lines
Filipe Manana (5):
btrfs: remove search start argument fromfirst_logical_byte()
btrfs: use rbtree with leftmost node cached for tracking lowest block group
btrfs: use a read/write lock for protecting the block groups tree
btrfs: return block group directly atbtrfs_next_block_group()
btrfs: avoid double search for block group during NOCOW writes
Added to misc-next, thanks.