Skip to contentSkip to conversation

btrfs: reduce lock contention when eb cache miss for btree search

PATCH Superseded robbieko 2 participants 5 messages Oct 9, 2024 – Oct 11, 2024 Btrfs file system

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

robbieko

author
#0 lore raw

PATCH 1/1 btrfs: reduce lock contention when eb cache miss for btree search

From: Robbie Ko <[email protected]>

When crawling btree, if an eb cache miss occurs, we change to use the eb read lock and release all previous locks to reduce lock contention.

Because we have prepared the check parameters and the read lock of eb we hold, we can ensure that no race will occur during the check and cause unexpected errors.

---
 fs/btrfs/ctree.c | 88 ++++++++++++++++++++++++++++--------------------
 1 file changed, 52 insertions(+), 36 deletions(-)
fs/btrfs/ctree.c +52 -36
index 0cc919d15b14..0efbe61497f3 100644
@@ -1515,12 +1515,12 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
 	struct btrfs_tree_parent_check check = { 0 };
 	u64 blocknr;
 	u64 gen;
-	struct extent_buffer *tmp;
-	int ret;
+	struct extent_buffer *tmp = NULL;
+	int ret, err;
 	int parent_level;
-	bool unlock_up;
+	bool create = false;
+	bool lock = false;
 
-	unlock_up = ((level + 1 < BTRFS_MAX_LEVEL) && p->locks[level + 1]);
 	blocknr = btrfs_node_blockptr(*eb_ret, slot);
 	gen = btrfs_node_ptr_generation(*eb_ret, slot);
 	parent_level = btrfs_header_level(*eb_ret);
@@ -1551,52 +1551,66 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
 			 */
 			if (btrfs_verify_level_key(tmp,
 					parent_level - 1, &check.first_key, gen)) {
-				free_extent_buffer(tmp);
-				return -EUCLEAN;
+				ret = -EUCLEAN;
+				goto out;
 			}
 			*eb_ret = tmp;
-			return 0;
+			tmp = NULL;
+			ret = 0;
+			goto out;
 		}
 
 		if (p->nowait) {
-			free_extent_buffer(tmp);
-			return -EAGAIN;
+			ret = -EAGAIN;
+			btrfs_release_path(p);
+			goto out;
 		}
 
-		if (unlock_up)
-			btrfs_unlock_up_safe(p, level + 1);
+		ret = -EAGAIN;
+		btrfs_unlock_up_safe(p, level + 1);
+		btrfs_tree_read_lock(tmp);
+		lock = true;
+		btrfs_release_path(p);
 
 		/* now we're allowed to do a blocking uptodate check */
-		ret = btrfs_read_extent_buffer(tmp, &check);
-		if (ret) {
-			free_extent_buffer(tmp);
-			btrfs_release_path(p);
-			return ret;
+		err = btrfs_read_extent_buffer(tmp, &check);
+		if (err) {
+			ret = err;
+			goto out;
 		}
-
-		if (unlock_up)
-			ret = -EAGAIN;
-
 		goto out;
 	} else if (p->nowait) {
-		return -EAGAIN;
-	}
-
-	if (unlock_up) {
-		btrfs_unlock_up_safe(p, level + 1);
 		ret = -EAGAIN;
-	} else {
-		ret = 0;
+		btrfs_release_path(p);
+		goto out;
 	}
 
+	ret = -EAGAIN;
+	btrfs_unlock_up_safe(p, level + 1);
+
 	if (p->reada != READA_NONE)
 		reada_for_search(fs_info, p, level, slot, key->objectid);
 
-	tmp = read_tree_block(fs_info, blocknr, &check);
+	tmp = btrfs_find_create_tree_block(fs_info, blocknr, check.owner_root, check.level);
 	if (IS_ERR(tmp)) {
+		ret = PTR_ERR(tmp);
+		tmp = NULL;
 		btrfs_release_path(p);
-		return PTR_ERR(tmp);
+		goto out;
 	}
+	create = true;
+
+	btrfs_tree_read_lock(tmp);
+	lock = true;
+	btrfs_release_path(p);
+
+	/* now we're allowed to do a blocking uptodate check */
+	err = btrfs_read_extent_buffer(tmp, &check);
+	if (err) {
+		ret = err;
+		goto out;
+	}
+
 	/*
 	 * If the read above didn't mark this buffer up to date,
 	 * it will never end up being up to date.  Set ret to EIO now
@@ -1607,11 +1621,13 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
 		ret = -EIO;
 
 out:
-	if (ret == 0) {
-		*eb_ret = tmp;
-	} else {
-		free_extent_buffer(tmp);
-		btrfs_release_path(p);
+	if (tmp) {
+		if (lock)
+			btrfs_tree_read_unlock(tmp);
+		if (create && ret && ret != -EAGAIN)
+			free_extent_buffer_stale(tmp);
+		else
+			free_extent_buffer(tmp);
 	}
 
 	return ret;
@@ -2198,7 +2214,7 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,
 		}
 
 		err = read_block_for_search(root, p, &b, level, slot, key);
-		if (err == -EAGAIN)
+		if (err == -EAGAIN && !p->nowait)
 			goto again;
 		if (err) {
 			ret = err;
@@ -2325,7 +2341,7 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key,
 		}
 
 		err = read_block_for_search(root, p, &b, level, slot, key);
-		if (err == -EAGAIN)
+		if (err == -EAGAIN && !p->nowait)
 			goto again;
 		if (err) {
 			ret = err;
Signature · 1 line
2.17.1
#1 lore raw
Quoted 20 lines · robbieko

From: Robbie Ko <[email protected]>

When crawling btree, if an eb cache miss occurs, we change to use the eb read lock and release all previous locks to reduce lock contention.

Because we have prepared the check parameters and the read lock of eb we hold, we can ensure that no race will occur during the check and cause unexpected errors.

---
 fs/btrfs/ctree.c | 88 ++++++++++++++++++++++++++++--------------------
 1 file changed, 52 insertions(+), 36 deletions(-)
fs/btrfs/ctree.c +0 -0
index 0cc919d15b14..0efbe61497f3 100644

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1519

Quoted 7 lines
fs/btrfs/ctree.c +2 -2
@@ -1515,12 +1515,12 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
        struct btrfs_tree_parent_check check = { 0 };
        u64 blocknr;
        u64 gen;
-       struct extent_buffer *tmp;
-       int ret;
+       struct extent_buffer *tmp = NULL;
+       int ret, err;

Please avoid declaring two (or more) variables in the same line. 1 per line is prefered for readability and be consistent with this function's code.

Quoted 13 lines
fs/btrfs/ctree.c +2 -2
        int parent_level;
-       bool unlock_up;
+       bool create = false;
+       bool lock = false;
 
-       unlock_up = ((level + 1 < BTRFS_MAX_LEVEL) && p->locks[level + 1]);
        blocknr = btrfs_node_blockptr(*eb_ret, slot);
        gen = btrfs_node_ptr_generation(*eb_ret, slot);
        parent_level = btrfs_header_level(*eb_ret);
@@ -1551,52 +1551,66 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
                         */
                        if (btrfs_verify_level_key(tmp,
                                        parent_level - 1, &check.first_key, gen)) {

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1560

Quoted 10 lines
fs/btrfs/ctree.c +5 -3
-                               free_extent_buffer(tmp);
-                               return -EUCLEAN;
+                               ret = -EUCLEAN;
+                               goto out;
                        }
                        *eb_ret = tmp;
-                       return 0;
+                       tmp = NULL;
+                       ret = 0;
+                       goto out;

By setting tmp to NULL and jumping to the out label, we leak a reference on the tmp extent buffer.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1565

Quoted 7 lines
fs/btrfs/ctree.c +2 -2
                }
 
                if (p->nowait) {
-                       free_extent_buffer(tmp);
-                       return -EAGAIN;
+                       ret = -EAGAIN;
+                       btrfs_release_path(p);

To reduce the critical section sizes, please set ret to -EAGAIN after releasing the path.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1570

Quoted 7 lines
fs/btrfs/ctree.c +3 -2
+                       goto out;
                }
 
-               if (unlock_up)
-                       btrfs_unlock_up_safe(p, level + 1);
+               ret = -EAGAIN;
+               btrfs_unlock_up_safe(p, level + 1);

Same here, set ret after the unlocking.

But why set ret to -EAGAIN? Here we know p->nowait is false.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1572

Quoted 2 lines
fs/btrfs/ctree.c +2 -0
+               btrfs_tree_read_lock(tmp);
+               lock = true;

And here, with the same reasoning, set 'lock' to true before calling btrfs_tree_read_lock().

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1579

Quoted 12 lines
fs/btrfs/ctree.c +5 -5
+               btrfs_release_path(p);
 
                /* now we're allowed to do a blocking uptodate check */
-               ret = btrfs_read_extent_buffer(tmp, &check);
-               if (ret) {
-                       free_extent_buffer(tmp);
-                       btrfs_release_path(p);
-                       return ret;
+               err = btrfs_read_extent_buffer(tmp, &check);
+               if (err) {
+                       ret = err;
+                       goto out;

Why do we need to have this 'err' variable?
Why not use 'ret' and simplify?

Quoted 6 lines
fs/btrfs/ctree.c +0 -4
                }
-
-               if (unlock_up)
-                       ret = -EAGAIN;
-
                goto out;

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1584

Quoted 10 lines
fs/btrfs/ctree.c +1 -7
        } else if (p->nowait) {
-               return -EAGAIN;
-       }
-
-       if (unlock_up) {
-               btrfs_unlock_up_safe(p, level + 1);
                ret = -EAGAIN;
-       } else {
-               ret = 0;
+               btrfs_release_path(p);

Same here, set 'ret' to -EAGAIN after releasing the path.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1570

Quoted 5 lines
fs/btrfs/ctree.c +3 -0
+               goto out;
        }
 
+       ret = -EAGAIN;
+       btrfs_unlock_up_safe(p, level + 1);

Same here.

But why set ret to -EAGAIN? Here we know p->nowait is false.

Quoted 7 lines
fs/btrfs/ctree.c +2 -1
+
        if (p->reada != READA_NONE)
                reada_for_search(fs_info, p, level, slot, key->objectid);
 
-       tmp = read_tree_block(fs_info, blocknr, &check);
+       tmp = btrfs_find_create_tree_block(fs_info, blocknr, check.owner_root, check.level);
        if (IS_ERR(tmp)) {

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1604

Quoted 10 lines
fs/btrfs/ctree.c +7 -1
+               ret = PTR_ERR(tmp);
+               tmp = NULL;
                btrfs_release_path(p);
-               return PTR_ERR(tmp);
+               goto out;
        }
+       create = true;
+
+       btrfs_tree_read_lock(tmp);
+       lock = true;

Same here, set 'lock' to true before the call to btrfs_tree_read_lock();

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1607

Quoted 3 lines
fs/btrfs/ctree.c +3 -0
+       btrfs_release_path(p);
+
+       /* now we're allowed to do a blocking uptodate check */

Please capitalize the first word of a sentence and end the sentence with punctuation.
This is the preferred style and we strive to do that for new comments or code that moves comments around.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1608

Quoted 1 line
fs/btrfs/ctree.c +1 -0
+       err = btrfs_read_extent_buffer(tmp, &check);

This can block, so if p->nowait at this point we should instead exit with -EAGAIN and not call this function.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1612

Quoted 4 lines
fs/btrfs/ctree.c +4 -0
+       if (err) {
+               ret = err;
+               goto out;
+       }

Same here, no need to use extra variable 'err', can just use 'ret'.

Quoted 10 lines
fs/btrfs/ctree.c +1 -2
+
        /*
         * If the read above didn't mark this buffer up to date,
         * it will never end up being up to date.  Set ret to EIO now
@@ -1607,11 +1621,13 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
                ret = -EIO;
 
 out:
-       if (ret == 0) {
-               *eb_ret = tmp;

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1630

Quoted 10 lines
fs/btrfs/ctree.c +7 -3
-       } else {
-               free_extent_buffer(tmp);
-               btrfs_release_path(p);
+       if (tmp) {
+               if (lock)
+                       btrfs_tree_read_unlock(tmp);
+               if (create && ret && ret != -EAGAIN)
+                       free_extent_buffer_stale(tmp);
+               else
+                       free_extent_buffer(tmp);

So in the success case we no longer set *eb_ret to tmp. Why? The callers expect that.

Thanks.

Quoted 25 lines
fs/btrfs/ctree.c +2 -3
        }
 
        return ret;
@@ -2198,7 +2214,7 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,
                }
 
                err = read_block_for_search(root, p, &b, level, slot, key);
-               if (err == -EAGAIN)
+               if (err == -EAGAIN && !p->nowait)
                        goto again;
                if (err) {
                        ret = err;
@@ -2325,7 +2341,7 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key,
                }
 
                err = read_block_for_search(root, p, &b, level, slot, key);
-               if (err == -EAGAIN)
+               if (err == -EAGAIN && !p->nowait)
                        goto again;
                if (err) {
                        ret = err;
--

2.17.1

#2 lore raw
Quoted 52 lines · Filipe Manana
Quoted 20 lines · robbieko

From: Robbie Ko <[email protected]>

When crawling btree, if an eb cache miss occurs, we change to use the eb read lock and release all previous locks to reduce lock contention.

Because we have prepared the check parameters and the read lock of eb we hold, we can ensure that no race will occur during the check and cause unexpected errors.

---
 fs/btrfs/ctree.c | 88 ++++++++++++++++++++++++++++--------------------
 1 file changed, 52 insertions(+), 36 deletions(-)
fs/btrfs/ctree.c +0 -0
index 0cc919d15b14..0efbe61497f3 100644

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1519

Quoted 7 lines
fs/btrfs/ctree.c +2 -2
@@ -1515,12 +1515,12 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
        struct btrfs_tree_parent_check check = { 0 };
        u64 blocknr;
        u64 gen;
-       struct extent_buffer *tmp;
-       int ret;
+       struct extent_buffer *tmp = NULL;
+       int ret, err;

Please avoid declaring two (or more) variables in the same line. 1 per line is prefered for readability and be consistent with this function's code.

Quoted 18 lines
fs/btrfs/ctree.c +4 -4
        int parent_level;
-       bool unlock_up;
+       bool create = false;
+       bool lock = false;
 
-       unlock_up = ((level + 1 < BTRFS_MAX_LEVEL) && p->locks[level + 1]);
        blocknr = btrfs_node_blockptr(*eb_ret, slot);
        gen = btrfs_node_ptr_generation(*eb_ret, slot);
        parent_level = btrfs_header_level(*eb_ret);
@@ -1551,52 +1551,66 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
                         */
                        if (btrfs_verify_level_key(tmp,
                                        parent_level - 1, &check.first_key, gen)) {
-                               free_extent_buffer(tmp);
-                               return -EUCLEAN;
+                               ret = -EUCLEAN;
+                               goto out;
                        }
Quoted 8 lines

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1560

Quoted 5 lines
fs/btrfs/ctree.c +3 -1
                        *eb_ret = tmp;
-                       return 0;
+                       tmp = NULL;
+                       ret = 0;
+                       goto out;

By setting tmp to NULL and jumping to the out label, we leak a reference on the tmp extent buffer.

Never mind about this one, it's the reference for the caller as we've set *eb_ret to tmp.

Quoted 167 lines

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1565

Quoted 8 lines
fs/btrfs/ctree.c +2 -2
                }
 
                if (p->nowait) {
-                       free_extent_buffer(tmp);
-                       return -EAGAIN;
+                       ret = -EAGAIN;
+                       btrfs_release_path(p);

To reduce the critical section sizes, please set ret to -EAGAIN after releasing the path.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1570

Quoted 7 lines
fs/btrfs/ctree.c +3 -2
+                       goto out;
                }
 
-               if (unlock_up)
-                       btrfs_unlock_up_safe(p, level + 1);
+               ret = -EAGAIN;
+               btrfs_unlock_up_safe(p, level + 1);

Same here, set ret after the unlocking.

But why set ret to -EAGAIN? Here we know p->nowait is false.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1572

Quoted 2 lines
fs/btrfs/ctree.c +2 -0
+               btrfs_tree_read_lock(tmp);
+               lock = true;

And here, with the same reasoning, set 'lock' to true before calling btrfs_tree_read_lock().

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1579

Quoted 12 lines
fs/btrfs/ctree.c +5 -5
+               btrfs_release_path(p);
 
                /* now we're allowed to do a blocking uptodate check */
-               ret = btrfs_read_extent_buffer(tmp, &check);
-               if (ret) {
-                       free_extent_buffer(tmp);
-                       btrfs_release_path(p);
-                       return ret;
+               err = btrfs_read_extent_buffer(tmp, &check);
+               if (err) {
+                       ret = err;
+                       goto out;

Why do we need to have this 'err' variable?
Why not use 'ret' and simplify?

Quoted 6 lines
fs/btrfs/ctree.c +0 -4
                }
-
-               if (unlock_up)
-                       ret = -EAGAIN;
-
                goto out;

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1584

Quoted 10 lines
fs/btrfs/ctree.c +1 -7
        } else if (p->nowait) {
-               return -EAGAIN;
-       }
-
-       if (unlock_up) {
-               btrfs_unlock_up_safe(p, level + 1);
                ret = -EAGAIN;
-       } else {
-               ret = 0;
+               btrfs_release_path(p);

Same here, set 'ret' to -EAGAIN after releasing the path.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1570

Quoted 5 lines
fs/btrfs/ctree.c +3 -0
+               goto out;
        }
 
+       ret = -EAGAIN;
+       btrfs_unlock_up_safe(p, level + 1);

Same here.

But why set ret to -EAGAIN? Here we know p->nowait is false.

Quoted 7 lines
fs/btrfs/ctree.c +2 -1
+
        if (p->reada != READA_NONE)
                reada_for_search(fs_info, p, level, slot, key->objectid);
 
-       tmp = read_tree_block(fs_info, blocknr, &check);
+       tmp = btrfs_find_create_tree_block(fs_info, blocknr, check.owner_root, check.level);
        if (IS_ERR(tmp)) {

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1604

Quoted 10 lines
fs/btrfs/ctree.c +7 -1
+               ret = PTR_ERR(tmp);
+               tmp = NULL;
                btrfs_release_path(p);
-               return PTR_ERR(tmp);
+               goto out;
        }
+       create = true;
+
+       btrfs_tree_read_lock(tmp);
+       lock = true;

Same here, set 'lock' to true before the call to btrfs_tree_read_lock();

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1607

Quoted 3 lines
fs/btrfs/ctree.c +3 -0
+       btrfs_release_path(p);
+
+       /* now we're allowed to do a blocking uptodate check */

Please capitalize the first word of a sentence and end the sentence with punctuation.
This is the preferred style and we strive to do that for new comments or code that moves comments around.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1608

Quoted 1 line
fs/btrfs/ctree.c +1 -0
+       err = btrfs_read_extent_buffer(tmp, &check);

This can block, so if p->nowait at this point we should instead exit with -EAGAIN and not call this function.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1612

Quoted 4 lines
fs/btrfs/ctree.c +4 -0
+       if (err) {
+               ret = err;
+               goto out;
+       }

Same here, no need to use extra variable 'err', can just use 'ret'.

Quoted 10 lines
fs/btrfs/ctree.c +1 -2
+
        /*
         * If the read above didn't mark this buffer up to date,
         * it will never end up being up to date.  Set ret to EIO now
@@ -1607,11 +1621,13 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
                ret = -EIO;
 
 out:
-       if (ret == 0) {
-               *eb_ret = tmp;

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1630

Quoted 10 lines
fs/btrfs/ctree.c +7 -3
-       } else {
-               free_extent_buffer(tmp);
-               btrfs_release_path(p);
+       if (tmp) {
+               if (lock)
+                       btrfs_tree_read_unlock(tmp);
+               if (create && ret && ret != -EAGAIN)
+                       free_extent_buffer_stale(tmp);
+               else
+                       free_extent_buffer(tmp);

So in the success case we no longer set *eb_ret to tmp. Why? The callers expect that.

Thanks.

Quoted 25 lines
fs/btrfs/ctree.c +2 -3
        }
 
        return ret;
@@ -2198,7 +2214,7 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,
                }
 
                err = read_block_for_search(root, p, &b, level, slot, key);
-               if (err == -EAGAIN)
+               if (err == -EAGAIN && !p->nowait)
                        goto again;
                if (err) {
                        ret = err;
@@ -2325,7 +2341,7 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key,
                }
 
                err = read_block_for_search(root, p, &b, level, slot, key);
-               if (err == -EAGAIN)
+               if (err == -EAGAIN && !p->nowait)
                        goto again;
                if (err) {
                        ret = err;
--

2.17.1

#3 lore raw
In reply to Filipe Manana
Quoted 75 lines · Filipe Manana
Quoted 20 lines · robbieko

From: Robbie Ko <[email protected]>

When crawling btree, if an eb cache miss occurs, we change to use the eb read lock and release all previous locks to reduce lock contention.

Because we have prepared the check parameters and the read lock of eb we hold, we can ensure that no race will occur during the check and cause unexpected errors.

---
 fs/btrfs/ctree.c | 88 ++++++++++++++++++++++++++++--------------------
 1 file changed, 52 insertions(+), 36 deletions(-)
fs/btrfs/ctree.c +0 -0
index 0cc919d15b14..0efbe61497f3 100644

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1519

Quoted 7 lines
fs/btrfs/ctree.c +2 -2
@@ -1515,12 +1515,12 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
        struct btrfs_tree_parent_check check = { 0 };
        u64 blocknr;
        u64 gen;
-       struct extent_buffer *tmp;
-       int ret;
+       struct extent_buffer *tmp = NULL;
+       int ret, err;

Please avoid declaring two (or more) variables in the same line. 1 per line is prefered for readability and be consistent with this function's code.

Quoted 13 lines
fs/btrfs/ctree.c +2 -2
        int parent_level;
-       bool unlock_up;
+       bool create = false;
+       bool lock = false;
 
-       unlock_up = ((level + 1 < BTRFS_MAX_LEVEL) && p->locks[level + 1]);
        blocknr = btrfs_node_blockptr(*eb_ret, slot);
        gen = btrfs_node_ptr_generation(*eb_ret, slot);
        parent_level = btrfs_header_level(*eb_ret);
@@ -1551,52 +1551,66 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
                         */
                        if (btrfs_verify_level_key(tmp,
                                        parent_level - 1, &check.first_key, gen)) {

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1560

Quoted 10 lines
fs/btrfs/ctree.c +5 -3
-                               free_extent_buffer(tmp);
-                               return -EUCLEAN;
+                               ret = -EUCLEAN;
+                               goto out;
                        }
                        *eb_ret = tmp;
-                       return 0;
+                       tmp = NULL;
+                       ret = 0;
+                       goto out;

By setting tmp to NULL and jumping to the out label, we leak a reference on the tmp extent buffer.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1565

Quoted 7 lines
fs/btrfs/ctree.c +2 -2
                }
 
                if (p->nowait) {
-                       free_extent_buffer(tmp);
-                       return -EAGAIN;
+                       ret = -EAGAIN;
+                       btrfs_release_path(p);

To reduce the critical section sizes, please set ret to -EAGAIN after releasing the path.

Quoted 3 lines
fs/btrfs/ctree.c +1 -0
+                       goto out;
                }
 
Quoted 8 lines

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1570

Quoted 4 lines
fs/btrfs/ctree.c +2 -2
-               if (unlock_up)
-                       btrfs_unlock_up_safe(p, level + 1);
+               ret = -EAGAIN;
+               btrfs_unlock_up_safe(p, level + 1);

Same here, set ret after the unlocking.

But why set ret to -EAGAIN? Here we know p->nowait is false.

Nevermind this is because of the unconditional unlock now.

Quoted 45 lines

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1572

Quoted 3 lines
fs/btrfs/ctree.c +2 -0
+               btrfs_tree_read_lock(tmp);
+               lock = true;

And here, with the same reasoning, set 'lock' to true before calling btrfs_tree_read_lock().

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1579

Quoted 12 lines
fs/btrfs/ctree.c +5 -5
+               btrfs_release_path(p);
 
                /* now we're allowed to do a blocking uptodate check */
-               ret = btrfs_read_extent_buffer(tmp, &check);
-               if (ret) {
-                       free_extent_buffer(tmp);
-                       btrfs_release_path(p);
-                       return ret;
+               err = btrfs_read_extent_buffer(tmp, &check);
+               if (err) {
+                       ret = err;
+                       goto out;

Why do we need to have this 'err' variable?
Why not use 'ret' and simplify?

Quoted 6 lines
fs/btrfs/ctree.c +0 -4
                }
-
-               if (unlock_up)
-                       ret = -EAGAIN;
-
                goto out;

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1584

Quoted 10 lines
fs/btrfs/ctree.c +1 -7
        } else if (p->nowait) {
-               return -EAGAIN;
-       }
-
-       if (unlock_up) {
-               btrfs_unlock_up_safe(p, level + 1);
                ret = -EAGAIN;
-       } else {
-               ret = 0;
+               btrfs_release_path(p);

Same here, set 'ret' to -EAGAIN after releasing the path.

Quoted 3 lines
fs/btrfs/ctree.c +1 -0
+               goto out;
        }
 
Quoted 6 lines

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1589

Quoted 2 lines
fs/btrfs/ctree.c +2 -0
+       ret = -EAGAIN;
+       btrfs_unlock_up_safe(p, level + 1);

Same here.

But why set ret to -EAGAIN? Here we know p->nowait is false.

Nevermind this is because of the unconditional unlock now.

In both cases we still don't need the 'err' variable.
Just set 'ret' to -EAGAIN if btrfs_find_create_tree_block() below doesn't return an error and btrfs_read_extent_buffer() below and above don't return an error.

Quoted 49 lines
Quoted 8 lines
fs/btrfs/ctree.c +2 -1
+
        if (p->reada != READA_NONE)
                reada_for_search(fs_info, p, level, slot, key->objectid);
 
-       tmp = read_tree_block(fs_info, blocknr, &check);
+       tmp = btrfs_find_create_tree_block(fs_info, blocknr, check.owner_root, check.level);
        if (IS_ERR(tmp)) {

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1604

Quoted 10 lines
fs/btrfs/ctree.c +7 -1
+               ret = PTR_ERR(tmp);
+               tmp = NULL;
                btrfs_release_path(p);
-               return PTR_ERR(tmp);
+               goto out;
        }
+       create = true;
+
+       btrfs_tree_read_lock(tmp);
+       lock = true;

Same here, set 'lock' to true before the call to btrfs_tree_read_lock();

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1607

Quoted 3 lines
fs/btrfs/ctree.c +3 -0
+       btrfs_release_path(p);
+
+       /* now we're allowed to do a blocking uptodate check */

Please capitalize the first word of a sentence and end the sentence with punctuation.
This is the preferred style and we strive to do that for new comments or code that moves comments around.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1608

Quoted 1 line
fs/btrfs/ctree.c +1 -0
+       err = btrfs_read_extent_buffer(tmp, &check);

This can block, so if p->nowait at this point we should instead exit with -EAGAIN and not call this function.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1612

Quoted 4 lines
fs/btrfs/ctree.c +4 -0
+       if (err) {
+               ret = err;
+               goto out;
+       }

Same here, no need to use extra variable 'err', can just use 'ret'.

Quoted 7 lines
fs/btrfs/ctree.c +1 -0
+
        /*
         * If the read above didn't mark this buffer up to date,
         * it will never end up being up to date.  Set ret to EIO now
@@ -1607,11 +1621,13 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
                ret = -EIO;
 
Quoted 16 lines

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1630

Quoted 13 lines
fs/btrfs/ctree.c +7 -5
 out:
-       if (ret == 0) {
-               *eb_ret = tmp;
-       } else {
-               free_extent_buffer(tmp);
-               btrfs_release_path(p);
+       if (tmp) {
+               if (lock)
+                       btrfs_tree_read_unlock(tmp);
+               if (create && ret && ret != -EAGAIN)
+                       free_extent_buffer_stale(tmp);
+               else
+                       free_extent_buffer(tmp);

So in the success case we no longer set *eb_ret to tmp. Why? The callers expect that.

Ok, it's because of the -EAGAIN due to having the unconditional unlock now.

And btw, have you observed any case where this improved anything? Any benchmarks?

I can't see how this can make anything better because this function is never called for a root node, and therefore level + 1 < BTRFS_MAX_LEVEL.

But there is one case where this patch causes unnecessary path releases and makes the caller retry a search:
when none of the levels above were locked. That's why we had the following expression before:

unlock_up = ((level + 1 < BTRFS_MAX_LEVEL) && p->locks[level + 1]);

So we wouldn't make the caller retry the search if upper levels aren't locked or p->skip_locking == 1.
But now after this patch we make the caller retry the search in that case.

Thanks.

Quoted 28 lines

Thanks.

Quoted 25 lines
fs/btrfs/ctree.c +2 -3
        }
 
        return ret;
@@ -2198,7 +2214,7 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,
                }
 
                err = read_block_for_search(root, p, &b, level, slot, key);
-               if (err == -EAGAIN)
+               if (err == -EAGAIN && !p->nowait)
                        goto again;
                if (err) {
                        ret = err;
@@ -2325,7 +2341,7 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key,
                }
 
                err = read_block_for_search(root, p, &b, level, slot, key);
-               if (err == -EAGAIN)
+               if (err == -EAGAIN && !p->nowait)
                        goto again;
                if (err) {
                        ret = err;
--

2.17.1

author
#4 lore raw

Filipe Manana 於 2024/10/9 下午11:09 寫道:

Quoted 16 lines · Filipe Manana
Quoted 15 lines · robbieko
Quoted 14 lines

From: Robbie Ko <[email protected]>

When crawling btree, if an eb cache miss occurs, we change to use the eb read lock and release all previous locks to reduce lock contention.

Because we have prepared the check parameters and the read lock of eb we hold, we can ensure that no race will occur during the check and cause unexpected errors.

---
  fs/btrfs/ctree.c | 88 ++++++++++++++++++++++++++++--------------------
  1 file changed, 52 insertions(+), 36 deletions(-)
Quoted 15 lines
Quoted 7 lines
Quoted 7 lines
fs/btrfs/ctree.c +0 -0
index 0cc919d15b14..0efbe61497f3 100644
@@ -1515,12 +1515,12 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
         struct btrfs_tree_parent_check check = { 0 };
         u64 blocknr;
Quoted 8 lines

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1519

Quoted 5 lines
fs/btrfs/ctree.c +2 -2
         u64 gen;
-       struct extent_buffer *tmp;
-       int ret;
+       struct extent_buffer *tmp = NULL;
+       int ret, err;

Please avoid declaring two (or more) variables in the same line. 1 per line is prefered for readability and be consistent with this function's code.

Okay, I'll modify it.

Quoted 18 lines
Quoted 18 lines
Quoted 18 lines
fs/btrfs/ctree.c +4 -4
         int parent_level;
-       bool unlock_up;
+       bool create = false;
+       bool lock = false;
 
-       unlock_up = ((level + 1 < BTRFS_MAX_LEVEL) && p->locks[level + 1]);
         blocknr = btrfs_node_blockptr(*eb_ret, slot);
         gen = btrfs_node_ptr_generation(*eb_ret, slot);
         parent_level = btrfs_header_level(*eb_ret);
@@ -1551,52 +1551,66 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
                          */
                         if (btrfs_verify_level_key(tmp,
                                         parent_level - 1, &check.first_key, gen)) {
-                               free_extent_buffer(tmp);
-                               return -EUCLEAN;
+                               ret = -EUCLEAN;
+                               goto out;
Quoted 8 lines
Quoted 8 lines

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1560

Quoted 6 lines
fs/btrfs/ctree.c +3 -1
                         }
                         *eb_ret = tmp;
-                       return 0;
+                       tmp = NULL;
+                       ret = 0;
+                       goto out;

By setting tmp to NULL and jumping to the out label, we leak a reference on the tmp extent buffer.

We assign tmp to eb_ret, so we don't leak.

Quoted 10 lines
Quoted 10 lines

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1565

Quoted 8 lines
fs/btrfs/ctree.c +2 -2
                 }
 
                 if (p->nowait) {
-                       free_extent_buffer(tmp);
-                       return -EAGAIN;
+                       ret = -EAGAIN;
+                       btrfs_release_path(p);

To reduce the critical section sizes, please set ret to -EAGAIN after releasing the path.

Okay, I'll modify it.

Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
fs/btrfs/ctree.c +1 -0
+                       goto out;
                 }
 
Quoted 8 lines
Quoted 7 lines

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1570

Quoted 4 lines
fs/btrfs/ctree.c +2 -2
-               if (unlock_up)
-                       btrfs_unlock_up_safe(p, level + 1);
+               ret = -EAGAIN;
+               btrfs_unlock_up_safe(p, level + 1);

Same here, set ret after the unlocking.

But why set ret to -EAGAIN? Here we know p->nowait is false.

Nevermind this is because of the unconditional unlock now.

Yes, because we unlock unconditionally, the ret value of this function must be -EAGAIN.

Quoted 46 lines
Quoted 46 lines

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1572

Quoted 3 lines
fs/btrfs/ctree.c +2 -0
+               btrfs_tree_read_lock(tmp);
+               lock = true;

And here, with the same reasoning, set 'lock' to true before calling btrfs_tree_read_lock().

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1579

Quoted 12 lines
fs/btrfs/ctree.c +5 -5
+               btrfs_release_path(p);
 
                 /* now we're allowed to do a blocking uptodate check */
-               ret = btrfs_read_extent_buffer(tmp, &check);
-               if (ret) {
-                       free_extent_buffer(tmp);
-                       btrfs_release_path(p);
-                       return ret;
+               err = btrfs_read_extent_buffer(tmp, &check);
+               if (err) {
+                       ret = err;
+                       goto out;

Why do we need to have this 'err' variable?
Why not use 'ret' and simplify?

Quoted 6 lines
fs/btrfs/ctree.c +0 -4
                 }
-
-               if (unlock_up)
-                       ret = -EAGAIN;
-
                 goto out;

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1584

Quoted 10 lines
fs/btrfs/ctree.c +1 -7
         } else if (p->nowait) {
-               return -EAGAIN;
-       }
-
-       if (unlock_up) {
-               btrfs_unlock_up_safe(p, level + 1);
                 ret = -EAGAIN;
-       } else {
-               ret = 0;
+               btrfs_release_path(p);

Same here, set 'ret' to -EAGAIN after releasing the path.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1570

Quoted 5 lines
fs/btrfs/ctree.c +3 -0
+               goto out;
         }
 
+       ret = -EAGAIN;
+       btrfs_unlock_up_safe(p, level + 1);

Same here.

Quoted 7 lines
Quoted 1 line

But why set ret to -EAGAIN? Here we know p->nowait is false.

Nevermind this is because of the unconditional unlock now.

In both cases we still don't need the 'err' variable.
Just set 'ret' to -EAGAIN if btrfs_find_create_tree_block() below doesn't return an error and btrfs_read_extent_buffer() below and above don't return an error.

Indeed, we can only use ret value to receive the result of btrfs_find_create_tree_block or btrfs_read_extent_buffer. If the function returns correctly (return 0), we must change ret to -EAGAIN again and again, which is undoubtedly an increase in the function complexity.
Therefore, I use the err variable. If other err occurs, the err can be set to ret,
so that the caller can immediately stop and continue searching.

Quoted 20 lines
Quoted 20 lines
Quoted 8 lines
fs/btrfs/ctree.c +2 -1
+
         if (p->reada != READA_NONE)
                 reada_for_search(fs_info, p, level, slot, key->objectid);
 
-       tmp = read_tree_block(fs_info, blocknr, &check);
+       tmp = btrfs_find_create_tree_block(fs_info, blocknr, check.owner_root, check.level);
         if (IS_ERR(tmp)) {

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1604

Quoted 10 lines
fs/btrfs/ctree.c +7 -1
+               ret = PTR_ERR(tmp);
+               tmp = NULL;
                 btrfs_release_path(p);
-               return PTR_ERR(tmp);
+               goto out;
         }
+       create = true;
+
+       btrfs_tree_read_lock(tmp);
+       lock = true;

Same here, set 'lock' to true before the call to btrfs_tree_read_lock();

Quoted 7 lines
Quoted 7 lines

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1607

Quoted 3 lines
fs/btrfs/ctree.c +3 -0
+       btrfs_release_path(p);
+
+       /* now we're allowed to do a blocking uptodate check */

Please capitalize the first word of a sentence and end the sentence with punctuation.
This is the preferred style and we strive to do that for new comments or code that moves comments around.

Okay, I'll modify it.

Quoted 33 lines
Quoted 18 lines

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1608

Quoted 2 lines
fs/btrfs/ctree.c +1 -0
+       err = btrfs_read_extent_buffer(tmp, &check);

This can block, so if p->nowait at this point we should instead exit with -EAGAIN and not call this function.

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1612

Quoted 4 lines
fs/btrfs/ctree.c +4 -0
+       if (err) {
+               ret = err;
+               goto out;
+       }

Same here, no need to use extra variable 'err', can just use 'ret'.

Quoted 7 lines
fs/btrfs/ctree.c +1 -0
+
         /*
          * If the read above didn't mark this buffer up to date,
          * it will never end up being up to date.  Set ret to EIO now
@@ -1607,11 +1621,13 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p,
                 ret = -EIO;
 
Quoted 15 lines

fs/btrfs/ctree.c read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, · line 1630

Quoted 13 lines
fs/btrfs/ctree.c +7 -5
  out:
-       if (ret == 0) {
-               *eb_ret = tmp;
-       } else {
-               free_extent_buffer(tmp);
-               btrfs_release_path(p);
+       if (tmp) {
+               if (lock)
+                       btrfs_tree_read_unlock(tmp);
+               if (create && ret && ret != -EAGAIN)
+                       free_extent_buffer_stale(tmp);
+               else
+                       free_extent_buffer(tmp);

So in the success case we no longer set *eb_ret to tmp. Why? The callers expect that.

Quoted 4 lines

Ok, it's because of the -EAGAIN due to having the unconditional unlock now.

And btw, have you observed any case where this improved anything? Any benchmarks?

In the case of very high-speed 4k random write (cow), using nvme will improve.

But I can't give it a clear data improvement, because there are still many

different bottlenecks in the context of 4k random write,

and other patches need to be used to achieve significant improvements.

Quoted 4 lines

I can't see how this can make anything better because this function is never called for a root node, and therefore level + 1 < BTRFS_MAX_LEVEL.

The main improvement point of this patch is that if a eb cache miss occurs in a leaf and needs to execute IO, our original approach is to release level 2~MAX, but this patch is changed to release level 1~MAX.
We even Level 1 has also been released, so before releasing level 1, we changed to take the lock of level 0 to avoid race. This change reduces the lock contention of level 1.

Quoted 11 lines

But there is one case where this patch causes unnecessary path releases and makes the caller retry a search:
when none of the levels above were locked. That's why we had the following expression before:

unlock_up = ((level + 1 < BTRFS_MAX_LEVEL) && p->locks[level + 1]);

So we wouldn't make the caller retry the search if upper levels aren't locked or p->skip_locking == 1.
But now after this patch we make the caller retry the search in that case.

Indeed, I missed something about skip_locking. In this case, there is no need to search again, I will correct it.

Thank you for your review.

Quoted 30 lines

Thanks.

Quoted 27 lines

Thanks.

Quoted 25 lines
fs/btrfs/ctree.c +2 -3
         }
 
         return ret;
@@ -2198,7 +2214,7 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root,
                 }
 
                 err = read_block_for_search(root, p, &b, level, slot, key);
-               if (err == -EAGAIN)
+               if (err == -EAGAIN && !p->nowait)
                         goto again;
                 if (err) {
                         ret = err;
@@ -2325,7 +2341,7 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key,
                 }
 
                 err = read_block_for_search(root, p, &b, level, slot, key);
-               if (err == -EAGAIN)
+               if (err == -EAGAIN && !p->nowait)
                         goto again;
                 if (err) {
                         ret = err;
--

2.17.1