Skip to contentSkip to conversation

btrfs: qgroup: fix sleep from invalid context bug in update_qgroup_limit_item()

PATCH Superseded ChenXiaoSong 3 participants 9 messages Nov 10, 2022 – Nov 11, 2022 Btrfs file system

v1 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 btrfs: qgroup: fix sleep from invalid context bug in update_qgroup_limit_item()

Syzkaller reported BUG as follows:

  BUG: sleeping function called from invalid context at
       include/linux/sched/mm.h:274
  Call Trace:
   <TASK>
   dump_stack_lvl+0xcd/0x134
   __might_resched.cold+0x222/0x26b
   kmem_cache_alloc+0x2e7/0x3c0
   update_qgroup_limit_item+0xe1/0x390
   btrfs_qgroup_inherit+0x147b/0x1ee0
   create_subvol+0x4eb/0x1710
   btrfs_mksubvol+0xfe5/0x13f0
   __btrfs_ioctl_snap_create+0x2b0/0x430
   btrfs_ioctl_snap_create_v2+0x25a/0x520
   btrfs_ioctl+0x2a1c/0x5ce0
   __x64_sys_ioctl+0x193/0x200
   do_syscall_64+0x35/0x80

Fix this by introducing __update_qgroup_limit_item() helper, allocate memory outside of the spin lock.

---
 fs/btrfs/qgroup.c | 35 ++++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)
fs/btrfs/qgroup.c +26 -9
index 9334c3157c22..99a61cc04b68 100644
@@ -768,11 +768,11 @@ static int del_qgroup_item(struct btrfs_trans_handle *trans, u64 qgroupid)
 	return ret;
 }
 
-static int update_qgroup_limit_item(struct btrfs_trans_handle *trans,
-				    struct btrfs_qgroup *qgroup)
+static int __update_qgroup_limit_item(struct btrfs_trans_handle *trans,
+				      struct btrfs_qgroup *qgroup,
+				      struct btrfs_path *path)
 {
 	struct btrfs_root *quota_root = trans->fs_info->quota_root;
-	struct btrfs_path *path;
 	struct btrfs_key key;
 	struct extent_buffer *l;
 	struct btrfs_qgroup_limit_item *qgroup_limit;
@@ -783,10 +783,6 @@ static int update_qgroup_limit_item(struct btrfs_trans_handle *trans,
 	key.type = BTRFS_QGROUP_LIMIT_KEY;
 	key.offset = qgroup->qgroupid;
 
-	path = btrfs_alloc_path();
-	if (!path)
-		return -ENOMEM;
-
 	ret = btrfs_search_slot(trans, quota_root, &key, path, 0, 1);
 	if (ret > 0)
 		ret = -ENOENT;
@@ -806,6 +802,21 @@ static int update_qgroup_limit_item(struct btrfs_trans_handle *trans,
 	btrfs_mark_buffer_dirty(l);
 
 out:
+	return ret;
+}
+
+static int update_qgroup_limit_item(struct btrfs_trans_handle *trans,
+				    struct btrfs_qgroup *qgroup)
+{
+	struct btrfs_path *path;
+	int ret;
+
+	path = btrfs_alloc_path();
+	if (!path)
+		return -ENOMEM;
+
+	ret = __update_qgroup_limit_item(trans, qgroup, path);
+
 	btrfs_free_path(path);
 	return ret;
 }
@@ -2860,6 +2871,7 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid,
 	bool need_rescan = false;
 	u32 level_size = 0;
 	u64 nums;
+	struct btrfs_path *path;
 
 	/*
 	 * There are only two callers of this function.
@@ -2935,6 +2947,11 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid,
 		ret = 0;
 	}
 
+	path = btrfs_alloc_path();
+	if (!path) {
+		ret = -ENOMEM;
+		goto out;
+	}
 
 	spin_lock(&fs_info->qgroup_lock);
 
@@ -2950,8 +2967,7 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid,
 		dstgroup->max_excl = inherit->lim.max_excl;
 		dstgroup->rsv_rfer = inherit->lim.rsv_rfer;
 		dstgroup->rsv_excl = inherit->lim.rsv_excl;
-
-		ret = update_qgroup_limit_item(trans, dstgroup);
+		ret = __update_qgroup_limit_item(trans, dstgroup, path);
 		if (ret) {
 			qgroup_mark_inconsistent(fs_info);
 			btrfs_info(fs_info,
@@ -3053,6 +3069,7 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid,
 
 unlock:
 	spin_unlock(&fs_info->qgroup_lock);
+	btrfs_free_path(path);
 	if (!ret)
 		ret = btrfs_sysfs_add_one_qgroup(fs_info, dstgroup);
 out:
Signature · 1 line
2.31.1
#1 lore raw
Quoted 1 line · ChenXiaoSong

Syzkaller reported BUG as follows:

Do you have link to the report? Or at least the identifier of the report, there's some automation that recognizes Reported-by: syzbot-... to close it once the patch is merged.

#2 lore raw

I have _no_ link to the report, I just reproduce it in my own qemu vm, and _no_ c or syz repro.

Quoted 8 lines · David Sterba
Quoted 1 line · ChenXiaoSong

Syzkaller reported BUG as follows:

Do you have link to the report? Or at least the identifier of the report, there's some automation that recognizes Reported-by: syzbot-... to close it once the patch is merged.
.

#3 lore raw
Quoted 2 lines · ChenXiaoSong

I have _no_ link to the report, I just reproduce it in my own qemu vm, and _no_ c or syz repro.

I see, we've got several reports from syzbot, I thought you're fixing one of them.

#4 lore raw
In reply to ChenXiaoSong
Quoted 19 lines · ChenXiaoSong

Syzkaller reported BUG as follows:

  BUG: sleeping function called from invalid context at
       include/linux/sched/mm.h:274
  Call Trace:
   <TASK>
   dump_stack_lvl+0xcd/0x134
   __might_resched.cold+0x222/0x26b
   kmem_cache_alloc+0x2e7/0x3c0
   update_qgroup_limit_item+0xe1/0x390
   btrfs_qgroup_inherit+0x147b/0x1ee0
   create_subvol+0x4eb/0x1710
   btrfs_mksubvol+0xfe5/0x13f0
   __btrfs_ioctl_snap_create+0x2b0/0x430
   btrfs_ioctl_snap_create_v2+0x25a/0x520
   btrfs_ioctl+0x2a1c/0x5ce0
   __x64_sys_ioctl+0x193/0x200
   do_syscall_64+0x35/0x80
Quoted 4 lines

Fix this by introducing __update_qgroup_limit_item() helper, allocate memory outside of the spin lock.

Added to misc-next, thanks.

Review

Quoted 1 line
(quoted diff) +1 -0
+	path = btrfs_alloc_path();

btrfs_alloc_path uses fixed GFP_NOFS flags for kmem_cache_alloc but that does not try to detect if it could sleep or not.

#5 lore raw
Quoted 20 lines · David Sterba
Quoted 19 lines · ChenXiaoSong

Syzkaller reported BUG as follows:

   BUG: sleeping function called from invalid context at
        include/linux/sched/mm.h:274
   Call Trace:
    <TASK>
    dump_stack_lvl+0xcd/0x134
    __might_resched.cold+0x222/0x26b
    kmem_cache_alloc+0x2e7/0x3c0
    update_qgroup_limit_item+0xe1/0x390
    btrfs_qgroup_inherit+0x147b/0x1ee0
    create_subvol+0x4eb/0x1710
    btrfs_mksubvol+0xfe5/0x13f0
    __btrfs_ioctl_snap_create+0x2b0/0x430
    btrfs_ioctl_snap_create_v2+0x25a/0x520
    btrfs_ioctl+0x2a1c/0x5ce0
    __x64_sys_ioctl+0x193/0x200
    do_syscall_64+0x35/0x80
Quoted 6 lines
Quoted 4 lines

Fix this by introducing __update_qgroup_limit_item() helper, allocate memory outside of the spin lock.

Added to misc-next, thanks.

Please remove it for now, the patch only addressed what MM layer reports, it doesn't really solve the root cause, we're doing a tree modification (btrfs_search_slot()), under a spinlock.

I'm pretty sure there will be a v2 version to properly fix it.

Thanks,
Qu

Quoted 5 lines

Review

Quoted 2 lines
(quoted diff) +1 -0
+	path = btrfs_alloc_path();

btrfs_alloc_path uses fixed GFP_NOFS flags for kmem_cache_alloc but that does not try to detect if it could sleep or not.

#6 lore raw
Quoted 26 lines · Qu Wenruo
Quoted 23 lines · David Sterba
Quoted 22 lines · ChenXiaoSong

Syzkaller reported BUG as follows:

   BUG: sleeping function called from invalid context at
        include/linux/sched/mm.h:274
   Call Trace:
    <TASK>
    dump_stack_lvl+0xcd/0x134
    __might_resched.cold+0x222/0x26b
    kmem_cache_alloc+0x2e7/0x3c0
    update_qgroup_limit_item+0xe1/0x390
    btrfs_qgroup_inherit+0x147b/0x1ee0
    create_subvol+0x4eb/0x1710
    btrfs_mksubvol+0xfe5/0x13f0
    __btrfs_ioctl_snap_create+0x2b0/0x430
    btrfs_ioctl_snap_create_v2+0x25a/0x520
    btrfs_ioctl+0x2a1c/0x5ce0
    __x64_sys_ioctl+0x193/0x200
    do_syscall_64+0x35/0x80

Fix this by introducing __update_qgroup_limit_item() helper, allocate memory outside of the spin lock.

Quoted 7 lines
Quoted 3 lines
Quoted 1 line

Added to misc-next, thanks.

Please remove it for now, the patch only addressed what MM layer reports, it doesn't really solve the root cause, we're doing a tree modification (btrfs_search_slot()), under a spinlock.

Removed. As the potential sleeping under spinlock is hard to spot we should add might_sleep to some places.

#7 lore raw
In reply to ChenXiaoSong
Quoted 19 lines · ChenXiaoSong

Syzkaller reported BUG as follows:

   BUG: sleeping function called from invalid context at
        include/linux/sched/mm.h:274
   Call Trace:
    <TASK>
    dump_stack_lvl+0xcd/0x134
    __might_resched.cold+0x222/0x26b
    kmem_cache_alloc+0x2e7/0x3c0
    update_qgroup_limit_item+0xe1/0x390
    btrfs_qgroup_inherit+0x147b/0x1ee0
    create_subvol+0x4eb/0x1710
    btrfs_mksubvol+0xfe5/0x13f0
    __btrfs_ioctl_snap_create+0x2b0/0x430
    btrfs_ioctl_snap_create_v2+0x25a/0x520
    btrfs_ioctl+0x2a1c/0x5ce0
    __x64_sys_ioctl+0x193/0x200
    do_syscall_64+0x35/0x80
Quoted 4 lines

Fix this by introducing __update_qgroup_limit_item() helper, allocate memory outside of the spin lock.

Unfortunately, __update_qgroup_limit_item() can still sleep.

As it calls btrfs_search_slot(), which can lead to disk IO if the qgroup tree is not cached.

I believe the proper way is to either unlock the spinlock inside btrfs_qgroup_inherit() (which needs extra scrutiny on the qgroup lock), or delayed the limit item updates until we have unlocked the spinlock.

To me, the latter one seems more reasonable, as it's just one qgroup (@dstgroup), and we're doing the same delayed work for sysfs interface creation.

Thanks,
Qu

Quoted 93 lines
---
  fs/btrfs/qgroup.c | 35 ++++++++++++++++++++++++++---------
  1 file changed, 26 insertions(+), 9 deletions(-)
fs/btrfs/qgroup.c +26 -9
index 9334c3157c22..99a61cc04b68 100644
@@ -768,11 +768,11 @@ static int del_qgroup_item(struct btrfs_trans_handle *trans, u64 qgroupid)
  	return ret;
  }
 
-static int update_qgroup_limit_item(struct btrfs_trans_handle *trans,
-				    struct btrfs_qgroup *qgroup)
+static int __update_qgroup_limit_item(struct btrfs_trans_handle *trans,
+				      struct btrfs_qgroup *qgroup,
+				      struct btrfs_path *path) >   {
  	struct btrfs_root *quota_root = trans->fs_info->quota_root;
-	struct btrfs_path *path;
  	struct btrfs_key key;
  	struct extent_buffer *l;
  	struct btrfs_qgroup_limit_item *qgroup_limit;
@@ -783,10 +783,6 @@ static int update_qgroup_limit_item(struct btrfs_trans_handle *trans,
  	key.type = BTRFS_QGROUP_LIMIT_KEY;
  	key.offset = qgroup->qgroupid;
 
-	path = btrfs_alloc_path();
-	if (!path)
-		return -ENOMEM;
-
  	ret = btrfs_search_slot(trans, quota_root, &key, path, 0, 1);
  	if (ret > 0)
  		ret = -ENOENT;
@@ -806,6 +802,21 @@ static int update_qgroup_limit_item(struct btrfs_trans_handle *trans,
  	btrfs_mark_buffer_dirty(l);
 
  out:
+	return ret;
+}
+
+static int update_qgroup_limit_item(struct btrfs_trans_handle *trans,
+				    struct btrfs_qgroup *qgroup)
+{
+	struct btrfs_path *path;
+	int ret;
+
+	path = btrfs_alloc_path();
+	if (!path)
+		return -ENOMEM;
+
+	ret = __update_qgroup_limit_item(trans, qgroup, path);
+
  	btrfs_free_path(path);
  	return ret;
  }
@@ -2860,6 +2871,7 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid,
  	bool need_rescan = false;
  	u32 level_size = 0;
  	u64 nums;
+	struct btrfs_path *path;
 
  	/*
  	 * There are only two callers of this function.
@@ -2935,6 +2947,11 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid,
  		ret = 0;
  	}
 
+	path = btrfs_alloc_path();
+	if (!path) {
+		ret = -ENOMEM;
+		goto out;
+	}
 
  	spin_lock(&fs_info->qgroup_lock);
 
@@ -2950,8 +2967,7 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid,
  		dstgroup->max_excl = inherit->lim.max_excl;
  		dstgroup->rsv_rfer = inherit->lim.rsv_rfer;
  		dstgroup->rsv_excl = inherit->lim.rsv_excl;
-
-		ret = update_qgroup_limit_item(trans, dstgroup);
+		ret = __update_qgroup_limit_item(trans, dstgroup, path);
  		if (ret) {
  			qgroup_mark_inconsistent(fs_info);
  			btrfs_info(fs_info,
@@ -3053,6 +3069,7 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid,
 
  unlock:
  	spin_unlock(&fs_info->qgroup_lock);
+	btrfs_free_path(path);
  	if (!ret)
  		ret = btrfs_sysfs_add_one_qgroup(fs_info, dstgroup);
  out:
#8 lore raw

Yes, at least two places will sleep in btrfs_search_slot():

update_qgroup_limit_item

    btrfs_search_slot
      setup_nodes_for_search
        reada_for_balance
          btrfs_readahead_node_child
            btrfs_readahead_tree_block
              btrfs_find_create_tree_block
                alloc_extent_buffer
                  kmem_cache_zalloc
                    /* will sleep */
                    kmem_cache_alloc(GFP_NOFS|__GFP_NOFAIL|__GFP_ZERO)
              read_extent_buffer_pages
                submit_extent_page
                  submit_one_bio /* disk IO, will sleep */
Quoted 133 lines · Qu Wenruo
Quoted 19 lines · ChenXiaoSong

Syzkaller reported BUG as follows:

   BUG: sleeping function called from invalid context at
        include/linux/sched/mm.h:274
   Call Trace:
    <TASK>
    dump_stack_lvl+0xcd/0x134
    __might_resched.cold+0x222/0x26b
    kmem_cache_alloc+0x2e7/0x3c0
    update_qgroup_limit_item+0xe1/0x390
    btrfs_qgroup_inherit+0x147b/0x1ee0
    create_subvol+0x4eb/0x1710
    btrfs_mksubvol+0xfe5/0x13f0
    __btrfs_ioctl_snap_create+0x2b0/0x430
    btrfs_ioctl_snap_create_v2+0x25a/0x520
    btrfs_ioctl+0x2a1c/0x5ce0
    __x64_sys_ioctl+0x193/0x200
    do_syscall_64+0x35/0x80
Quoted 4 lines

Fix this by introducing __update_qgroup_limit_item() helper, allocate memory outside of the spin lock.

Unfortunately, __update_qgroup_limit_item() can still sleep.

As it calls btrfs_search_slot(), which can lead to disk IO if the qgroup tree is not cached.

I believe the proper way is to either unlock the spinlock inside btrfs_qgroup_inherit() (which needs extra scrutiny on the qgroup lock), or delayed the limit item updates until we have unlocked the spinlock.

To me, the latter one seems more reasonable, as it's just one qgroup (@dstgroup), and we're doing the same delayed work for sysfs interface creation.

Thanks,
Qu

Quoted 86 lines

---
  fs/btrfs/qgroup.c | 35 ++++++++++++++++++++++++++---------

  1 file changed, 26 insertions(+), 9 deletions(-)
fs/btrfs/qgroup.c +3 -7
index 9334c3157c22..99a61cc04b68 100644
@@ -768,11 +768,11 @@ static int del_qgroup_item(struct 
 btrfs_trans_handle *trans, u64 qgroupid)
       return ret;
   }
-static int update_qgroup_limit_item(struct btrfs_trans_handle *trans,
-                    struct btrfs_qgroup *qgroup)
+static int __update_qgroup_limit_item(struct btrfs_trans_handle *trans,
+                      struct btrfs_qgroup *qgroup,
+                      struct btrfs_path *path) >   {
       struct btrfs_root *quota_root = trans->fs_info->quota_root;
-    struct btrfs_path *path;
       struct btrfs_key key;
       struct extent_buffer *l;
       struct btrfs_qgroup_limit_item *qgroup_limit;
@@ -783,10 +783,6 @@ static int update_qgroup_limit_item(struct 
 btrfs_trans_handle *trans,
       key.type = BTRFS_QGROUP_LIMIT_KEY;
       key.offset = qgroup->qgroupid;
-    path = btrfs_alloc_path();
-    if (!path)
-        return -ENOMEM;
-
       ret = btrfs_search_slot(trans, quota_root, &key, path, 0, 1);
       if (ret > 0)
           ret = -ENOENT;
@@ -806,6 +802,21 @@ static int update_qgroup_limit_item(struct 
 btrfs_trans_handle *trans,
       btrfs_mark_buffer_dirty(l);

  out:
+    return ret;

fs/btrfs/qgroup.c +14 -0
+}
+
+static int update_qgroup_limit_item(struct btrfs_trans_handle *trans,
+                    struct btrfs_qgroup *qgroup)
+{
+    struct btrfs_path *path;
+    int ret;
+
+    path = btrfs_alloc_path();
+    if (!path)
+        return -ENOMEM;
+
+    ret = __update_qgroup_limit_item(trans, qgroup, path);
+

      btrfs_free_path(path);
      return ret;
  }

fs/btrfs/qgroup.c +7 -2
@@ -2860,6 +2871,7 @@ int btrfs_qgroup_inherit(struct 
 btrfs_trans_handle *trans, u64 srcid,
       bool need_rescan = false;
       u32 level_size = 0;
       u64 nums;
+    struct btrfs_path *path;
       /*
        * There are only two callers of this function.
@@ -2935,6 +2947,11 @@ int btrfs_qgroup_inherit(struct 
 btrfs_trans_handle *trans, u64 srcid,
           ret = 0;
       }
+    path = btrfs_alloc_path();
+    if (!path) {
+        ret = -ENOMEM;
+        goto out;
+    }
       spin_lock(&fs_info->qgroup_lock);
@@ -2950,8 +2967,7 @@ int btrfs_qgroup_inherit(struct 
 btrfs_trans_handle *trans, u64 srcid,
           dstgroup->max_excl = inherit->lim.max_excl;
           dstgroup->rsv_rfer = inherit->lim.rsv_rfer;
           dstgroup->rsv_excl = inherit->lim.rsv_excl;
-
-        ret = update_qgroup_limit_item(trans, dstgroup);
+        ret = __update_qgroup_limit_item(trans, dstgroup, path);
           if (ret) {
               qgroup_mark_inconsistent(fs_info);

              btrfs_info(fs_info,

fs/btrfs/qgroup.c +0 -0
@@ -3053,6 +3069,7 @@ int btrfs_qgroup_inherit(struct 
 btrfs_trans_handle *trans, u64 srcid,
Quoted 6 lines

  unlock:
      spin_unlock(&fs_info->qgroup_lock);
+    btrfs_free_path(path);
      if (!ret)
          ret = btrfs_sysfs_add_one_qgroup(fs_info, dstgroup); out:

.