btrfs: fix use-after-free in btrfs_get_global_root
PATCH 1/1 btrfs: fix use-after-free in btrfs_get_global_root
Syzkaller reported UAF as follows:
==================================================================
BUG: KASAN: use-after-free in btrfs_get_global_root+0x663/0xa10 Read of size 4 at addr ffff88811ddbb3c0 by task kworker/u16:1/11 CPU: 4 PID: 11 Comm: kworker/u16:1 Not tainted 6.0.0-rc1-next-20220822+ #2 Workqueue: btrfs-qgroup-rescan btrfs_work_helper Call Trace: <TASK> dump_stack_lvl+0x6e/0x91 print_report.cold+0xb2/0x6bb kasan_report+0xa8/0x130 kasan_check_range+0x13f/0x1d0 btrfs_get_global_root+0x663/0xa10 btrfs_get_fs_root_commit_root+0xa5/0x150 find_parent_nodes+0x92f/0x2990 btrfs_find_all_roots_safe+0x12d/0x220 btrfs_find_all_roots+0xbb/0xd0 btrfs_qgroup_rescan_worker+0x600/0xc30 btrfs_work_helper+0xff/0x750 process_one_work+0x52c/0x930 worker_thread+0x352/0x8c0 kthread+0x1b9/0x200 ret_from_fork+0x22/0x30 </TASK> Allocated by task 1895: kasan_save_stack+0x1e/0x40 __kasan_kmalloc+0xa9/0xe0 btrfs_alloc_root+0x40/0x820 btrfs_create_tree+0xf8/0x500 btrfs_quota_enable+0x30a/0x1120 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd Freed by task 1895: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x40 __kasan_slab_free+0x127/0x1c0 kfree+0xa8/0x2d0 btrfs_put_root+0x1ca/0x230 btrfs_quota_enable+0x87c/0x1120 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd ==================================================================
Above issue may happens as follows:
p1 p2btrfs_quota_enable
spin_lock(&fs_info->qgroup_lock);
fs_info->quota_root = quota_root;
spin_unlock(&fs_info->qgroup_lock);
ret = qgroup_rescan_init -> return error
if (ret)
btrfs_put_root(quota_root);
kfree(root); if (ret) {
ulist_free(fs_info->qgroup_ulist);
fs_info->qgroup_ulist = NULL;
btrfs_sysfs_del_qgroups(fs_info);
} btrfs_qgroup_rescan_worker
btrfs_find_all_roots btrfs_find_all_roots_safe find_parent_nodes btrfs_get_fs_root_commit_root btrfs_grab_root(fs_info->quota_root) -> quota_root already freed
Syzkaller also reported another issue:
==================================================================
BUG: KASAN: use-after-free in ulist_release+0x30/0xb3 Read of size 8 at addr ffff88811413d048 by task rep/2921 CPU: 3 PID: 2921 Comm: rep Not tainted 6.0.0-rc1-next-20220822+ #3 rep[2921] cmdline: ./rep Call Trace: <TASK> dump_stack_lvl+0x6e/0x91 print_report.cold+0xb2/0x6bb kasan_report+0xa8/0x130 ulist_release+0x30/0xb3 ulist_reinit+0x16/0x56 btrfs_qgroup_free_refroot+0x288/0x3f0 btrfs_qgroup_free_meta_all_pertrans+0xed/0x1e0 commit_fs_roots+0x28c/0x430 btrfs_commit_transaction+0x9a6/0x1b40 btrfs_qgroup_rescan+0x7e/0x130 btrfs_ioctl+0x48ed/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd </TASK> Allocated by task 2900: kasan_save_stack+0x1e/0x40 __kasan_kmalloc+0xa9/0xe0 ulist_alloc+0x5c/0xe0 btrfs_quota_enable+0x1b2/0x1160 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd Freed by task 2900: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x40 __kasan_slab_free+0x127/0x1c0 kfree+0xa8/0x2d0 ulist_free.cold+0x15/0x1a btrfs_quota_enable+0x8bf/0x1160 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd ==================================================================
To solve above issues just set 'fs_info->quota_root' after qgroup_rescan_init return success.
--- fs/btrfs/qgroup.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)
fs/btrfs/qgroup.c +10 -10
@@ -1158,18 +1158,18 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info)if (ret) goto out_free_path; -/* -* Set quota enabled flag after committing the transaction, to avoid -* deadlocks on fs_info->qgroup_ioctl_lock with concurrent snapshot -* creation. -*/ -spin_lock (&fs_info->qgroup_lock); - fs_info->quota_root = quota_root; -set_bit (BTRFS_FS_QUOTA_ENABLED , &fs_info->flags); -spin_unlock (&fs_info->qgroup_lock); - ret =qgroup_rescan_init (fs_info,0 ,1 );if (!ret) { +/* +* Set quota enabled flag after committing the transaction, to +* avoid deadlocks on fs_info->qgroup_ioctl_lock with concurrent +* snapshot creation. +*/ +spin_lock (&fs_info->qgroup_lock); + fs_info->quota_root = quota_root; +set_bit (BTRFS_FS_QUOTA_ENABLED , &fs_info->flags); +spin_unlock (&fs_info->qgroup_lock); +qgroup_rescan_zero_tracking (fs_info); fs_info->qgroup_rescan_running = true; btrfs_queue_work(fs_info->qgroup_rescan_workers,
Signature · 1 line
2.31.1
Quoted 152 lines · Ye Bin
Syzkaller reported UAF as follows:
==================================================================BUG: KASAN: use-after-free in btrfs_get_global_root+0x663/0xa10 Read of size 4 at addr ffff88811ddbb3c0 by task kworker/u16:1/11 CPU: 4 PID: 11 Comm: kworker/u16:1 Not tainted 6.0.0-rc1-next-20220822+ #2 Workqueue: btrfs-qgroup-rescan btrfs_work_helper Call Trace: <TASK> dump_stack_lvl+0x6e/0x91 print_report.cold+0xb2/0x6bb kasan_report+0xa8/0x130 kasan_check_range+0x13f/0x1d0 btrfs_get_global_root+0x663/0xa10 btrfs_get_fs_root_commit_root+0xa5/0x150 find_parent_nodes+0x92f/0x2990 btrfs_find_all_roots_safe+0x12d/0x220 btrfs_find_all_roots+0xbb/0xd0 btrfs_qgroup_rescan_worker+0x600/0xc30 btrfs_work_helper+0xff/0x750 process_one_work+0x52c/0x930 worker_thread+0x352/0x8c0 kthread+0x1b9/0x200 ret_from_fork+0x22/0x30 </TASK> Allocated by task 1895: kasan_save_stack+0x1e/0x40 __kasan_kmalloc+0xa9/0xe0 btrfs_alloc_root+0x40/0x820 btrfs_create_tree+0xf8/0x500 btrfs_quota_enable+0x30a/0x1120 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd Freed by task 1895: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x40 __kasan_slab_free+0x127/0x1c0 kfree+0xa8/0x2d0 btrfs_put_root+0x1ca/0x230 btrfs_quota_enable+0x87c/0x1120 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd ==================================================================Above issue may happens as follows:
p1 p2btrfs_quota_enable
spin_lock(&fs_info->qgroup_lock);
fs_info->quota_root=quota_root;
spin_unlock(&fs_info->qgroup_lock);ret = qgroup_rescan_init -> return error if (ret) btrfs_put_root(quota_root); kfree(root);if (ret) { ulist_free(fs_info->qgroup_ulist); fs_info->qgroup_ulist = NULL; btrfs_sysfs_del_qgroups(fs_info); } btrfs_qgroup_rescan_worker btrfs_find_all_roots btrfs_find_all_roots_safe find_parent_nodes btrfs_get_fs_root_commit_root btrfs_grab_root(fs_info->quota_root) -> quota_root already freedSyzkaller also reported another issue:
==================================================================BUG: KASAN: use-after-free in ulist_release+0x30/0xb3 Read of size 8 at addr ffff88811413d048 by task rep/2921 CPU: 3 PID: 2921 Comm: rep Not tainted 6.0.0-rc1-next-20220822+ #3 rep[2921] cmdline: ./rep Call Trace: <TASK> dump_stack_lvl+0x6e/0x91 print_report.cold+0xb2/0x6bb kasan_report+0xa8/0x130 ulist_release+0x30/0xb3 ulist_reinit+0x16/0x56 btrfs_qgroup_free_refroot+0x288/0x3f0 btrfs_qgroup_free_meta_all_pertrans+0xed/0x1e0 commit_fs_roots+0x28c/0x430 btrfs_commit_transaction+0x9a6/0x1b40 btrfs_qgroup_rescan+0x7e/0x130 btrfs_ioctl+0x48ed/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd </TASK> Allocated by task 2900: kasan_save_stack+0x1e/0x40 __kasan_kmalloc+0xa9/0xe0 ulist_alloc+0x5c/0xe0 btrfs_quota_enable+0x1b2/0x1160 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd Freed by task 2900: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x40 __kasan_slab_free+0x127/0x1c0 kfree+0xa8/0x2d0 ulist_free.cold+0x15/0x1a btrfs_quota_enable+0x8bf/0x1160 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd ==================================================================To solve above issues just set '
fs_info->quota_root' afterqgroup_rescan_initreturn success.Signed-off-by Ye Bin <[email protected]>--- fs/btrfs/qgroup.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)fs/btrfs/qgroup.c +0
-10@@ -1158,18 +1158,18 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info)if (ret) goto out_free_path; -/* -* Set quota enabled flag after committing the transaction, to avoid -* deadlocks on fs_info->qgroup_ioctl_lock with concurrent snapshot -* creation. -*/ -spin_lock (&fs_info->qgroup_lock); - fs_info->quota_root = quota_root; -set_bit (BTRFS_FS_QUOTA_ENABLED , &fs_info->flags); -spin_unlock (&fs_info->qgroup_lock); - ret =qgroup_rescan_init (fs_info,0 ,1 );if (!ret) {
fs/btrfs/qgroup.c int btrfs_quota_enable(struct btrfs_fs_info *fs_info) · line 1172
Quoted 10 lines
fs/btrfs/qgroup.c +10
-0+/* +* Set quota enabled flag after committing the transaction, to +* avoid deadlocks on fs_info->qgroup_ioctl_lock with concurrent +* snapshot creation. +*/ +spin_lock (&fs_info->qgroup_lock); + fs_info->quota_root = quota_root; +set_bit (BTRFS_FS_QUOTA_ENABLED , &fs_info->flags); +spin_unlock (&fs_info->qgroup_lock); +
But how can the race happen? The changelog should explain that.
To me this suggests that after we set BTRFS_FS_QUOTA_ENABLED and set the quota root, but before we called qgroup_rescan_init() at btrfs_quota_enable(), some other task started the rescan worker first - I can only think of someone else calling the ioctl to start the rescan worker (btrfs_ioctl_quota_rescan()).
In that case we get "ret == -EINPROGRESS" at btrfs_quota_enable().
So please provide a detailed explanation in the log of how the race can happen.
This solution is also buggy. Because in case of an error, we will leave the quota tree created, the qgroup relation, etc. That is, we don't undo what btrfs_create_tree(), add_qgroup_item(), add_qgroup_rb(), etc did Which means a future btrfs_quota_enable() call would fail, and calling btrfs_quota_disable() to undo all those things will not work either, because fs_info->quota_root is NULL.
I would suggest ignoring the error of qgroup_rescan_init() if it's -EINPROGRESS, and ASSERT if it's anything different from 0 or -EINPROGRESS. Also add a comment mentioning we can get -EINPROGRESS because someone may have called the qgroup rescan ioctl.
Thanks.
Quoted 6 lines
fs/btrfs/qgroup.c +0
-0qgroup_rescan_zero_tracking (fs_info); fs_info->qgroup_rescan_running = true; btrfs_queue_work(fs_info->qgroup_rescan_workers,Signature · 1 line
2.31.1
Quoted 64 lines · Filipe Manana
Quoted 63 lines · Ye Bin
Syzkaller reported UAF as follows:
==================================================================BUG: KASAN: use-after-free in btrfs_get_global_root+0x663/0xa10 Read of size 4 at addr ffff88811ddbb3c0 by task kworker/u16:1/11 CPU: 4 PID: 11 Comm: kworker/u16:1 Not tainted 6.0.0-rc1-next-20220822+ #2 Workqueue: btrfs-qgroup-rescan btrfs_work_helper Call Trace: <TASK> dump_stack_lvl+0x6e/0x91 print_report.cold+0xb2/0x6bb kasan_report+0xa8/0x130 kasan_check_range+0x13f/0x1d0 btrfs_get_global_root+0x663/0xa10 btrfs_get_fs_root_commit_root+0xa5/0x150 find_parent_nodes+0x92f/0x2990 btrfs_find_all_roots_safe+0x12d/0x220 btrfs_find_all_roots+0xbb/0xd0 btrfs_qgroup_rescan_worker+0x600/0xc30 btrfs_work_helper+0xff/0x750 process_one_work+0x52c/0x930 worker_thread+0x352/0x8c0 kthread+0x1b9/0x200 ret_from_fork+0x22/0x30 </TASK> Allocated by task 1895: kasan_save_stack+0x1e/0x40 __kasan_kmalloc+0xa9/0xe0 btrfs_alloc_root+0x40/0x820 btrfs_create_tree+0xf8/0x500 btrfs_quota_enable+0x30a/0x1120 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd Freed by task 1895: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x40 __kasan_slab_free+0x127/0x1c0 kfree+0xa8/0x2d0 btrfs_put_root+0x1ca/0x230 btrfs_quota_enable+0x87c/0x1120 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd ==================================================================Above issue may happens as follows:
p1 p2btrfs_quota_enablespin_lock(&fs_info->qgroup_lock); fs_info->quota_root = quota_root; spin_unlock(&fs_info->qgroup_lock); ret = qgroup_rescan_init -> return error if (ret) btrfs_put_root(quota_root); kfree(root);
Quoted 11 lines
Quoted 11 lines
if (ret) { ulist_free(fs_info->qgroup_ulist); fs_info->qgroup_ulist = NULL; btrfs_sysfs_del_qgroups(fs_info); } btrfs_qgroup_rescan_worker btrfs_find_all_roots btrfs_find_all_roots_safe find_parent_nodes btrfs_get_fs_root_commit_root btrfs_grab_root(fs_info->quota_root) -> quota_root already freed
I have described the process of the issue here. I can write it in more detail.
Quoted 89 lines
Quoted 77 lines
Syzkaller also reported another issue:
==================================================================BUG: KASAN: use-after-free in ulist_release+0x30/0xb3 Read of size 8 at addr ffff88811413d048 by task rep/2921 CPU: 3 PID: 2921 Comm: rep Not tainted 6.0.0-rc1-next-20220822+ #3 rep[2921] cmdline: ./rep Call Trace: <TASK> dump_stack_lvl+0x6e/0x91 print_report.cold+0xb2/0x6bb kasan_report+0xa8/0x130 ulist_release+0x30/0xb3 ulist_reinit+0x16/0x56 btrfs_qgroup_free_refroot+0x288/0x3f0 btrfs_qgroup_free_meta_all_pertrans+0xed/0x1e0 commit_fs_roots+0x28c/0x430 btrfs_commit_transaction+0x9a6/0x1b40 btrfs_qgroup_rescan+0x7e/0x130 btrfs_ioctl+0x48ed/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd </TASK> Allocated by task 2900: kasan_save_stack+0x1e/0x40 __kasan_kmalloc+0xa9/0xe0 ulist_alloc+0x5c/0xe0 btrfs_quota_enable+0x1b2/0x1160 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd Freed by task 2900: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x40 __kasan_slab_free+0x127/0x1c0 kfree+0xa8/0x2d0 ulist_free.cold+0x15/0x1a btrfs_quota_enable+0x8bf/0x1160 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd ==================================================================To solve above issues just set '
fs_info->quota_root' afterqgroup_rescan_initreturn success.Signed-off-by Ye Bin <[email protected]>--- fs/btrfs/qgroup.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)fs/btrfs/qgroup.c +0
-10@@ -1158,18 +1158,18 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info)if (ret) goto out_free_path; -/* -* Set quota enabled flag after committing the transaction, to avoid -* deadlocks on fs_info->qgroup_ioctl_lock with concurrent snapshot -* creation. -*/ -spin_lock (&fs_info->qgroup_lock); - fs_info->quota_root = quota_root; -set_bit (BTRFS_FS_QUOTA_ENABLED , &fs_info->flags); -spin_unlock (&fs_info->qgroup_lock); - ret =qgroup_rescan_init (fs_info,0 ,1 );if (!ret) {fs/btrfs/qgroup.c int btrfs_quota_enable(struct btrfs_fs_info *fs_info) · line 1172
Quoted 10 lines
fs/btrfs/qgroup.c +10
-0+/* +* Set quota enabled flag after committing the transaction, to +* avoid deadlocks on fs_info->qgroup_ioctl_lock with concurrent +* snapshot creation. +*/ +spin_lock (&fs_info->qgroup_lock); + fs_info->quota_root = quota_root; +set_bit (BTRFS_FS_QUOTA_ENABLED , &fs_info->flags); +spin_unlock (&fs_info->qgroup_lock); +But how can the race happen? The changelog should explain that.
Quoted 4 lines
To me this suggests that after we set
BTRFS_FS_QUOTA_ENABLEDand set the quota root, but before we calledqgroup_rescan_init()atbtrfs_quota_enable(), some other task started the rescan worker first - I can only think of someone else calling the ioctl to start the rescan worker (btrfs_ioctl_quota_rescan()).
Yes, rescan worker is triggered by ioctl(btrfs_ioctl_quota_rescan()).
Quoted 11 lines
In that case we get "ret ==
-EINPROGRESS" atbtrfs_quota_enable().So please provide a detailed explanation in the log of how the race can happen.
This solution is also buggy. Because in case of an error, we will leave the quota tree created, the qgroup relation, etc. That is, we don't undo what
btrfs_create_tree(),add_qgroup_item(),add_qgroup_rb(), etc did Which means a futurebtrfs_quota_enable()call would fail, and callingbtrfs_quota_disable()to undo all those things will not work either, becausefs_info->quota_rootis NULL.
As 'ret' is non-zero will release resources which you mentioned.
btrfs_quota_enable
...
out_free_path:
btrfs_free_path(path);
out_free_root:
if (ret)
btrfs_put_root(quota_root);
out:
if (ret) {
ulist_free(fs_info->qgroup_ulist);
fs_info->qgroup_ulist = NULL;
btrfs_sysfs_del_qgroups(fs_info);
}
mutex_unlock(&fs_info->qgroup_ioctl_lock);
if (ret && trans)
btrfs_end_transaction(trans);
else if (trans)
ret = btrfs_end_transaction(trans);
ulist_free(ulist);...
Quoted 19 lines
I would suggest ignoring the error of
qgroup_rescan_init()if it's(quoted diff) +0
-2-EINPROGRESS, and ASSERT if it's anything different from 0 or -EINPROGRESS. Also add a comment mentioning we can get -EINPROGRESSbecause someone may have called the qgroup rescan ioctl.
Thanks.
Quoted 6 lines
qgroup_rescan_zero_tracking(fs_info);fs_info->qgroup_rescan_running= true; btrfs_queue_work(fs_info->qgroup_rescan_workers,Signature · 1 line
2.31.1.
Quoted 178 lines · yebin
Quoted 64 lines · Filipe Manana
Quoted 63 lines · Ye Bin
Syzkaller reported UAF as follows:
==================================================================BUG: KASAN: use-after-free in btrfs_get_global_root+0x663/0xa10 Read of size 4 at addr ffff88811ddbb3c0 by task kworker/u16:1/11 CPU: 4 PID: 11 Comm: kworker/u16:1 Not tainted 6.0.0-rc1-next-20220822+ #2 Workqueue: btrfs-qgroup-rescan btrfs_work_helper Call Trace: <TASK> dump_stack_lvl+0x6e/0x91 print_report.cold+0xb2/0x6bb kasan_report+0xa8/0x130 kasan_check_range+0x13f/0x1d0 btrfs_get_global_root+0x663/0xa10 btrfs_get_fs_root_commit_root+0xa5/0x150 find_parent_nodes+0x92f/0x2990 btrfs_find_all_roots_safe+0x12d/0x220 btrfs_find_all_roots+0xbb/0xd0 btrfs_qgroup_rescan_worker+0x600/0xc30 btrfs_work_helper+0xff/0x750 process_one_work+0x52c/0x930 worker_thread+0x352/0x8c0 kthread+0x1b9/0x200 ret_from_fork+0x22/0x30 </TASK> Allocated by task 1895: kasan_save_stack+0x1e/0x40 __kasan_kmalloc+0xa9/0xe0 btrfs_alloc_root+0x40/0x820 btrfs_create_tree+0xf8/0x500 btrfs_quota_enable+0x30a/0x1120 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd Freed by task 1895: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x40 __kasan_slab_free+0x127/0x1c0 kfree+0xa8/0x2d0 btrfs_put_root+0x1ca/0x230 btrfs_quota_enable+0x87c/0x1120 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd ==================================================================Above issue may happens as follows:
p1 p2btrfs_quota_enablespin_lock(&fs_info->qgroup_lock); fs_info->quota_root = quota_root; spin_unlock(&fs_info->qgroup_lock); ret = qgroup_rescan_init -> return error if (ret) btrfs_put_root(quota_root); kfree(root);Quoted 11 lines
Quoted 11 lines
if (ret) { ulist_free(fs_info->qgroup_ulist); fs_info->qgroup_ulist = NULL; btrfs_sysfs_del_qgroups(fs_info); } btrfs_qgroup_rescan_worker btrfs_find_all_roots btrfs_find_all_roots_safe find_parent_nodes btrfs_get_fs_root_commit_root btrfs_grab_root(fs_info->quota_root) -> quota_root already freedI have described the process of the issue here. I can write it in more detail.
Quoted 88 lines
Quoted 76 lines
Syzkaller also reported another issue:
==================================================================BUG: KASAN: use-after-free in ulist_release+0x30/0xb3 Read of size 8 at addr ffff88811413d048 by task rep/2921 CPU: 3 PID: 2921 Comm: rep Not tainted 6.0.0-rc1-next-20220822+ #3 rep[2921] cmdline: ./rep Call Trace: <TASK> dump_stack_lvl+0x6e/0x91 print_report.cold+0xb2/0x6bb kasan_report+0xa8/0x130 ulist_release+0x30/0xb3 ulist_reinit+0x16/0x56 btrfs_qgroup_free_refroot+0x288/0x3f0 btrfs_qgroup_free_meta_all_pertrans+0xed/0x1e0 commit_fs_roots+0x28c/0x430 btrfs_commit_transaction+0x9a6/0x1b40 btrfs_qgroup_rescan+0x7e/0x130 btrfs_ioctl+0x48ed/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd </TASK> Allocated by task 2900: kasan_save_stack+0x1e/0x40 __kasan_kmalloc+0xa9/0xe0 ulist_alloc+0x5c/0xe0 btrfs_quota_enable+0x1b2/0x1160 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd Freed by task 2900: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x40 __kasan_slab_free+0x127/0x1c0 kfree+0xa8/0x2d0 ulist_free.cold+0x15/0x1a btrfs_quota_enable+0x8bf/0x1160 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd ==================================================================To solve above issues just set '
fs_info->quota_root' afterqgroup_rescan_initreturn success.Signed-off-by Ye Bin <[email protected]>--- fs/btrfs/qgroup.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)fs/btrfs/qgroup.c +0
-10@@ -1158,18 +1158,18 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info)if (ret) goto out_free_path; -/* -* Set quota enabled flag after committing the transaction, to avoid -* deadlocks on fs_info->qgroup_ioctl_lock with concurrent snapshot -* creation. -*/ -spin_lock (&fs_info->qgroup_lock); - fs_info->quota_root = quota_root; -set_bit (BTRFS_FS_QUOTA_ENABLED , &fs_info->flags); -spin_unlock (&fs_info->qgroup_lock); - ret =qgroup_rescan_init (fs_info,0 ,1 );if (!ret) {fs/btrfs/qgroup.c int btrfs_quota_enable(struct btrfs_fs_info *fs_info) · line 1171
Quoted 10 lines
fs/btrfs/qgroup.c +10
-0+/* +* Set quota enabled flag after committing the transaction, to +* avoid deadlocks on fs_info->qgroup_ioctl_lock with concurrent +* snapshot creation. +*/ +spin_lock (&fs_info->qgroup_lock); + fs_info->quota_root = quota_root; +set_bit (BTRFS_FS_QUOTA_ENABLED , &fs_info->flags); +spin_unlock (&fs_info->qgroup_lock); +But how can the race happen? The changelog should explain that.
Quoted 4 lines
To me this suggests that after we set
BTRFS_FS_QUOTA_ENABLEDand set the quota root, but before we calledqgroup_rescan_init()atbtrfs_quota_enable(), some other task started the rescan worker first - I can only think of someone else calling the ioctl to start the rescan worker (btrfs_ioctl_quota_rescan()).Yes, rescan worker is triggered by ioctl(btrfs_ioctl_quota_rescan()).
Quoted 5 lines
In that case we get "ret ==
-EINPROGRESS" atbtrfs_quota_enable().So please provide a detailed explanation in the log of how the race can happen.
Quoted 7 lines
Quoted 6 lines
This solution is also buggy. Because in case of an error, we will leave the quota tree created, the qgroup relation, etc. That is, we don't undo what
btrfs_create_tree(),add_qgroup_item(),add_qgroup_rb(), etc did Which means a futurebtrfs_quota_enable()call would fail, and callingbtrfs_quota_disable()to undo all those things will not work either, becausefs_info->quota_rootis NULL.As 'ret' is non-zero will release resources which you mentioned.
Nop, it won't.
I'm talking about what was *done in the transaction*, which at this point was already committed.
Nothing below under the labels will undo that...
Quoted 40 lines
btrfs_quota_enable
...
out_free_path:btrfs_free_path(path); out_free_root: if (ret) btrfs_put_root(quota_root); out: if (ret) { ulist_free(fs_info->qgroup_ulist); fs_info->qgroup_ulist = NULL; btrfs_sysfs_del_qgroups(fs_info); } mutex_unlock(&fs_info->qgroup_ioctl_lock); if (ret && trans) btrfs_end_transaction(trans); else if (trans) ret = btrfs_end_transaction(trans); ulist_free(ulist);...
Quoted 19 lines
I would suggest ignoring the error of
qgroup_rescan_init()if it's(quoted diff) +0
-2-EINPROGRESS, and ASSERT if it's anything different from 0 or -EINPROGRESS. Also add a comment mentioning we can get -EINPROGRESSbecause someone may have called the qgroup rescan ioctl.
Thanks.
Quoted 6 lines
qgroup_rescan_zero_tracking(fs_info);fs_info->qgroup_rescan_running= true; btrfs_queue_work(fs_info->qgroup_rescan_workers,Signature · 1 line
2.31.1.
Quoted 141 lines · Ye Bin
Syzkaller reported UAF as follows:
==================================================================BUG: KASAN: use-after-free in btrfs_get_global_root+0x663/0xa10 Read of size 4 at addr ffff88811ddbb3c0 by task kworker/u16:1/11 CPU: 4 PID: 11 Comm: kworker/u16:1 Not tainted 6.0.0-rc1-next-20220822+ #2 Workqueue: btrfs-qgroup-rescan btrfs_work_helper Call Trace: <TASK> dump_stack_lvl+0x6e/0x91 print_report.cold+0xb2/0x6bb kasan_report+0xa8/0x130 kasan_check_range+0x13f/0x1d0 btrfs_get_global_root+0x663/0xa10 btrfs_get_fs_root_commit_root+0xa5/0x150 find_parent_nodes+0x92f/0x2990 btrfs_find_all_roots_safe+0x12d/0x220 btrfs_find_all_roots+0xbb/0xd0 btrfs_qgroup_rescan_worker+0x600/0xc30 btrfs_work_helper+0xff/0x750 process_one_work+0x52c/0x930 worker_thread+0x352/0x8c0 kthread+0x1b9/0x200 ret_from_fork+0x22/0x30 </TASK> Allocated by task 1895: kasan_save_stack+0x1e/0x40 __kasan_kmalloc+0xa9/0xe0 btrfs_alloc_root+0x40/0x820 btrfs_create_tree+0xf8/0x500 btrfs_quota_enable+0x30a/0x1120 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd Freed by task 1895: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x40 __kasan_slab_free+0x127/0x1c0 kfree+0xa8/0x2d0 btrfs_put_root+0x1ca/0x230 btrfs_quota_enable+0x87c/0x1120 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd ==================================================================Above issue may happens as follows:
p1 p2btrfs_quota_enable
spin_lock(&fs_info->qgroup_lock);
fs_info->quota_root=quota_root;
spin_unlock(&fs_info->qgroup_lock);ret = qgroup_rescan_init -> return error if (ret) btrfs_put_root(quota_root); kfree(root);if (ret) { ulist_free(fs_info->qgroup_ulist); fs_info->qgroup_ulist = NULL; btrfs_sysfs_del_qgroups(fs_info); } btrfs_qgroup_rescan_worker btrfs_find_all_roots btrfs_find_all_roots_safe find_parent_nodes btrfs_get_fs_root_commit_root btrfs_grab_root(fs_info->quota_root) -> quota_root already freedSyzkaller also reported another issue:
==================================================================BUG: KASAN: use-after-free in ulist_release+0x30/0xb3 Read of size 8 at addr ffff88811413d048 by task rep/2921 CPU: 3 PID: 2921 Comm: rep Not tainted 6.0.0-rc1-next-20220822+ #3 rep[2921] cmdline: ./rep Call Trace: <TASK> dump_stack_lvl+0x6e/0x91 print_report.cold+0xb2/0x6bb kasan_report+0xa8/0x130 ulist_release+0x30/0xb3 ulist_reinit+0x16/0x56 btrfs_qgroup_free_refroot+0x288/0x3f0 btrfs_qgroup_free_meta_all_pertrans+0xed/0x1e0 commit_fs_roots+0x28c/0x430 btrfs_commit_transaction+0x9a6/0x1b40 btrfs_qgroup_rescan+0x7e/0x130 btrfs_ioctl+0x48ed/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd </TASK> Allocated by task 2900: kasan_save_stack+0x1e/0x40 __kasan_kmalloc+0xa9/0xe0 ulist_alloc+0x5c/0xe0 btrfs_quota_enable+0x1b2/0x1160 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd Freed by task 2900: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x40 __kasan_slab_free+0x127/0x1c0 kfree+0xa8/0x2d0 ulist_free.cold+0x15/0x1a btrfs_quota_enable+0x8bf/0x1160 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd ==================================================================To solve above issues just set '
fs_info->quota_root' afterqgroup_rescan_initreturn success.Signed-off-by Ye Bin <[email protected]>--- fs/btrfs/qgroup.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)fs/btrfs/qgroup.c +0
-1@@ -1158,18 +1158,18 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info)if (ret) goto out_free_path; -/*
fs/btrfs/qgroup.c int btrfs_quota_enable(struct btrfs_fs_info *fs_info) · line 1161
Quoted 10 lines
fs/btrfs/qgroup.c +0
-9-* Set quota enabled flag after committing the transaction, to avoid -* deadlocks on fs_info->qgroup_ioctl_lock with concurrent snapshot -* creation. -*/ -spin_lock (&fs_info->qgroup_lock); - fs_info->quota_root = quota_root; -set_bit (BTRFS_FS_QUOTA_ENABLED , &fs_info->flags); -spin_unlock (&fs_info->qgroup_lock); - ret =qgroup_rescan_init (fs_info,0 ,1 );
In addition to what I mentioned in the previous mail, this is also not correct because the qgroup_rescan_init() call will always fail if BTRFS_FS_QUOTA_ENABLED is not set (with an -EBUSY error).
Have you tested this at all? Every attempt to enable quotas will result in a -EBUSY being returned from the ioctl.
All we need is:
fs/btrfs/qgroup.c +11 -0
@@ -1174,6 +1174,17 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info) fs_info->qgroup_rescan_running = true;btrfs_queue_work (fs_info->qgroup_rescan_workers, &fs_info->qgroup_rescan_work); + }else { +/* +* We have set both BTRFS_FS_QUOTA_ENABLED and +* BTRFS_QGROUP_STATUS_FLAG_ON, so we can only fail with +* -EINPROGRESS because someone started the rescan worker with +* the ioctl before we attempted to initialize it. +* Ignore such error, and any other error would need to undo +* everything we did in the transaction we just committed. +*/ +ASSERT (ret == -EINPROGRESS ); + ret =0 ; } out_free_path:
Quoted 17 lines
fs/btrfs/qgroup.c +10
-0if (!ret) { +/* +* Set quota enabled flag after committing the transaction, to +* avoid deadlocks on fs_info->qgroup_ioctl_lock with concurrent +* snapshot creation. +*/ +spin_lock (&fs_info->qgroup_lock); + fs_info->quota_root = quota_root; +set_bit (BTRFS_FS_QUOTA_ENABLED , &fs_info->flags); +spin_unlock (&fs_info->qgroup_lock); +qgroup_rescan_zero_tracking (fs_info); fs_info->qgroup_rescan_running = true; btrfs_queue_work(fs_info->qgroup_rescan_workers,Signature · 1 line
2.31.1
Quoted 18 lines · Ye Bin
Syzkaller reported UAF as follows:
==================================================================BUG: KASAN: use-after-free in btrfs_get_global_root+0x663/0xa10 Read of size 4 at addr ffff88811ddbb3c0 by task kworker/u16:1/11 CPU: 4 PID: 11 Comm: kworker/u16:1 Not tainted 6.0.0-rc1-next-20220822+ #2 Workqueue: btrfs-qgroup-rescan btrfs_work_helper Call Trace: <TASK> dump_stack_lvl+0x6e/0x91 print_report.cold+0xb2/0x6bb kasan_report+0xa8/0x130 kasan_check_range+0x13f/0x1d0 btrfs_get_global_root+0x663/0xa10 btrfs_get_fs_root_commit_root+0xa5/0x150 find_parent_nodes+0x92f/0x2990 btrfs_find_all_roots_safe+0x12d/0x220 btrfs_find_all_roots+0xbb/0xd0
This means, we're looking for all of the parents subvolumes roots of a bytenr.
Function btrfs_get_fs_root_commit_root() can only be called for indirect ref, and at that stage, we're pretty sure that we're grabbing a subvolume root.
Quoted 5 lines
btrfs_qgroup_rescan_worker+0x600/0xc30 btrfs_work_helper+0xff/0x750 process_one_work+0x52c/0x930 worker_thread+0x352/0x8c0 kthread+0x1b9/0x200
Quoted 8 lines
ret_from_fork+0x22/0x30 </TASK> Allocated by task 1895: kasan_save_stack+0x1e/0x40 __kasan_kmalloc+0xa9/0xe0 btrfs_alloc_root+0x40/0x820 btrfs_create_tree+0xf8/0x500
But the allocated-by stack shows, the root belongs to quota root, the only root that is created during qgroup enabling.
This doesn't sound sane to me.
The stacks are saying that, at least some tree blocks are shared by both a subvolume and quota root, which is already against the on-disk format.
Mind to share the reproducer?
Thanks,
Qu
Quoted 134 lines
btrfs_quota_enable+0x30a/0x1120 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd Freed by task 1895: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x40 __kasan_slab_free+0x127/0x1c0 kfree+0xa8/0x2d0 btrfs_put_root+0x1ca/0x230 btrfs_quota_enable+0x87c/0x1120 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd ==================================================================Above issue may happens as follows:
p1 p2btrfs_quota_enablespin_lock(&fs_info->qgroup_lock); fs_info->quota_root = quota_root; spin_unlock(&fs_info->qgroup_lock); ret = qgroup_rescan_init -> return error if (ret) btrfs_put_root(quota_root); kfree(root); if (ret) { ulist_free(fs_info->qgroup_ulist); fs_info->qgroup_ulist = NULL; btrfs_sysfs_del_qgroups(fs_info); } btrfs_qgroup_rescan_worker btrfs_find_all_roots btrfs_find_all_roots_safe find_parent_nodes btrfs_get_fs_root_commit_root btrfs_grab_root(fs_info->quota_root) -> quota_root already freedSyzkaller also reported another issue:
==================================================================BUG: KASAN: use-after-free in ulist_release+0x30/0xb3 Read of size 8 at addr ffff88811413d048 by task rep/2921 CPU: 3 PID: 2921 Comm: rep Not tainted 6.0.0-rc1-next-20220822+ #3 rep[2921] cmdline: ./rep Call Trace: <TASK> dump_stack_lvl+0x6e/0x91 print_report.cold+0xb2/0x6bb kasan_report+0xa8/0x130 ulist_release+0x30/0xb3 ulist_reinit+0x16/0x56 btrfs_qgroup_free_refroot+0x288/0x3f0 btrfs_qgroup_free_meta_all_pertrans+0xed/0x1e0 commit_fs_roots+0x28c/0x430 btrfs_commit_transaction+0x9a6/0x1b40 btrfs_qgroup_rescan+0x7e/0x130 btrfs_ioctl+0x48ed/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd </TASK> Allocated by task 2900: kasan_save_stack+0x1e/0x40 __kasan_kmalloc+0xa9/0xe0 ulist_alloc+0x5c/0xe0 btrfs_quota_enable+0x1b2/0x1160 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd Freed by task 2900: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x40 __kasan_slab_free+0x127/0x1c0 kfree+0xa8/0x2d0 ulist_free.cold+0x15/0x1a btrfs_quota_enable+0x8bf/0x1160 btrfs_ioctl+0x50a3/0x59f0 __x64_sys_ioctl+0x130/0x170 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd ==================================================================To solve above issues just set '
fs_info->quota_root' afterqgroup_rescan_initreturn success.Signed-off-by Ye Bin <[email protected]>--- fs/btrfs/qgroup.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-)fs/btrfs/qgroup.c +10
-10@@ -1158,18 +1158,18 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info)if (ret) goto out_free_path; -/* -* Set quota enabled flag after committing the transaction, to avoid -* deadlocks on fs_info->qgroup_ioctl_lock with concurrent snapshot -* creation. -*/ -spin_lock (&fs_info->qgroup_lock); - fs_info->quota_root = quota_root; -set_bit (BTRFS_FS_QUOTA_ENABLED , &fs_info->flags); -spin_unlock (&fs_info->qgroup_lock); - ret =qgroup_rescan_init (fs_info,0 ,1 );if (!ret) { +/* +* Set quota enabled flag after committing the transaction, to +* avoid deadlocks on fs_info->qgroup_ioctl_lock with concurrent +* snapshot creation. +*/ +spin_lock (&fs_info->qgroup_lock); + fs_info->quota_root = quota_root; +set_bit (BTRFS_FS_QUOTA_ENABLED , &fs_info->flags); +spin_unlock (&fs_info->qgroup_lock); +qgroup_rescan_zero_tracking (fs_info); fs_info->qgroup_rescan_running = true; btrfs_queue_work(fs_info->qgroup_rescan_workers,