ocfs2: fix deadlock in inline-data truncate transactions
PATCH 1/1 ocfs2: fix deadlock in inline-data truncate transactions
[BUG]
Shrinking an inline-data file can trigger a real ABBA deadlock cycle and lockdep warning:
WARNING: possible circular locking dependency detected
------------------------------------------------------
syz.0.6/353 is trying to acquire lock:ffff888015936610 (sb_internal#2){.+.+}-{0:0}, at: ocfs2_truncate_inline+0x389/0xbf0 fs/ocfs2/alloc.c:7436
but task is already holding lock:ffff888019156a20 (&ocfs2_file_ip_alloc_sem_key){++++}-{4:4}, at: ocfs2_truncate_file+0x23a/0x13c0 fs/ocfs2/file.c:471
which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #3 (&ocfs2_file_ip_alloc_sem_key){++++}-{4:4}:
down_write+0x8f/0x200 kernel/locking/rwsem.c:1590
ocfs2_xattr_ibody_set+0x119/0xc50 fs/ocfs2/xattr.c:2783
__ocfs2_xattr_set_handle+0xdb/0xdb0 fs/ocfs2/xattr.c:3322
ocfs2_xattr_set+0x1447/0x2610 fs/ocfs2/xattr.c:3650
ocfs2_xattr_security_set+0x37/0x50 fs/ocfs2/xattr.c:7241
__vfs_setxattr+0x14f/0x1c0 fs/xattr.c:200
__vfs_setxattr_noperm+0x10b/0x5c0 fs/xattr.c:234
__vfs_setxattr_locked+0x172/0x240 fs/xattr.c:295
vfs_setxattr+0x167/0x390 fs/xattr.c:321
do_setxattr+0x13c/0x180 fs/xattr.c:636
filename_setxattr+0x16b/0x1c0 fs/xattr.c:665
path_setxattrat+0x1d8/0x280 fs/xattr.c:713
__do_sys_setxattr fs/xattr.c:747 [inline]
__se_sys_setxattr fs/xattr.c:743 [inline]
__x64_sys_setxattr+0xcd/0x150 fs/xattr.c:743
x64_sys_call+0x148c/0x26a0 arch/x86/include/generated/asm/syscalls_64.h:189
do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline]
do_syscall_64+0x93/0xf80 arch/x86/entry/syscall_64.c:94
entry_SYSCALL_64_after_hwframe+0x76/0x7e-> #2 (jbd2_handle){++++}-{0:0}:
start_this_handle+0x5c7/0x13c0 fs/jbd2/transaction.c:444
jbd2__journal_start+0x397/0x690 fs/jbd2/transaction.c:501
jbd2_journal_start+0x31/0x50 fs/jbd2/transaction.c:540
ocfs2_start_trans+0x39b/0x870 fs/ocfs2/journal.c:374
ocfs2_xattr_set+0x1401/0x2610 fs/ocfs2/xattr.c:3643
ocfs2_xattr_security_set+0x37/0x50 fs/ocfs2/xattr.c:7241
__vfs_setxattr+0x14f/0x1c0 fs/xattr.c:200
__vfs_setxattr_noperm+0x10b/0x5c0 fs/xattr.c:234
__vfs_setxattr_locked+0x172/0x240 fs/xattr.c:295
vfs_setxattr+0x167/0x390 fs/xattr.c:321
do_setxattr+0x13c/0x180 fs/xattr.c:636
filename_setxattr+0x16b/0x1c0 fs/xattr.c:665
path_setxattrat+0x1d8/0x280 fs/xattr.c:713
__do_sys_setxattr fs/xattr.c:747 [inline]
__se_sys_setxattr fs/xattr.c:743 [inline]
__x64_sys_setxattr+0xcd/0x150 fs/xattr.c:743
x64_sys_call+0x148c/0x26a0 arch/x86/include/generated/asm/syscalls_64.h:189
do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline]
do_syscall_64+0x93/0xf80 arch/x86/entry/syscall_64.c:94
entry_SYSCALL_64_after_hwframe+0x76/0x7e-> #1 (&journal->j_trans_barrier){.+.+}-{4:4}:
down_read+0x9c/0x4a0 kernel/locking/rwsem.c:1537
ocfs2_start_trans+0x390/0x870 fs/ocfs2/journal.c:372
ocfs2_xattr_set+0x1401/0x2610 fs/ocfs2/xattr.c:3643
ocfs2_xattr_security_set+0x37/0x50 fs/ocfs2/xattr.c:7241
__vfs_setxattr+0x14f/0x1c0 fs/xattr.c:200
__vfs_setxattr_noperm+0x10b/0x5c0 fs/xattr.c:234
__vfs_setxattr_locked+0x172/0x240 fs/xattr.c:295
vfs_setxattr+0x167/0x390 fs/xattr.c:321
do_setxattr+0x13c/0x180 fs/xattr.c:636
filename_setxattr+0x16b/0x1c0 fs/xattr.c:665
path_setxattrat+0x1d8/0x280 fs/xattr.c:713
__do_sys_setxattr fs/xattr.c:747 [inline]
__se_sys_setxattr fs/xattr.c:743 [inline]
__x64_sys_setxattr+0xcd/0x150 fs/xattr.c:743
x64_sys_call+0x148c/0x26a0 arch/x86/include/generated/asm/syscalls_64.h:189
do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline]
do_syscall_64+0x93/0xf80 arch/x86/entry/syscall_64.c:94
entry_SYSCALL_64_after_hwframe+0x76/0x7e-> #0 (sb_internal#2){.+.+}-{0:0}:
check_prev_add kernel/locking/lockdep.c:3165 [inline]
check_prevs_add kernel/locking/lockdep.c:3284 [inline]
validate_chain kernel/locking/lockdep.c:3908 [inline]
__lock_acquire+0x14ae/0x21e0 kernel/locking/lockdep.c:5237
lock_acquire kernel/locking/lockdep.c:5868 [inline]
lock_acquire+0x169/0x2f0 kernel/locking/lockdep.c:5825
percpu_down_read_internal include/linux/percpu-rwsem.h:53 [inline]
percpu_down_read_freezable include/linux/percpu-rwsem.h:83 [inline]
__sb_start_write include/linux/fs.h:1916 [inline]
sb_start_intwrite include/linux/fs.h:2099 [inline]
ocfs2_start_trans+0x2a8/0x870 fs/ocfs2/journal.c:370
ocfs2_truncate_inline+0x389/0xbf0 fs/ocfs2/alloc.c:7436
ocfs2_truncate_file+0x6d3/0x13c0 fs/ocfs2/file.c:488
ocfs2_setattr+0xa6d/0x1fd0 fs/ocfs2/file.c:1212
notify_change+0x4b5/0x1030 fs/attr.c:546
do_truncate+0x1d2/0x230 fs/open.c:68
vfs_truncate+0x56c/0x670 fs/open.c:118
do_sys_truncate fs/open.c:141 [inline]
__do_sys_truncate fs/open.c:153 [inline]
__se_sys_truncate fs/open.c:151 [inline]
__x64_sys_truncate+0x165/0x1d0 fs/open.c:151
x64_sys_call+0xbc8/0x26a0 arch/x86/include/generated/asm/syscalls_64.h:77
do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline]
do_syscall_64+0x93/0xf80 arch/x86/entry/syscall_64.c:94
entry_SYSCALL_64_after_hwframe+0x76/0x7eother info that might help us debug this:
Chain exists of: sb_internal#2 --> jbd2_handle --> &ocfs2_file_ip_alloc_sem_key Possible unsafe locking scenario:
CPU0 CPU1
---- ----
lock(&ocfs2_file_ip_alloc_sem_key);
lock(jbd2_handle);
lock(&ocfs2_file_ip_alloc_sem_key); rlock(sb_internal#2);
*** DEADLOCK ***
3 locks held by syz.0.6/353:
#0: ffff888015936420 (sb_writers#12){.+.+}-{0:0}, at: vfs_truncate+0x30c/0x670 fs/open.c:96
#1: ffff888019156d80 (&sb->s_type->i_mutex_key#18){+.+.}-{4:4}, at: inode_lock_killable include/linux/fs.h:985 [inline] #1: ffff888019156d80 (&sb->s_type->i_mutex_key#18){+.+.}-{4:4}, at: do_truncate+0x139/0x230 fs/open.c:63 #2: ffff888019156a20 (&ocfs2_file_ip_alloc_sem_key){++++}-{4:4}, at: ocfs2_truncate_file+0x23a/0x13c0 fs/ocfs2/file.c:471
Call Trace: ... lock_acquire+0x169/0x2f0 kernel/locking/lockdep.c:5825 percpu_down_read_internal include/linux/percpu-rwsem.h:53 [inline] percpu_down_read_freezable include/linux/percpu-rwsem.h:83 [inline] __sb_start_write include/linux/fs.h:1916 [inline] sb_start_intwrite include/linux/fs.h:2099 [inline] ocfs2_start_trans+0x2a8/0x870 fs/ocfs2/journal.c:370 ocfs2_truncate_inline+0x389/0xbf0 fs/ocfs2/alloc.c:7436 ocfs2_truncate_file+0x6d3/0x13c0 fs/ocfs2/file.c:488 ocfs2_setattr+0xa6d/0x1fd0 fs/ocfs2/file.c:1212 notify_change+0x4b5/0x1030 fs/attr.c:546 do_truncate+0x1d2/0x230 fs/open.c:68 vfs_truncate+0x56c/0x670 fs/open.c:118 do_sys_truncate fs/open.c:141 [inline] __do_sys_truncate fs/open.c:153 [inline] __se_sys_truncate fs/open.c:151 [inline] __x64_sys_truncate+0x165/0x1d0 fs/open.c:151 ...
[CAUSE]ocfs2_truncate_file() and inline range removal hold ip_alloc_sem when calling ocfs2_truncate_inline(). The helper starts a transaction, which adds an ip_alloc_sem-to-transaction dependency opposite to the existing transaction-to-ip_alloc_sem order in the xattr path.
[FIX]
Make ocfs2_truncate_inline() use a caller-owned transaction handle. Start the inline update transaction before taking ip_alloc_sem in the truncate and range-removal callers, and release an unused handle if inline data was converted to extents while waiting for the semaphore. This keeps the inline update serialized by ip_alloc_sem while preserving a single transaction-before-ip_alloc_sem order.
--- fs/ocfs2/alloc.c | 18 ++------- fs/ocfs2/alloc.h | 5 ++- fs/ocfs2/file.c | 83 +++++++++++++++++++++++++++++++++++------ fs/ocfs2/file.h | 2 +- fs/ocfs2/refcounttree.c | 2 +- 5 files changed, 81 insertions(+), 29 deletions(-)
fs/ocfs2/alloc.c +4 -14
@@ -7412,12 +7412,12 @@ int ocfs2_commit_truncate(struct ocfs2_super *osb,/* * 'start' is inclusive, 'end' is not. */ -int ocfs2_truncate_inline (struct inode *inode,struct buffer_head *di_bh, -unsigned int start,unsigned int end,int trunc) +int ocfs2_truncate_inline (handle_t *handle,struct inode *inode, +struct buffer_head *di_bh,unsigned int start, +unsigned int end,int trunc) {int ret;unsigned int numbytes; -handle_t *handle;struct ocfs2_super *osb =OCFS2_SB (inode->i_sb);struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data;struct ocfs2_inline_data *idata = &di->id2.i_data;
@@ -7444,18 +7444,11 @@ int ocfs2_truncate_inline(struct inode *inode, struct buffer_head *di_bh, goto out; } - handle =ocfs2_start_trans (osb,OCFS2_INODE_UPDATE_CREDITS ); -if (IS_ERR (handle)) { - ret =PTR_ERR (handle); -mlog_errno (ret); - goto out; - } - ret =ocfs2_journal_access_di (handle,INODE_CACHE (inode), di_bh,OCFS2_JOURNAL_ACCESS_WRITE );if (ret) {mlog_errno (ret); - goto out_commit; + goto out; } numbytes = end - start;
@@ -7481,9 +7474,6 @@ int ocfs2_truncate_inline(struct inode *inode, struct buffer_head *di_bh,ocfs2_update_inode_fsync_trans (handle, inode,1 );ocfs2_journal_dirty (handle, di_bh); -out_commit: -ocfs2_commit_trans (osb, handle); - out:return ret; }
fs/ocfs2/alloc.h +3 -2
@@ -217,8 +217,9 @@ int ocfs2_zero_range_for_truncate(struct inode *inode, handle_t *handle,int ocfs2_commit_truncate (struct ocfs2_super *osb,struct inode *inode,struct buffer_head *di_bh); -int ocfs2_truncate_inline (struct inode *inode,struct buffer_head *di_bh, -unsigned int start,unsigned int end,int trunc); +int ocfs2_truncate_inline (handle_t *handle,struct inode *inode, +struct buffer_head *di_bh,unsigned int start, +unsigned int end,int trunc); int ocfs2_find_leaf(struct ocfs2_caching_info *ci, struct ocfs2_extent_list *root_el, u32 cpos,
fs/ocfs2/file.c +72 -11
145 lines not on this page · raw message
fs/ocfs2/file.h +1 -1
@@ -70,5 +70,5 @@ int ocfs2_check_range_for_refcount(struct inode *inode, loff_t pos, size_t count);int ocfs2_remove_inode_range (struct inode *inode,struct buffer_head *di_bh,u64 byte_start, -u64 byte_len); +u64 byte_len,handle_t *inline_handle);#endif /* OCFS2_FILE_H */
fs/ocfs2/refcounttree.c +1 -1
@@ -4560,7 +4560,7 @@ static loff_t ocfs2_reflink_remap_extent(struct inode *s_inode,/* Punch out the dest range. */ pstart =ocfs2_clusters_to_bytes (t_inode->i_sb, tpos); plen =ocfs2_clusters_to_bytes (t_inode->i_sb, num_clusters); - ret =ocfs2_remove_inode_range (t_inode, t_bh, pstart, plen); + ret =ocfs2_remove_inode_range (t_inode, t_bh, pstart, plen,NULL );if (ret) {mlog_errno (ret); goto out;
Signature · 1 line
2.43.0
Joseph Qi
maintainerQuoted 154 lines · ZhengYuan Huang
[BUG]
Shrinking an inline-data file can trigger a real ABBA deadlock cycle and lockdep warning:WARNING: possible circular locking dependency detected
------------------------------------------------------
syz.0.6/353 is trying to acquire lock:ffff888015936610(sb_internal#2){.+.+}-{0:0}, at:ocfs2_truncate_inline+0x389/0xbf0fs/ocfs2/alloc.c:7436but task is already holding lock:
ffff888019156a20(&ocfs2_file_ip_alloc_sem_key){++++}-{4:4}, at:ocfs2_truncate_file+0x23a/0x13c0fs/ocfs2/file.c:471which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #3 (&ocfs2_file_ip_alloc_sem_key){++++}-{4:4}:
down_write+0x8f/0x200 kernel/locking/rwsem.c:1590 ocfs2_xattr_ibody_set+0x119/0xc50 fs/ocfs2/xattr.c:2783 __ocfs2_xattr_set_handle+0xdb/0xdb0 fs/ocfs2/xattr.c:3322 ocfs2_xattr_set+0x1447/0x2610 fs/ocfs2/xattr.c:3650 ocfs2_xattr_security_set+0x37/0x50 fs/ocfs2/xattr.c:7241 __vfs_setxattr+0x14f/0x1c0 fs/xattr.c:200 __vfs_setxattr_noperm+0x10b/0x5c0 fs/xattr.c:234 __vfs_setxattr_locked+0x172/0x240 fs/xattr.c:295 vfs_setxattr+0x167/0x390 fs/xattr.c:321 do_setxattr+0x13c/0x180 fs/xattr.c:636 filename_setxattr+0x16b/0x1c0 fs/xattr.c:665 path_setxattrat+0x1d8/0x280 fs/xattr.c:713 __do_sys_setxattr fs/xattr.c:747 [inline] __se_sys_setxattr fs/xattr.c:743 [inline] __x64_sys_setxattr+0xcd/0x150 fs/xattr.c:743 x64_sys_call+0x148c/0x26a0 arch/x86/include/generated/asm/syscalls_64.h:189 do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline] do_syscall_64+0x93/0xf80 arch/x86/entry/syscall_64.c:94 entry_SYSCALL_64_after_hwframe+0x76/0x7e-> #2 (jbd2_handle){++++}-{0:0}:
start_this_handle+0x5c7/0x13c0 fs/jbd2/transaction.c:444 jbd2__journal_start+0x397/0x690 fs/jbd2/transaction.c:501 jbd2_journal_start+0x31/0x50 fs/jbd2/transaction.c:540 ocfs2_start_trans+0x39b/0x870 fs/ocfs2/journal.c:374 ocfs2_xattr_set+0x1401/0x2610 fs/ocfs2/xattr.c:3643 ocfs2_xattr_security_set+0x37/0x50 fs/ocfs2/xattr.c:7241 __vfs_setxattr+0x14f/0x1c0 fs/xattr.c:200 __vfs_setxattr_noperm+0x10b/0x5c0 fs/xattr.c:234 __vfs_setxattr_locked+0x172/0x240 fs/xattr.c:295 vfs_setxattr+0x167/0x390 fs/xattr.c:321 do_setxattr+0x13c/0x180 fs/xattr.c:636 filename_setxattr+0x16b/0x1c0 fs/xattr.c:665 path_setxattrat+0x1d8/0x280 fs/xattr.c:713 __do_sys_setxattr fs/xattr.c:747 [inline] __se_sys_setxattr fs/xattr.c:743 [inline] __x64_sys_setxattr+0xcd/0x150 fs/xattr.c:743 x64_sys_call+0x148c/0x26a0 arch/x86/include/generated/asm/syscalls_64.h:189 do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline] do_syscall_64+0x93/0xf80 arch/x86/entry/syscall_64.c:94 entry_SYSCALL_64_after_hwframe+0x76/0x7e-> #1 (&journal->j_trans_barrier){.+.+}-{4:4}:
down_read+0x9c/0x4a0 kernel/locking/rwsem.c:1537 ocfs2_start_trans+0x390/0x870 fs/ocfs2/journal.c:372 ocfs2_xattr_set+0x1401/0x2610 fs/ocfs2/xattr.c:3643 ocfs2_xattr_security_set+0x37/0x50 fs/ocfs2/xattr.c:7241 __vfs_setxattr+0x14f/0x1c0 fs/xattr.c:200 __vfs_setxattr_noperm+0x10b/0x5c0 fs/xattr.c:234 __vfs_setxattr_locked+0x172/0x240 fs/xattr.c:295 vfs_setxattr+0x167/0x390 fs/xattr.c:321 do_setxattr+0x13c/0x180 fs/xattr.c:636 filename_setxattr+0x16b/0x1c0 fs/xattr.c:665 path_setxattrat+0x1d8/0x280 fs/xattr.c:713 __do_sys_setxattr fs/xattr.c:747 [inline] __se_sys_setxattr fs/xattr.c:743 [inline] __x64_sys_setxattr+0xcd/0x150 fs/xattr.c:743 x64_sys_call+0x148c/0x26a0 arch/x86/include/generated/asm/syscalls_64.h:189 do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline] do_syscall_64+0x93/0xf80 arch/x86/entry/syscall_64.c:94 entry_SYSCALL_64_after_hwframe+0x76/0x7e-> #0 (sb_internal#2){.+.+}-{0:0}:
check_prev_add kernel/locking/lockdep.c:3165 [inline] check_prevs_add kernel/locking/lockdep.c:3284 [inline] validate_chain kernel/locking/lockdep.c:3908 [inline] __lock_acquire+0x14ae/0x21e0 kernel/locking/lockdep.c:5237 lock_acquire kernel/locking/lockdep.c:5868 [inline] lock_acquire+0x169/0x2f0 kernel/locking/lockdep.c:5825 percpu_down_read_internal include/linux/percpu-rwsem.h:53 [inline] percpu_down_read_freezable include/linux/percpu-rwsem.h:83 [inline] __sb_start_write include/linux/fs.h:1916 [inline] sb_start_intwrite include/linux/fs.h:2099 [inline] ocfs2_start_trans+0x2a8/0x870 fs/ocfs2/journal.c:370 ocfs2_truncate_inline+0x389/0xbf0 fs/ocfs2/alloc.c:7436 ocfs2_truncate_file+0x6d3/0x13c0 fs/ocfs2/file.c:488 ocfs2_setattr+0xa6d/0x1fd0 fs/ocfs2/file.c:1212 notify_change+0x4b5/0x1030 fs/attr.c:546 do_truncate+0x1d2/0x230 fs/open.c:68 vfs_truncate+0x56c/0x670 fs/open.c:118 do_sys_truncate fs/open.c:141 [inline] __do_sys_truncate fs/open.c:153 [inline] __se_sys_truncate fs/open.c:151 [inline] __x64_sys_truncate+0x165/0x1d0 fs/open.c:151 x64_sys_call+0xbc8/0x26a0 arch/x86/include/generated/asm/syscalls_64.h:77 do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline] do_syscall_64+0x93/0xf80 arch/x86/entry/syscall_64.c:94 entry_SYSCALL_64_after_hwframe+0x76/0x7eother info that might help us debug this:
Chain exists of: sb_internal#2 --> jbd2_handle --> &ocfs2_file_ip_alloc_sem_key Possible unsafe locking scenario:CPU0 CPU1 ---- ---- lock(&ocfs2_file_ip_alloc_sem_key); lock(jbd2_handle); lock(&ocfs2_file_ip_alloc_sem_key);
rlock(sb_internal#2);*** DEADLOCK ***
3 locks held by syz.0.6/353:
#0:ffff888015936420(sb_writers#12){.+.+}-{0:0}, at:vfs_truncate+0x30c/0x670fs/open.c:96
#1:ffff888019156d80(&sb->s_type->i_mutex_key#18){+.+.}-{4:4}, at:inode_lock_killableinclude/linux/fs.h:985[inline] #1:ffff888019156d80(&sb->s_type->i_mutex_key#18){+.+.}-{4:4}, at:do_truncate+0x139/0x230fs/open.c:63#2:ffff888019156a20(&ocfs2_file_ip_alloc_sem_key){++++}-{4:4}, at:ocfs2_truncate_file+0x23a/0x13c0fs/ocfs2/file.c:471Call Trace: ... lock_acquire+0x169/0x2f0 kernel/locking/lockdep.c:5825 percpu_down_read_internal include/linux/percpu-rwsem.h:53 [inline] percpu_down_read_freezable include/linux/percpu-rwsem.h:83 [inline] __sb_start_write include/linux/fs.h:1916 [inline] sb_start_intwrite include/linux/fs.h:2099 [inline] ocfs2_start_trans+0x2a8/0x870 fs/ocfs2/journal.c:370 ocfs2_truncate_inline+0x389/0xbf0 fs/ocfs2/alloc.c:7436 ocfs2_truncate_file+0x6d3/0x13c0 fs/ocfs2/file.c:488 ocfs2_setattr+0xa6d/0x1fd0 fs/ocfs2/file.c:1212 notify_change+0x4b5/0x1030 fs/attr.c:546 do_truncate+0x1d2/0x230 fs/open.c:68 vfs_truncate+0x56c/0x670 fs/open.c:118 do_sys_truncate fs/open.c:141 [inline] __do_sys_truncate fs/open.c:153 [inline] __se_sys_truncate fs/open.c:151 [inline] __x64_sys_truncate+0x165/0x1d0 fs/open.c:151 ...[CAUSE]
ocfs2_truncate_file()and inline range removal holdip_alloc_semwhen callingocfs2_truncate_inline(). The helper starts a transaction, which adds an ip_alloc_sem-to-transaction dependency opposite to the existing transaction-to-ip_alloc_sem order in the xattr path.
Quoted 3 lines
[FIX]
Makeocfs2_truncate_inline()use a caller-owned transaction handle. Start the inline update transaction before takingip_alloc_semin the truncate
But most places (e.g. allocation path) all use ip_alloc_sem -> start transaction. So I think the correct choice is to fix the xattr side.
Thanks,
Joseph
Quoted 7 lines
and range-removal callers, and release an unused handle if inline data was converted to extents while waiting for the semaphore. This keeps the inline update serialized by
ip_alloc_semwhile preserving a single transaction-before-ip_alloc_sem order.Fixes 85db90e77806 ("ocfs2/xattr: Merge xattr set transaction.")Signed-off-by ZhengYuan Huang <[email protected]>
Quoted 162 lines · Joseph Qi
Quoted 154 lines · ZhengYuan Huang
[BUG]
Shrinking an inline-data file can trigger a real ABBA deadlock cycle and lockdep warning:WARNING: possible circular locking dependency detected
------------------------------------------------------
syz.0.6/353 is trying to acquire lock:ffff888015936610(sb_internal#2){.+.+}-{0:0}, at:ocfs2_truncate_inline+0x389/0xbf0fs/ocfs2/alloc.c:7436but task is already holding lock:
ffff888019156a20(&ocfs2_file_ip_alloc_sem_key){++++}-{4:4}, at:ocfs2_truncate_file+0x23a/0x13c0fs/ocfs2/file.c:471which lock already depends on the new lock.
the existing dependency chain (in reverse order) is:
-> #3 (&ocfs2_file_ip_alloc_sem_key){++++}-{4:4}:
down_write+0x8f/0x200 kernel/locking/rwsem.c:1590 ocfs2_xattr_ibody_set+0x119/0xc50 fs/ocfs2/xattr.c:2783 __ocfs2_xattr_set_handle+0xdb/0xdb0 fs/ocfs2/xattr.c:3322 ocfs2_xattr_set+0x1447/0x2610 fs/ocfs2/xattr.c:3650 ocfs2_xattr_security_set+0x37/0x50 fs/ocfs2/xattr.c:7241 __vfs_setxattr+0x14f/0x1c0 fs/xattr.c:200 __vfs_setxattr_noperm+0x10b/0x5c0 fs/xattr.c:234 __vfs_setxattr_locked+0x172/0x240 fs/xattr.c:295 vfs_setxattr+0x167/0x390 fs/xattr.c:321 do_setxattr+0x13c/0x180 fs/xattr.c:636 filename_setxattr+0x16b/0x1c0 fs/xattr.c:665 path_setxattrat+0x1d8/0x280 fs/xattr.c:713 __do_sys_setxattr fs/xattr.c:747 [inline] __se_sys_setxattr fs/xattr.c:743 [inline] __x64_sys_setxattr+0xcd/0x150 fs/xattr.c:743 x64_sys_call+0x148c/0x26a0 arch/x86/include/generated/asm/syscalls_64.h:189 do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline] do_syscall_64+0x93/0xf80 arch/x86/entry/syscall_64.c:94 entry_SYSCALL_64_after_hwframe+0x76/0x7e-> #2 (jbd2_handle){++++}-{0:0}:
start_this_handle+0x5c7/0x13c0 fs/jbd2/transaction.c:444 jbd2__journal_start+0x397/0x690 fs/jbd2/transaction.c:501 jbd2_journal_start+0x31/0x50 fs/jbd2/transaction.c:540 ocfs2_start_trans+0x39b/0x870 fs/ocfs2/journal.c:374 ocfs2_xattr_set+0x1401/0x2610 fs/ocfs2/xattr.c:3643 ocfs2_xattr_security_set+0x37/0x50 fs/ocfs2/xattr.c:7241 __vfs_setxattr+0x14f/0x1c0 fs/xattr.c:200 __vfs_setxattr_noperm+0x10b/0x5c0 fs/xattr.c:234 __vfs_setxattr_locked+0x172/0x240 fs/xattr.c:295 vfs_setxattr+0x167/0x390 fs/xattr.c:321 do_setxattr+0x13c/0x180 fs/xattr.c:636 filename_setxattr+0x16b/0x1c0 fs/xattr.c:665 path_setxattrat+0x1d8/0x280 fs/xattr.c:713 __do_sys_setxattr fs/xattr.c:747 [inline] __se_sys_setxattr fs/xattr.c:743 [inline] __x64_sys_setxattr+0xcd/0x150 fs/xattr.c:743 x64_sys_call+0x148c/0x26a0 arch/x86/include/generated/asm/syscalls_64.h:189 do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline] do_syscall_64+0x93/0xf80 arch/x86/entry/syscall_64.c:94 entry_SYSCALL_64_after_hwframe+0x76/0x7e-> #1 (&journal->j_trans_barrier){.+.+}-{4:4}:
down_read+0x9c/0x4a0 kernel/locking/rwsem.c:1537 ocfs2_start_trans+0x390/0x870 fs/ocfs2/journal.c:372 ocfs2_xattr_set+0x1401/0x2610 fs/ocfs2/xattr.c:3643 ocfs2_xattr_security_set+0x37/0x50 fs/ocfs2/xattr.c:7241 __vfs_setxattr+0x14f/0x1c0 fs/xattr.c:200 __vfs_setxattr_noperm+0x10b/0x5c0 fs/xattr.c:234 __vfs_setxattr_locked+0x172/0x240 fs/xattr.c:295 vfs_setxattr+0x167/0x390 fs/xattr.c:321 do_setxattr+0x13c/0x180 fs/xattr.c:636 filename_setxattr+0x16b/0x1c0 fs/xattr.c:665 path_setxattrat+0x1d8/0x280 fs/xattr.c:713 __do_sys_setxattr fs/xattr.c:747 [inline] __se_sys_setxattr fs/xattr.c:743 [inline] __x64_sys_setxattr+0xcd/0x150 fs/xattr.c:743 x64_sys_call+0x148c/0x26a0 arch/x86/include/generated/asm/syscalls_64.h:189 do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline] do_syscall_64+0x93/0xf80 arch/x86/entry/syscall_64.c:94 entry_SYSCALL_64_after_hwframe+0x76/0x7e-> #0 (sb_internal#2){.+.+}-{0:0}:
check_prev_add kernel/locking/lockdep.c:3165 [inline] check_prevs_add kernel/locking/lockdep.c:3284 [inline] validate_chain kernel/locking/lockdep.c:3908 [inline] __lock_acquire+0x14ae/0x21e0 kernel/locking/lockdep.c:5237 lock_acquire kernel/locking/lockdep.c:5868 [inline] lock_acquire+0x169/0x2f0 kernel/locking/lockdep.c:5825 percpu_down_read_internal include/linux/percpu-rwsem.h:53 [inline] percpu_down_read_freezable include/linux/percpu-rwsem.h:83 [inline] __sb_start_write include/linux/fs.h:1916 [inline] sb_start_intwrite include/linux/fs.h:2099 [inline] ocfs2_start_trans+0x2a8/0x870 fs/ocfs2/journal.c:370 ocfs2_truncate_inline+0x389/0xbf0 fs/ocfs2/alloc.c:7436 ocfs2_truncate_file+0x6d3/0x13c0 fs/ocfs2/file.c:488 ocfs2_setattr+0xa6d/0x1fd0 fs/ocfs2/file.c:1212 notify_change+0x4b5/0x1030 fs/attr.c:546 do_truncate+0x1d2/0x230 fs/open.c:68 vfs_truncate+0x56c/0x670 fs/open.c:118 do_sys_truncate fs/open.c:141 [inline] __do_sys_truncate fs/open.c:153 [inline] __se_sys_truncate fs/open.c:151 [inline] __x64_sys_truncate+0x165/0x1d0 fs/open.c:151 x64_sys_call+0xbc8/0x26a0 arch/x86/include/generated/asm/syscalls_64.h:77 do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline] do_syscall_64+0x93/0xf80 arch/x86/entry/syscall_64.c:94 entry_SYSCALL_64_after_hwframe+0x76/0x7eother info that might help us debug this:
Chain exists of: sb_internal#2 --> jbd2_handle --> &ocfs2_file_ip_alloc_sem_key Possible unsafe locking scenario:CPU0 CPU1 ---- ---- lock(&ocfs2_file_ip_alloc_sem_key); lock(jbd2_handle); lock(&ocfs2_file_ip_alloc_sem_key);
rlock(sb_internal#2);*** DEADLOCK ***
3 locks held by syz.0.6/353:
#0:ffff888015936420(sb_writers#12){.+.+}-{0:0}, at:vfs_truncate+0x30c/0x670fs/open.c:96
#1:ffff888019156d80(&sb->s_type->i_mutex_key#18){+.+.}-{4:4}, at:inode_lock_killableinclude/linux/fs.h:985[inline] #1:ffff888019156d80(&sb->s_type->i_mutex_key#18){+.+.}-{4:4}, at:do_truncate+0x139/0x230fs/open.c:63#2:ffff888019156a20(&ocfs2_file_ip_alloc_sem_key){++++}-{4:4}, at:ocfs2_truncate_file+0x23a/0x13c0fs/ocfs2/file.c:471Call Trace: ... lock_acquire+0x169/0x2f0 kernel/locking/lockdep.c:5825 percpu_down_read_internal include/linux/percpu-rwsem.h:53 [inline] percpu_down_read_freezable include/linux/percpu-rwsem.h:83 [inline] __sb_start_write include/linux/fs.h:1916 [inline] sb_start_intwrite include/linux/fs.h:2099 [inline] ocfs2_start_trans+0x2a8/0x870 fs/ocfs2/journal.c:370 ocfs2_truncate_inline+0x389/0xbf0 fs/ocfs2/alloc.c:7436 ocfs2_truncate_file+0x6d3/0x13c0 fs/ocfs2/file.c:488 ocfs2_setattr+0xa6d/0x1fd0 fs/ocfs2/file.c:1212 notify_change+0x4b5/0x1030 fs/attr.c:546 do_truncate+0x1d2/0x230 fs/open.c:68 vfs_truncate+0x56c/0x670 fs/open.c:118 do_sys_truncate fs/open.c:141 [inline] __do_sys_truncate fs/open.c:153 [inline] __se_sys_truncate fs/open.c:151 [inline] __x64_sys_truncate+0x165/0x1d0 fs/open.c:151 ...[CAUSE]
ocfs2_truncate_file()and inline range removal holdip_alloc_semwhen callingocfs2_truncate_inline(). The helper starts a transaction, which adds an ip_alloc_sem-to-transaction dependency opposite to the existing transaction-to-ip_alloc_sem order in the xattr path.Quoted 3 lines
[FIX]
Makeocfs2_truncate_inline()use a caller-owned transaction handle. Start the inline update transaction before takingip_alloc_semin the truncate
Quoted 5 lines
But most places (e.g. allocation path) all use
ip_alloc_sem-> start transaction. So I think the correct choice is to fix the xattr side.Thanks,
Joseph
Thanks for the suggestion. I've fixed the xattr side and sent out v2 of the patch.
Thanks,
ZhengYuan Huang