f2fs: Fix deadlock in f2fs_gc() context during atomic files handling
PATCH 1/1 f2fs: Fix deadlock in f2fs_gc() context during atomic files handling
The FS got stuck in the below stack when the storage is almost full/dirty condition (when FG_GC is being done).
schedule_timeoutio_schedule_timeoutcongestion_waitf2fs_drop_inmem_pages_allf2fs_gcf2fs_balance_fs__write_node_pagef2fs_fsync_node_pagesf2fs_do_sync_filef2fs_ioctl
The root cause for this issue is there is a potential infinite loop in f2fs_drop_inmem_pages_all() for the case where gc_failure is true and when there an inode whose i_gc_failures[GC_FAILURE_ATOMIC] is not set. Fix this by keeping track of the total atomic files currently opened and using that to exit from this condition.
---
v2:
- change fix as per Chao's suggestion
- decrement sbi->atomic_files protected under sbi->inode_lock[ATOMIC_FILE] and only when atomic flag is cleared for the first time, otherwise, the count goes to an invalid/high value as f2fs_drop_inmem_pages() can be called from two contexts at the same time.
fs/f2fs/f2fs.h | 1 + fs/f2fs/file.c | 1 + fs/f2fs/segment.c | 21 +++++++++++++++------ 3 files changed, 17 insertions(+), 6 deletions(-)
fs/f2fs/f2fs.h +1 -0
@@ -1297,6 +1297,7 @@ struct f2fs_sb_info {unsigned int gc_mode;/* current GC state */ unsigned int next_victim_seg[2 ];/* next segment in victim section */ /* for skip statistic */ +unsigned int atomic_files;/* # of opened atomic file */ unsigned long long skipped_atomic_files[2 ];/* FG_GC and BG_GC */ unsigned long long skipped_gc_rwsem;/* FG_GC only */
fs/f2fs/file.c +1 -0
@@ -1919,6 +1919,7 @@ static int f2fs_ioc_start_atomic_write(struct file *filp)spin_lock (&sbi->inode_lock[ATOMIC_FILE ]);if (list_empty (&fi->inmem_ilist))list_add_tail (&fi->inmem_ilist, &sbi->inode_list[ATOMIC_FILE ]); + sbi->atomic_files++;spin_unlock (&sbi->inode_lock[ATOMIC_FILE ]);/* add inode in inmem_list first and set atomic_file */
fs/f2fs/segment.c +15 -6
@@ -288,6 +288,8 @@ void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure)struct list_head *head = &sbi->inode_list[ATOMIC_FILE ];struct inode *inode;struct f2fs_inode_info *fi; +unsigned int count = sbi->atomic_files; +unsigned int looped =0 ; next:spin_lock (&sbi->inode_lock[ATOMIC_FILE ]);if (list_empty (head)) {
@@ -296,22 +298,26 @@ void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure) } fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist); inode =igrab (&fi->vfs_inode); +if (inode) +list_move_tail (&fi->inmem_ilist, head);spin_unlock (&sbi->inode_lock[ATOMIC_FILE ]);if (inode) {if (gc_failure) { -if (fi->i_gc_failures[GC_FAILURE_ATOMIC ]) - goto drop; - goto skip; +if (!fi->i_gc_failures[GC_FAILURE_ATOMIC ]) + goto skip; } -drop:set_inode_flag (inode,FI_ATOMIC_REVOKE_REQUEST );f2fs_drop_inmem_pages (inode); +skip:iput (inode); } -skip:congestion_wait (BLK_RW_ASYNC ,HZ /50 );cond_resched (); +if (gc_failure) { +if (++looped >= count) +return ; + } goto next; }
@@ -327,13 +333,16 @@ void f2fs_drop_inmem_pages(struct inode *inode)mutex_unlock (&fi->inmem_lock); } -clear_inode_flag (inode,FI_ATOMIC_FILE ); fi->i_gc_failures[GC_FAILURE_ATOMIC ] =0 ;stat_dec_atomic_write (inode);spin_lock (&sbi->inode_lock[ATOMIC_FILE ]);if (!list_empty (&fi->inmem_ilist))list_del_init (&fi->inmem_ilist); +if (f2fs_is_atomic_file (inode)) { +clear_inode_flag (inode,FI_ATOMIC_FILE ); + sbi->atomic_files--; + }spin_unlock (&sbi->inode_lock[ATOMIC_FILE ]); }
Signature · 2 lines
Qualcomm India Private Limited, on behalf of Qualcomm Innovation Center, Inc. Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, a Linux Foundation Collaborative Project.
Jaegeuk Kim
maintainerQuoted 90 lines · Sahitya Tummala
The FS got stuck in the below stack when the storage is almost full/dirty condition (when
FG_GCis being done).
schedule_timeoutio_schedule_timeoutcongestion_waitf2fs_drop_inmem_pages_allf2fs_gcf2fs_balance_fs__write_node_pagef2fs_fsync_node_pagesf2fs_do_sync_filef2fs_ioctlThe root cause for this issue is there is a potential infinite loop in
f2fs_drop_inmem_pages_all()for the case wheregc_failureis true and when there an inode whose i_gc_failures[GC_FAILURE_ATOMIC] is not set. Fix this by keeping track of the total atomic files currently opened and using that to exit from this condition.Fix-suggested-by Chao Yu <[email protected]>Signed-off-by Chao Yu <[email protected]>Signed-off-by Sahitya Tummala <[email protected]>---
v2:
- change fix as per Chao's suggestion
- decrementsbi->atomic_filesprotected under sbi->inode_lock[ATOMIC_FILE] and only when atomic flag is cleared for the first time, otherwise, the count goes to an invalid/high value asf2fs_drop_inmem_pages()can be called from two contexts at the same time.fs/f2fs/f2fs.h | 1 + fs/f2fs/file.c | 1 + fs/f2fs/segment.c | 21 +++++++++++++++------ 3 files changed, 17 insertions(+), 6 deletions(-)fs/f2fs/f2fs.h +1
-0@@ -1297,6 +1297,7 @@ struct f2fs_sb_info {unsigned int gc_mode;/* current GC state */ unsigned int next_victim_seg[2 ];/* next segment in victim section */ /* for skip statistic */ +unsigned int atomic_files;/* # of opened atomic file */ unsigned long long skipped_atomic_files[2 ];/* FG_GC and BG_GC */ unsigned long long skipped_gc_rwsem;/* FG_GC only */ fs/f2fs/file.c +1
-0@@ -1919,6 +1919,7 @@ static int f2fs_ioc_start_atomic_write(struct file *filp)spin_lock (&sbi->inode_lock[ATOMIC_FILE ]);if (list_empty (&fi->inmem_ilist))list_add_tail (&fi->inmem_ilist, &sbi->inode_list[ATOMIC_FILE ]); + sbi->atomic_files++;spin_unlock (&sbi->inode_lock[ATOMIC_FILE ]);/* add inode in inmem_list first and set atomic_file */ fs/f2fs/segment.c +6
-4@@ -288,6 +288,8 @@ void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure)struct list_head *head = &sbi->inode_list[ATOMIC_FILE ];struct inode *inode;struct f2fs_inode_info *fi; +unsigned int count = sbi->atomic_files; +unsigned int looped =0 ; next:spin_lock (&sbi->inode_lock[ATOMIC_FILE ]);if (list_empty (head)) {@@ -296,22 +298,26 @@ void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure) } fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist); inode =igrab (&fi->vfs_inode); +if (inode) +list_move_tail (&fi->inmem_ilist, head);spin_unlock (&sbi->inode_lock[ATOMIC_FILE ]);if (inode) {if (gc_failure) { -if (fi->i_gc_failures[GC_FAILURE_ATOMIC ]) - goto drop; - goto skip; +if (!fi->i_gc_failures[GC_FAILURE_ATOMIC ]) + goto skip; } -drop:
fs/f2fs/segment.c void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure) · line 318
Quoted 10 lines
fs/f2fs/segment.c +3
-1set_inode_flag (inode,FI_ATOMIC_REVOKE_REQUEST );f2fs_drop_inmem_pages (inode); +skip:iput (inode); } -skip:congestion_wait (BLK_RW_ASYNC ,HZ /50 );cond_resched (); +if (gc_failure) { +if (++looped >= count)
There is a race condition when handling sbi->atomic_files?
Quoted 26 lines
fs/f2fs/segment.c +6
-1+return ; + } goto next; }@@ -327,13 +333,16 @@ void f2fs_drop_inmem_pages(struct inode *inode)mutex_unlock (&fi->inmem_lock); } -clear_inode_flag (inode,FI_ATOMIC_FILE ); fi->i_gc_failures[GC_FAILURE_ATOMIC ] =0 ;stat_dec_atomic_write (inode);spin_lock (&sbi->inode_lock[ATOMIC_FILE ]);if (!list_empty (&fi->inmem_ilist))list_del_init (&fi->inmem_ilist); +if (f2fs_is_atomic_file (inode)) { +clear_inode_flag (inode,FI_ATOMIC_FILE ); + sbi->atomic_files--; + }spin_unlock (&sbi->inode_lock[ATOMIC_FILE ]); }Signature · 2 lines
Qualcomm India Private Limited, on behalf of Qualcomm Innovation Center, Inc. Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, a Linux Foundation Collaborative Project.
Hi Jaegeuk,
Quoted 95 lines · Jaegeuk Kim
Quoted 94 lines · Sahitya Tummala
The FS got stuck in the below stack when the storage is almost full/dirty condition (when
FG_GCis being done).
schedule_timeoutio_schedule_timeoutcongestion_waitf2fs_drop_inmem_pages_allf2fs_gcf2fs_balance_fs__write_node_pagef2fs_fsync_node_pagesf2fs_do_sync_filef2fs_ioctlThe root cause for this issue is there is a potential infinite loop in
f2fs_drop_inmem_pages_all()for the case wheregc_failureis true and when there an inode whose i_gc_failures[GC_FAILURE_ATOMIC] is not set. Fix this by keeping track of the total atomic files currently opened and using that to exit from this condition.Fix-suggested-by Chao Yu <[email protected]>Signed-off-by Chao Yu <[email protected]>Signed-off-by Sahitya Tummala <[email protected]>---
v2:
- change fix as per Chao's suggestion
- decrementsbi->atomic_filesprotected under sbi->inode_lock[ATOMIC_FILE] and only when atomic flag is cleared for the first time, otherwise, the count goes to an invalid/high value asf2fs_drop_inmem_pages()can be called from two contexts at the same time.fs/f2fs/f2fs.h | 1 + fs/f2fs/file.c | 1 + fs/f2fs/segment.c | 21 +++++++++++++++------ 3 files changed, 17 insertions(+), 6 deletions(-)fs/f2fs/f2fs.h +1
-0@@ -1297,6 +1297,7 @@ struct f2fs_sb_info {unsigned int gc_mode;/* current GC state */ unsigned int next_victim_seg[2 ];/* next segment in victim section */ /* for skip statistic */ +unsigned int atomic_files;/* # of opened atomic file */ unsigned long long skipped_atomic_files[2 ];/* FG_GC and BG_GC */ unsigned long long skipped_gc_rwsem;/* FG_GC only */ fs/f2fs/file.c +1
-0@@ -1919,6 +1919,7 @@ static int f2fs_ioc_start_atomic_write(struct file *filp)spin_lock (&sbi->inode_lock[ATOMIC_FILE ]);if (list_empty (&fi->inmem_ilist))list_add_tail (&fi->inmem_ilist, &sbi->inode_list[ATOMIC_FILE ]); + sbi->atomic_files++;spin_unlock (&sbi->inode_lock[ATOMIC_FILE ]);/* add inode in inmem_list first and set atomic_file */ fs/f2fs/segment.c +7
-4@@ -288,6 +288,8 @@ void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure)struct list_head *head = &sbi->inode_list[ATOMIC_FILE ];struct inode *inode;struct f2fs_inode_info *fi; +unsigned int count = sbi->atomic_files; +unsigned int looped =0 ; next:spin_lock (&sbi->inode_lock[ATOMIC_FILE ]);if (list_empty (head)) {@@ -296,22 +298,26 @@ void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure) } fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist); inode =igrab (&fi->vfs_inode); +if (inode) +list_move_tail (&fi->inmem_ilist, head);spin_unlock (&sbi->inode_lock[ATOMIC_FILE ]);if (inode) {if (gc_failure) { -if (fi->i_gc_failures[GC_FAILURE_ATOMIC ]) - goto drop; - goto skip; +if (!fi->i_gc_failures[GC_FAILURE_ATOMIC ]) + goto skip; } -drop:set_inode_flag (inode,FI_ATOMIC_REVOKE_REQUEST );f2fs_drop_inmem_pages (inode); +skip:iput (inode);
Quoted 9 lines
fs/f2fs/segment.c void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure) · line 318
Quoted 6 lines
fs/f2fs/segment.c +2
-1} -skip:congestion_wait (BLK_RW_ASYNC ,HZ /50 );cond_resched (); +if (gc_failure) { +if (++looped >= count)There is a race condition when handling
sbi->atomic_files?
There is no concern here in this function w.r.t sbi->atomic_files value. Since when we loop over all the atomic files, the looped counter will increment and will exit when all the files are looped at least once.
There is an issue with f2fs_drop_inmem_pages() which is actually decrementing the sbi->atomic_files and that was handled below.
Thanks,
Sahitya.
Quoted 26 lines
Quoted 26 lines
fs/f2fs/segment.c +6
-1+return ; + } goto next; }@@ -327,13 +333,16 @@ void f2fs_drop_inmem_pages(struct inode *inode)mutex_unlock (&fi->inmem_lock); } -clear_inode_flag (inode,FI_ATOMIC_FILE ); fi->i_gc_failures[GC_FAILURE_ATOMIC ] =0 ;stat_dec_atomic_write (inode);spin_lock (&sbi->inode_lock[ATOMIC_FILE ]);if (!list_empty (&fi->inmem_ilist))list_del_init (&fi->inmem_ilist); +if (f2fs_is_atomic_file (inode)) { +clear_inode_flag (inode,FI_ATOMIC_FILE ); + sbi->atomic_files--; + }spin_unlock (&sbi->inode_lock[ATOMIC_FILE ]); }Signature · 2 lines
Qualcomm India Private Limited, on behalf of Qualcomm Innovation Center, Inc. Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, a Linux Foundation Collaborative Project.
Signature · 3 lines
-- Sent by a consultant of the Qualcomm Innovation Center, Inc. The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum.
Hi Jaegeuk,
Quoted 95 lines · Jaegeuk Kim
Quoted 94 lines · Sahitya Tummala
The FS got stuck in the below stack when the storage is almost full/dirty condition (when
FG_GCis being done).
schedule_timeoutio_schedule_timeoutcongestion_waitf2fs_drop_inmem_pages_allf2fs_gcf2fs_balance_fs__write_node_pagef2fs_fsync_node_pagesf2fs_do_sync_filef2fs_ioctlThe root cause for this issue is there is a potential infinite loop in
f2fs_drop_inmem_pages_all()for the case wheregc_failureis true and when there an inode whose i_gc_failures[GC_FAILURE_ATOMIC] is not set. Fix this by keeping track of the total atomic files currently opened and using that to exit from this condition.Fix-suggested-by Chao Yu <[email protected]>Signed-off-by Chao Yu <[email protected]>Signed-off-by Sahitya Tummala <[email protected]>---
v2:
- change fix as per Chao's suggestion
- decrementsbi->atomic_filesprotected under sbi->inode_lock[ATOMIC_FILE] and only when atomic flag is cleared for the first time, otherwise, the count goes to an invalid/high value asf2fs_drop_inmem_pages()can be called from two contexts at the same time.fs/f2fs/f2fs.h | 1 + fs/f2fs/file.c | 1 + fs/f2fs/segment.c | 21 +++++++++++++++------ 3 files changed, 17 insertions(+), 6 deletions(-)fs/f2fs/f2fs.h +1
-0@@ -1297,6 +1297,7 @@ struct f2fs_sb_info {unsigned int gc_mode;/* current GC state */ unsigned int next_victim_seg[2 ];/* next segment in victim section */ /* for skip statistic */ +unsigned int atomic_files;/* # of opened atomic file */ unsigned long long skipped_atomic_files[2 ];/* FG_GC and BG_GC */ unsigned long long skipped_gc_rwsem;/* FG_GC only */ fs/f2fs/file.c +1
-0@@ -1919,6 +1919,7 @@ static int f2fs_ioc_start_atomic_write(struct file *filp)spin_lock (&sbi->inode_lock[ATOMIC_FILE ]);if (list_empty (&fi->inmem_ilist))list_add_tail (&fi->inmem_ilist, &sbi->inode_list[ATOMIC_FILE ]); + sbi->atomic_files++;spin_unlock (&sbi->inode_lock[ATOMIC_FILE ]);/* add inode in inmem_list first and set atomic_file */ fs/f2fs/segment.c +7
-4@@ -288,6 +288,8 @@ void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure)struct list_head *head = &sbi->inode_list[ATOMIC_FILE ];struct inode *inode;struct f2fs_inode_info *fi; +unsigned int count = sbi->atomic_files; +unsigned int looped =0 ; next:spin_lock (&sbi->inode_lock[ATOMIC_FILE ]);if (list_empty (head)) {@@ -296,22 +298,26 @@ void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure) } fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist); inode =igrab (&fi->vfs_inode); +if (inode) +list_move_tail (&fi->inmem_ilist, head);spin_unlock (&sbi->inode_lock[ATOMIC_FILE ]);if (inode) {if (gc_failure) { -if (fi->i_gc_failures[GC_FAILURE_ATOMIC ]) - goto drop; - goto skip; +if (!fi->i_gc_failures[GC_FAILURE_ATOMIC ]) + goto skip; } -drop:set_inode_flag (inode,FI_ATOMIC_REVOKE_REQUEST );f2fs_drop_inmem_pages (inode); +skip:iput (inode);
Quoted 9 lines
fs/f2fs/segment.c void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure) · line 318
Quoted 6 lines
fs/f2fs/segment.c +2
-1} -skip:congestion_wait (BLK_RW_ASYNC ,HZ /50 );cond_resched (); +if (gc_failure) { +if (++looped >= count)There is a race condition when handling
sbi->atomic_files?
There is no concern here in this function w.r.t sbi->atomic_files value. Since when we loop over all the atomic files, the looped counter will increment and will exit when all the files are looped at least once.
There is an issue with f2fs_drop_inmem_pages() which is actually decrementing the sbi->atomic_files and that was handled below.
Thanks,
Sahitya.
Quoted 26 lines
Quoted 26 lines
fs/f2fs/segment.c +6
-1+return ; + } goto next; }@@ -327,13 +333,16 @@ void f2fs_drop_inmem_pages(struct inode *inode)mutex_unlock (&fi->inmem_lock); } -clear_inode_flag (inode,FI_ATOMIC_FILE ); fi->i_gc_failures[GC_FAILURE_ATOMIC ] =0 ;stat_dec_atomic_write (inode);spin_lock (&sbi->inode_lock[ATOMIC_FILE ]);if (!list_empty (&fi->inmem_ilist))list_del_init (&fi->inmem_ilist); +if (f2fs_is_atomic_file (inode)) { +clear_inode_flag (inode,FI_ATOMIC_FILE ); + sbi->atomic_files--; + }spin_unlock (&sbi->inode_lock[ATOMIC_FILE ]); }Signature · 2 lines
Qualcomm India Private Limited, on behalf of Qualcomm Innovation Center, Inc. Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum, a Linux Foundation Collaborative Project.
Signature · 9 lines
-- Sent by a consultant of the Qualcomm Innovation Center, Inc. The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum. _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel