f2fs: do checkpoint in kill_sb

PATCH In review Jaegeuk Kim 3 participants 6 messages Jul 7, 2018 – Jul 12, 2018 F2FS file system

lore

Share & tools

lore

Jaegeuk Kim

author maintainer

2018-07-07 01:20 UTC #0 lore raw

PATCH 1/1 f2fs: do checkpoint in kill_sb

When unmounting f2fs in force mode, we can get it stuck by io_schedule() by some pending IOs in meta_inode.

io_schedule+0xd/0x30
wait_on_page_bit_common+0xc6/0x130
__filemap_fdatawait_range+0xbd/0x100
filemap_fdatawait_keep_errors+0x15/0x40
sync_inodes_sb+0x1cf/0x240
sync_filesystem+0x52/0x90
generic_shutdown_super+0x1d/0x110
ceph_kill_sb+0x28/0x80 [ceph]
deactivate_locked_super+0x35/0x60
cleanup_mnt+0x36/0x70
task_work_run+0x79/0xa0
exit_to_usermode_loop+0x62/0x70
do_syscall_64+0xdb/0xf0
entry_SYSCALL_64_after_hwframe+0x44/0xa9

0xffffffffffffffff

Signed-off-by Jaegeuk Kim <[email protected]>

---
 fs/f2fs/super.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

fs/f2fs/super.c +13 -3

index 3995e926ba3a..1dc6809fac38 100644
@@ -3089,9 +3089,19 @@ static struct dentry *f2fs_mount(struct file_system_type *fs_type, int flags,
 static void kill_f2fs_super(struct super_block *sb)
 {
 	if (sb->s_root) {
-		set_sbi_flag(F2FS_SB(sb), SBI_IS_CLOSE);
-		f2fs_stop_gc_thread(F2FS_SB(sb));
-		f2fs_stop_discard_thread(F2FS_SB(sb));
+		struct f2fs_sb_info *sbi = F2FS_SB(sb);
+
+		set_sbi_flag(sbi, SBI_IS_CLOSE);
+		f2fs_stop_gc_thread(sbi);
+		f2fs_stop_discard_thread(sbi);
+
+		if (is_sbi_flag_set(sbi, SBI_IS_DIRTY) ||
+				!is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) {
+			struct cp_control cpc = {
+				.reason = CP_UMOUNT,
+			};
+			f2fs_write_checkpoint(sbi, &cpc);
+		}
 	}
 	kill_block_super(sb);
 }

Signature · 1 line

2.17.0.441.gb46fe60e1d-goog

Chao Yu

maintainer

2018-07-09 15:04 UTC #1 lore raw

Quoted 1 line · Jaegeuk Kim

When unmounting f2fs in force mode, we can get it stuck by io_schedule()

force mode means force shutdown?

Quoted 4 lines

by some pending IOs in meta_inode.
io_schedule+0xd/0x30
wait_on_page_bit_common+0xc6/0x130

Looks like a deadlock here? Does this mean we forget to submit cached bio in somewhere?

Thanks,

Quoted 47 lines

__filemap_fdatawait_range+0xbd/0x100
filemap_fdatawait_keep_errors+0x15/0x40
sync_inodes_sb+0x1cf/0x240
sync_filesystem+0x52/0x90
generic_shutdown_super+0x1d/0x110
ceph_kill_sb+0x28/0x80 [ceph]
deactivate_locked_super+0x35/0x60
cleanup_mnt+0x36/0x70
task_work_run+0x79/0xa0
exit_to_usermode_loop+0x62/0x70
do_syscall_64+0xdb/0xf0
entry_SYSCALL_64_after_hwframe+0x44/0xa9

0xffffffffffffffff

Signed-off-by Jaegeuk Kim <[email protected]>

---
 fs/f2fs/super.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

fs/f2fs/super.c +13 -3

index 3995e926ba3a..1dc6809fac38 100644
@@ -3089,9 +3089,19 @@ static struct dentry *f2fs_mount(struct file_system_type *fs_type, int flags,
 static void kill_f2fs_super(struct super_block *sb)
 {
 	if (sb->s_root) {
-		set_sbi_flag(F2FS_SB(sb), SBI_IS_CLOSE);
-		f2fs_stop_gc_thread(F2FS_SB(sb));
-		f2fs_stop_discard_thread(F2FS_SB(sb));
+		struct f2fs_sb_info *sbi = F2FS_SB(sb);
+
+		set_sbi_flag(sbi, SBI_IS_CLOSE);
+		f2fs_stop_gc_thread(sbi);
+		f2fs_stop_discard_thread(sbi);
+
+		if (is_sbi_flag_set(sbi, SBI_IS_DIRTY) ||
+				!is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) {
+			struct cp_control cpc = {
+				.reason = CP_UMOUNT,
+			};
+			f2fs_write_checkpoint(sbi, &cpc);
+		}
 	}
 	kill_block_super(sb);
 }

Jaegeuk Kim

author maintainer

2018-07-09 20:36 UTC #2 lore raw

Quoted 4 lines · Chao Yu

Quoted 1 line · Jaegeuk Kim
When unmounting f2fs in force mode, we can get it stuck by io_schedule()
force mode means force shutdown?

Yes.

Quoted 8 lines

Quoted 5 lines
by some pending IOs in meta_inode.
io_schedule+0xd/0x30
wait_on_page_bit_common+0xc6/0x130
Looks like a deadlock here? Does this mean we forget to submit cached bio in somewhere?

I'm hitting this randomly, when running all the testcases in xfstests. I can't find the missing flow, and only get this during kill_sb.

Quoted 50 lines

Thanks,

Quoted 47 lines

__filemap_fdatawait_range+0xbd/0x100
filemap_fdatawait_keep_errors+0x15/0x40
sync_inodes_sb+0x1cf/0x240
sync_filesystem+0x52/0x90
generic_shutdown_super+0x1d/0x110
ceph_kill_sb+0x28/0x80 [ceph]
deactivate_locked_super+0x35/0x60
cleanup_mnt+0x36/0x70
task_work_run+0x79/0xa0
exit_to_usermode_loop+0x62/0x70
do_syscall_64+0xdb/0xf0
entry_SYSCALL_64_after_hwframe+0x44/0xa9

0xffffffffffffffff

Signed-off-by Jaegeuk Kim <[email protected]>

---
 fs/f2fs/super.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

fs/f2fs/super.c +13 -3

index 3995e926ba3a..1dc6809fac38 100644
@@ -3089,9 +3089,19 @@ static struct dentry *f2fs_mount(struct file_system_type *fs_type, int flags,
 static void kill_f2fs_super(struct super_block *sb)
 {
 	if (sb->s_root) {
-		set_sbi_flag(F2FS_SB(sb), SBI_IS_CLOSE);
-		f2fs_stop_gc_thread(F2FS_SB(sb));
-		f2fs_stop_discard_thread(F2FS_SB(sb));
+		struct f2fs_sb_info *sbi = F2FS_SB(sb);
+
+		set_sbi_flag(sbi, SBI_IS_CLOSE);
+		f2fs_stop_gc_thread(sbi);
+		f2fs_stop_discard_thread(sbi);
+
+		if (is_sbi_flag_set(sbi, SBI_IS_DIRTY) ||
+				!is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) {
+			struct cp_control cpc = {
+				.reason = CP_UMOUNT,
+			};
+			f2fs_write_checkpoint(sbi, &cpc);
+		}
 	}
 	kill_block_super(sb);
 }

Chao Yu

2018-07-10 06:10 UTC #3 lore raw

Quoted 11 lines · Jaegeuk Kim

Quoted 4 lines · Chao Yu
Quoted 1 line · Jaegeuk Kim
When unmounting f2fs in force mode, we can get it stuck by io_schedule()
force mode means force shutdown?
Yes.
Quoted 3 lines
Quoted 3 lines
by some pending IOs in meta_inode.

Quoted 8 lines

Quoted 5 lines
Quoted 2 lines
io_schedule+0xd/0x30
wait_on_page_bit_common+0xc6/0x130
Looks like a deadlock here? Does this mean we forget to submit cached bio in somewhere?
I'm hitting this randomly, when running all the testcases in xfstests. I can't find the missing flow, and only get this during kill_sb.

I guess we need this before returning from ->writepage{,s}:

if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN))
f2fs_submit_merged_write()

How about trying this instead?

Thanks,

Quoted 45 lines

Thanks,

Quoted 41 lines

__filemap_fdatawait_range+0xbd/0x100
filemap_fdatawait_keep_errors+0x15/0x40
sync_inodes_sb+0x1cf/0x240
sync_filesystem+0x52/0x90
generic_shutdown_super+0x1d/0x110
ceph_kill_sb+0x28/0x80 [ceph]
deactivate_locked_super+0x35/0x60
cleanup_mnt+0x36/0x70
task_work_run+0x79/0xa0
exit_to_usermode_loop+0x62/0x70
do_syscall_64+0xdb/0xf0
entry_SYSCALL_64_after_hwframe+0x44/0xa9

0xffffffffffffffff

Signed-off-by Jaegeuk Kim <[email protected]>

---
 fs/f2fs/super.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

fs/f2fs/super.c +11 -3

index 3995e926ba3a..1dc6809fac38 100644
@@ -3089,9 +3089,19 @@ static struct dentry *f2fs_mount(struct file_system_type *fs_type, int flags,
 static void kill_f2fs_super(struct super_block *sb)
 {
 	if (sb->s_root) {
-		set_sbi_flag(F2FS_SB(sb), SBI_IS_CLOSE);
-		f2fs_stop_gc_thread(F2FS_SB(sb));
-		f2fs_stop_discard_thread(F2FS_SB(sb));
+		struct f2fs_sb_info *sbi = F2FS_SB(sb);
+
+		set_sbi_flag(sbi, SBI_IS_CLOSE);
+		f2fs_stop_gc_thread(sbi);
+		f2fs_stop_discard_thread(sbi);
+
+		if (is_sbi_flag_set(sbi, SBI_IS_DIRTY) ||
+				!is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) {
+			struct cp_control cpc = {
+				.reason = CP_UMOUNT,
+			};

Quoted 9 lines

Quoted 6 lines
fs/f2fs/super.c static struct dentry *f2fs_mount(struct file_system_type *fs_type, int flags, · line 3107
Quoted 6 lines
fs/f2fs/super.c +2 -0
+			f2fs_write_checkpoint(sbi, &cpc);
+		}
 	}
 	kill_block_super(sb);
 }
.

------------------------------------------------------------------------------
Check out the vibrant tech community on one of the world's most engaging tech sites, Slashdot.org! http://sdm.link/slashdot

Jaegeuk Kim

author maintainer

2018-07-12 07:34 UTC #4 lore raw

Quoted 21 lines · Chao Yu

Quoted 11 lines · Jaegeuk Kim
Quoted 4 lines · Chao Yu
Quoted 1 line · Jaegeuk Kim
When unmounting f2fs in force mode, we can get it stuck by io_schedule()
force mode means force shutdown?
Yes.
Quoted 3 lines
Quoted 3 lines
by some pending IOs in meta_inode.
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
io_schedule+0xd/0x30
wait_on_page_bit_common+0xc6/0x130
Looks like a deadlock here? Does this mean we forget to submit cached bio in somewhere?
I'm hitting this randomly, when running all the testcases in xfstests. I can't find the missing flow, and only get this during kill_sb.

Quoted 6 lines

I guess we need this before returning from ->writepage{,s}:
if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN))
f2fs_submit_merged_write()
How about trying this instead?

No, it's not shutdown case unfortunately.

Quoted 57 lines

Thanks,

Quoted 54 lines

Quoted 43 lines

Thanks,

Quoted 39 lines

__filemap_fdatawait_range+0xbd/0x100
filemap_fdatawait_keep_errors+0x15/0x40
sync_inodes_sb+0x1cf/0x240
sync_filesystem+0x52/0x90
generic_shutdown_super+0x1d/0x110
ceph_kill_sb+0x28/0x80 [ceph]
deactivate_locked_super+0x35/0x60
cleanup_mnt+0x36/0x70
task_work_run+0x79/0xa0
exit_to_usermode_loop+0x62/0x70
do_syscall_64+0xdb/0xf0
entry_SYSCALL_64_after_hwframe+0x44/0xa9

0xffffffffffffffff

Signed-off-by Jaegeuk Kim <[email protected]>

---
 fs/f2fs/super.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

fs/f2fs/super.c +9 -3

index 3995e926ba3a..1dc6809fac38 100644
@@ -3089,9 +3089,19 @@ static struct dentry *f2fs_mount(struct file_system_type *fs_type, int flags,
 static void kill_f2fs_super(struct super_block *sb)
 {
 	if (sb->s_root) {
-		set_sbi_flag(F2FS_SB(sb), SBI_IS_CLOSE);
-		f2fs_stop_gc_thread(F2FS_SB(sb));
-		f2fs_stop_discard_thread(F2FS_SB(sb));
+		struct f2fs_sb_info *sbi = F2FS_SB(sb);
+
+		set_sbi_flag(sbi, SBI_IS_CLOSE);
+		f2fs_stop_gc_thread(sbi);
+		f2fs_stop_discard_thread(sbi);
+
+		if (is_sbi_flag_set(sbi, SBI_IS_DIRTY) ||
+				!is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) {
+			struct cp_control cpc = {

Quoted 8 lines

fs/f2fs/super.c static struct dentry *f2fs_mount(struct file_system_type *fs_type, int flags, · line 3107
Quoted 8 lines
fs/f2fs/super.c +4 -0
+				.reason = CP_UMOUNT,
+			};
+			f2fs_write_checkpoint(sbi, &cpc);
+		}
 	}
 	kill_block_super(sb);
 }

Chao Yu

2018-07-12 08:23 UTC #5 lore raw

Quoted 22 lines · Jaegeuk Kim

Quoted 21 lines · Chao Yu
Quoted 19 lines · Jaegeuk Kim
Quoted 4 lines · Chao Yu
Quoted 1 line · Jaegeuk Kim
When unmounting f2fs in force mode, we can get it stuck by io_schedule()
force mode means force shutdown?
Yes.
Quoted 8 lines
Quoted 5 lines
by some pending IOs in meta_inode.
io_schedule+0xd/0x30
wait_on_page_bit_common+0xc6/0x130
Looks like a deadlock here? Does this mean we forget to submit cached bio in somewhere?
I'm hitting this randomly, when running all the testcases in xfstests. I can't find the missing flow, and only get this during kill_sb.

Quoted 8 lines

Quoted 6 lines
I guess we need this before returning from ->writepage{,s}:
if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN))
f2fs_submit_merged_write()
How about trying this instead?
No, it's not shutdown case unfortunately.

Still I didn't understand why this patch can fix the issue... :(

Thanks,

Quoted 61 lines

Quoted 58 lines

Thanks,

Quoted 54 lines

Quoted 43 lines

Thanks,

Quoted 39 lines

__filemap_fdatawait_range+0xbd/0x100
filemap_fdatawait_keep_errors+0x15/0x40
sync_inodes_sb+0x1cf/0x240
sync_filesystem+0x52/0x90
generic_shutdown_super+0x1d/0x110
ceph_kill_sb+0x28/0x80 [ceph]
deactivate_locked_super+0x35/0x60
cleanup_mnt+0x36/0x70
task_work_run+0x79/0xa0
exit_to_usermode_loop+0x62/0x70
do_syscall_64+0xdb/0xf0
entry_SYSCALL_64_after_hwframe+0x44/0xa9

0xffffffffffffffff

Signed-off-by Jaegeuk Kim <[email protected]>

---
 fs/f2fs/super.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

fs/f2fs/super.c +9 -3

index 3995e926ba3a..1dc6809fac38 100644
@@ -3089,9 +3089,19 @@ static struct dentry *f2fs_mount(struct file_system_type *fs_type, int flags,
 static void kill_f2fs_super(struct super_block *sb)
 {
 	if (sb->s_root) {
-		set_sbi_flag(F2FS_SB(sb), SBI_IS_CLOSE);
-		f2fs_stop_gc_thread(F2FS_SB(sb));
-		f2fs_stop_discard_thread(F2FS_SB(sb));
+		struct f2fs_sb_info *sbi = F2FS_SB(sb);
+
+		set_sbi_flag(sbi, SBI_IS_CLOSE);
+		f2fs_stop_gc_thread(sbi);
+		f2fs_stop_discard_thread(sbi);
+
+		if (is_sbi_flag_set(sbi, SBI_IS_DIRTY) ||
+				!is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) {
+			struct cp_control cpc = {

Quoted 8 lines

fs/f2fs/super.c static struct dentry *f2fs_mount(struct file_system_type *fs_type, int flags, · line 3107
Quoted 8 lines
fs/f2fs/super.c +4 -0
+				.reason = CP_UMOUNT,
+			};
+			f2fs_write_checkpoint(sbi, &cpc);
+		}
 	}
 	kill_block_super(sb);
 }