ocfs2: exit recovery thread on mount error path
Joseph Qi
maintainerPATCH v3 1/3 ocfs2: exit recovery thread on mount error path
When a mount fails after the cluster connection has been established, e.g. in ocfs2_mount_volume(), ocfs2_fill_super() unwinds via out_debugfs/out_super and frees the osb without disabling recovery.
A node failure event can concurrently launch the recovery thread, which blocks in __ocfs2_wait_on_mount() waiting for the volume state to become VOLUME_MOUNTED or VOLUME_DISABLED. As the mount error path neither sets VOLUME_DISABLED nor wakes osb_mount_event, the thread can never make progress: the kthread leaks and stays blocked on the wait queue embedded in the freed osb, which may then be accessed as freed memory.
Fix it by setting VOLUME_DISABLED and waking osb_mount_event on this path so the thread bails out, and replace the plain kfree(osb->recovery_map) with ocfs2_recovery_exit(), which waits for a running recovery thread to exit before the recovery map is freed.
--- fs/ocfs2/super.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-)
fs/ocfs2/super.c +10 -1
@@ -1169,8 +1169,17 @@ static int ocfs2_fill_super(struct super_block *sb, struct fs_context *fc) out_debugfs:debugfs_remove_recursive (osb->osb_debug_root); out_super: +/* +* A recovery thread launched by a node failure event may still be +* waiting for the volume to be mounted. Set VOLUME_DISABLED and +* wake it up, then wait for it to exit before osb is freed, +* otherwise the kthread would leak and stay blocked on the wait +* queue embedded in the freed osb. +*/ +atomic_set (&osb->vol_state,VOLUME_DISABLED ); +wake_up (&osb->osb_mount_event);ocfs2_release_system_inodes (osb); -kfree (osb->recovery_map); +ocfs2_recovery_exit (osb);ocfs2_delete_osb (osb);kfree (osb); out:
Signature · 1 line
2.39.3
Joseph Qi
maintainerPATCH v3 2/3 ocfs2: free replay slots in ocfs2_recovery_exit()
Commit ce2fcf1516d6 ("ocfs2: fix memory leak in ocfs2_mount_volume()") added ocfs2_free_replay_slots() calls to the mount error paths out_dismount and out_check_volume to fix a leak of osb->replay_map. However these calls are unlocked while the bail path of the recovery thread, which is woken up by out_dismount right before the call, also frees the replay slots under osb->recovery_lock. Both sides can thus observe a non-NULL osb->replay_map and trigger a double free.
Fix this by moving ocfs2_free_replay_slots() into ocfs2_recovery_exit() after ocfs2_recovery_disable(), which waits for a running recovery thread to exit under osb->recovery_lock, and drop the unlocked call sites. Both ocfs2_dismount_volume() and the out_super path of ocfs2_fill_super() call ocfs2_recovery_exit(), so the replay slots are freed on every path. Since super.c no longer references it, make ocfs2_free_replay_slots() static again.
--- fs/ocfs2/journal.c | 3 ++- fs/ocfs2/journal.h | 1 - fs/ocfs2/super.c | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-)
fs/ocfs2/journal.c +2 -1
@@ -156,7 +156,7 @@ static void ocfs2_queue_replay_slots(struct ocfs2_super *osb, replay_map->rm_state =REPLAY_DONE ; } -void ocfs2_free_replay_slots (struct ocfs2_super *osb) +static void ocfs2_free_replay_slots (struct ocfs2_super *osb) {struct ocfs2_replay_map *replay_map = osb->replay_map;
@@ -243,6 +243,7 @@ void ocfs2_recovery_exit(struct ocfs2_super *osb)/* XXX: Should we bug if there are dirty entries? */ kfree (rm); +ocfs2_free_replay_slots (osb); } static int __ocfs2_recovery_map_test(struct ocfs2_super *osb,
fs/ocfs2/journal.h +0 -1
@@ -151,7 +151,6 @@ void ocfs2_recovery_exit(struct ocfs2_super *osb);void ocfs2_recovery_disable_quota (struct ocfs2_super *osb);int ocfs2_compute_replay_slots (struct ocfs2_super *osb); -void ocfs2_free_replay_slots (struct ocfs2_super *osb);/* * Journal Control: * Initialize, Load, Shutdown, Wipe a journal.
fs/ocfs2/super.c +1 -4
@@ -1162,7 +1162,6 @@ static int ocfs2_fill_super(struct super_block *sb, struct fs_context *fc) out_dismount:atomic_set (&osb->vol_state,VOLUME_DISABLED );wake_up (&osb->osb_mount_event); -ocfs2_free_replay_slots (osb);ocfs2_dismount_volume (sb,1 ); goto out;
@@ -1776,14 +1775,12 @@ static int ocfs2_mount_volume(struct super_block *sb) status =ocfs2_truncate_log_init (osb);if (status <0 ) {mlog_errno (status); - goto out_check_volume; + goto out_system_inodes; }ocfs2_super_unlock (osb,1 );return 0 ; -out_check_volume: -ocfs2_free_replay_slots (osb); out_system_inodes:if (osb->local_alloc_state ==OCFS2_LA_ENABLED )ocfs2_shutdown_local_alloc (osb);
Signature · 1 line
2.39.3
Re: [PATCH v3 2/3] ocfs2: free replay slots in ocfs2_recovery_exit()
Quoted 16 lines · Joseph Qi
Commit
ce2fcf1516d6("ocfs2: fix memory leak inocfs2_mount_volume()") addedocfs2_free_replay_slots()calls to the mount error pathsout_dismountandout_check_volumeto fix a leak ofosb->replay_map. However these calls are unlocked while the bail path of the recovery thread, which is woken up byout_dismountright before the call, also frees the replay slots underosb->recovery_lock. Both sides can thus observe a non-NULLosb->replay_mapand trigger a double free.Fix this by moving
ocfs2_free_replay_slots()intoocfs2_recovery_exit()afterocfs2_recovery_disable(), which waits for a running recovery thread to exit underosb->recovery_lock, and drop the unlocked call sites. Bothocfs2_dismount_volume()and theout_superpath ofocfs2_fill_super()callocfs2_recovery_exit(), so the replay slots are freed on every path. Sincesuper.cno longer references it, makeocfs2_free_replay_slots()static again.
Quoted 3 lines
Fixes ce2fcf1516d6 ("ocfs2: fix memory leak in ocfs2_mount_volume()")Cc <[email protected]>Signed-off-by Joseph Qi <[email protected]>
LGTM
Quoted 70 lines
--- fs/ocfs2/journal.c | 3 ++- fs/ocfs2/journal.h | 1 - fs/ocfs2/super.c | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-)fs/ocfs2/journal.c +2
-1@@ -156,7 +156,7 @@ static void ocfs2_queue_replay_slots(struct ocfs2_super *osb, replay_map->rm_state =REPLAY_DONE ; } -void ocfs2_free_replay_slots (struct ocfs2_super *osb) +static void ocfs2_free_replay_slots (struct ocfs2_super *osb) {struct ocfs2_replay_map *replay_map = osb->replay_map;@@ -243,6 +243,7 @@ void ocfs2_recovery_exit(struct ocfs2_super *osb)/* XXX: Should we bug if there are dirty entries? */ kfree (rm); +ocfs2_free_replay_slots (osb); } static int __ocfs2_recovery_map_test(struct ocfs2_super *osb,fs/ocfs2/journal.h +0
-1@@ -151,7 +151,6 @@ void ocfs2_recovery_exit(struct ocfs2_super *osb);void ocfs2_recovery_disable_quota (struct ocfs2_super *osb);int ocfs2_compute_replay_slots (struct ocfs2_super *osb); -void ocfs2_free_replay_slots (struct ocfs2_super *osb);/* * Journal Control: * Initialize, Load, Shutdown, Wipe a journal. fs/ocfs2/super.c +1
-4@@ -1162,7 +1162,6 @@ static int ocfs2_fill_super(struct super_block *sb, struct fs_context *fc) out_dismount:atomic_set (&osb->vol_state,VOLUME_DISABLED );wake_up (&osb->osb_mount_event); -ocfs2_free_replay_slots (osb);ocfs2_dismount_volume (sb,1 ); goto out;@@ -1776,14 +1775,12 @@ static int ocfs2_mount_volume(struct super_block *sb) status =ocfs2_truncate_log_init (osb);if (status <0 ) {mlog_errno (status); - goto out_check_volume; + goto out_system_inodes; }ocfs2_super_unlock (osb,1 );return 0 ; -out_check_volume: -ocfs2_free_replay_slots (osb); out_system_inodes:if (osb->local_alloc_state ==OCFS2_LA_ENABLED )ocfs2_shutdown_local_alloc (osb);Signature · 1 line
2.39.3
Joseph Qi
maintainerPATCH v3 3/3 ocfs2: defer suballocator block group reclaim to workqueue
In reply to Joseph QiWhen the last bit in a suballocator block group is freed, _ocfs2_free_suballoc_bits() reclaims the group back to the global bitmap. The reclaim takes inode_lock() on the global bitmap inode while running inside the freeing transaction, adding a lock dependency of
j_trans_barrier -> global bitmap inode i_rwsem
This forms a circular dependency with paths such as ocfs2_shutdown_local_alloc(), which take the global bitmap inode lock before starting a transaction:
Task1 (dealloc):
ocfs2_run_deallocs
ocfs2_free_cached_blocks
ocfs2_start_trans
down_read(j_trans_barrier)
_ocfs2_free_suballoc_bits
_ocfs2_reclaim_suballoc_to_main
inode_lock(main_bm_inode) <- wait on Task2
Task2 (dismount):
ocfs2_shutdown_local_alloc
inode_lock(main_bm_inode) ocfs2_start_trans
down_read(j_trans_barrier) <- wait on Task3
Task3 (ocfs2cmt):
ocfs2_commit_cache
down_write(j_trans_barrier) <- wait on Task1's handle jbd2_journal_flush
Task1 waits for Task2's inode_lock(), Task2 waits for the j_trans_barrier down_write() held by ocfs2cmt, and ocfs2cmt waits for Task1's running transaction to commit - a real deadlock, observed with aio-stress direct IO writes racing dismount.
Fix it by deferring the reclaim to the per-superblock ocfs2_wq workqueue, so the freeing transaction no longer takes the global bitmap inode lock. The worker re-checks under the suballocator locks that the block group is still fully freed (it may have been allocated from again in the meantime), takes the global bitmap inode locks before starting its own transaction, and performs the same suballocator cleanup and space return. The inode lock order (suballocator inode -> global bitmap inode) is consistent with the existing "inode lock before transaction" order, breaking the cycle.
Reclaim work can still be queued late in dismount, e.g. when the truncate log is flushed or orphan dir recovery frees inode bits, so both ocfs2_dismount_volume() and the mount error path flush ocfs2_wq right before the system inodes are released, while the journal is still alive, to make sure no reclaim work is left running. The worker also bails out if the journal is already gone.
Tested with the ocfs2 testsuite (including aio-stress direct IO) and umount/mount cycles on a CONFIG_PROVE_LOCKING kernel: the circular locking dependency is gone and freed block groups are still returned to the global bitmap.
--- fs/ocfs2/ocfs2.h | 5 ++ fs/ocfs2/suballoc.c | 202 +++++++++++++++++++++++++++++++++++++------- fs/ocfs2/suballoc.h | 2 +- fs/ocfs2/super.c | 16 ++++ 4 files changed, 195 insertions(+), 30 deletions(-)
fs/ocfs2/ocfs2.h +5 -0
@@ -502,6 +502,11 @@ struct ocfs2_super*/ struct workqueue_struct *ocfs2_wq; +/* deferred reclaim of fully freed suballocator block groups */ +spinlock_t os_suballoc_reclaim_lock; +struct list_head os_suballoc_reclaim_list; +struct work_struct os_suballoc_reclaim_work; +/* sysfs directory per partition */ struct kset *osb_dev_kset;
fs/ocfs2/suballoc.c +173 -29
275 lines not on this page · raw message
fs/ocfs2/suballoc.h +1 -1
@@ -206,7 +206,7 @@ int ocfs2_lock_allocators(struct inode *inode, struct ocfs2_extent_tree *et,int ocfs2_test_inode_bit (struct ocfs2_super *osb,u64 blkno,int *res); - +void ocfs2_suballoc_reclaim_worker (struct work_struct *work);/* * The following two interfaces are for ocfs2_create_inode_in_orphan().
fs/ocfs2/super.c +16 -0
@@ -1784,6 +1784,9 @@ static int ocfs2_mount_volume(struct super_block *sb) out_system_inodes:if (osb->local_alloc_state ==OCFS2_LA_ENABLED )ocfs2_shutdown_local_alloc (osb); +/* Drain pending suballoc reclaim work before the journal goes away */ +if (osb->ocfs2_wq) +flush_workqueue (osb->ocfs2_wq);ocfs2_release_system_inodes (osb);/* before journal shutdown, we should release slot_info */ ocfs2_free_slot_info (osb);
@@ -1854,6 +1857,14 @@ static void ocfs2_dismount_volume(struct super_block *sb, int mnt_err)if (osb->cconn)ocfs2_super_unlock (osb,1 ); +/* +* Drain pending suballoc reclaim work while the system inodes and +* the journal are still alive, since the worker needs to look up +* the global bitmap inode and start a transaction. +*/ +if (osb->ocfs2_wq) +flush_workqueue (osb->ocfs2_wq); +ocfs2_release_system_inodes (osb);ocfs2_journal_shutdown (osb);
@@ -2140,6 +2151,11 @@ static int ocfs2_initialize_super(struct super_block *sb,INIT_WORK (&osb->dquot_drop_work, ocfs2_drop_dquot_refs);init_llist_head (&osb->dquot_drop_list); +spin_lock_init (&osb->os_suballoc_reclaim_lock); +INIT_LIST_HEAD (&osb->os_suballoc_reclaim_list); +INIT_WORK (&osb->os_suballoc_reclaim_work, + ocfs2_suballoc_reclaim_worker); +/* get some pseudo constants for clustersize bits */ osb->s_clustersize_bits =le32_to_cpu (di->id2.i_super.s_clustersize_bits);
Signature · 1 line
2.39.3