Skip to contentSkip to conversation

xfs: shutdown is a racy mess

PATCH Superseded Dave Chinner 4 participants 34 messages Jun 30, 2021 – Jul 14, 2021 XFS filesystem

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#11 lore raw

PATCH 3/9 xfs: move recovery needed state updates to xfs_log_mount_finish

In reply to Dave Chinner

From: Dave Chinner <[email protected]>

xfs_log_mount_finish() needs to know if recovery is needed or not to make descisions on whether to flush the log and AIL. Move the handling of the NEED_RECOVERY state out to this function rather than needing a temporary variable to store this state over the call to xlog_recover_finish().

---
 fs/xfs/xfs_log.c         | 24 ++++++++-----
 fs/xfs/xfs_log_recover.c | 73 +++++++++++++++-------------------------
 2 files changed, 43 insertions(+), 54 deletions(-)
fs/xfs/xfs_log.c +16 -8
index 3ea67a90bcde..6e6f490a8ab5 100644
@@ -698,9 +698,9 @@ int
 xfs_log_mount_finish(
 	struct xfs_mount	*mp)
 {
-	int	error = 0;
-	bool	readonly = (mp->m_flags & XFS_MOUNT_RDONLY);
-	bool	recovered = mp->m_log->l_flags & XLOG_RECOVERY_NEEDED;
+	struct xlog		*log = mp->m_log;
+	bool			readonly = (mp->m_flags & XFS_MOUNT_RDONLY);
+	int			error = 0;
 
 	if (mp->m_flags & XFS_MOUNT_NORECOVERY) {
 		ASSERT(mp->m_flags & XFS_MOUNT_RDONLY);
@@ -731,7 +731,8 @@ xfs_log_mount_finish(
 	 * mount failure occurs.
 	 */
 	mp->m_super->s_flags |= SB_ACTIVE;
-	error = xlog_recover_finish(mp->m_log);
+	if (log->l_flags & XLOG_RECOVERY_NEEDED)
+		error = xlog_recover_finish(log);
 	if (!error)
 		xfs_log_work_queue(mp);
 	mp->m_super->s_flags &= ~SB_ACTIVE;
@@ -746,17 +747,24 @@ xfs_log_mount_finish(
 	 * Don't push in the error case because the AIL may have pending intents
 	 * that aren't removed until recovery is cancelled.
 	 */
-	if (!error && recovered) {
-		xfs_log_force(mp, XFS_LOG_SYNC);
-		xfs_ail_push_all_sync(mp->m_ail);
+	if (log->l_flags & XLOG_RECOVERY_NEEDED) {
+		if (!error) {
+			xfs_log_force(mp, XFS_LOG_SYNC);
+			xfs_ail_push_all_sync(mp->m_ail);
+		}
+		xfs_notice(mp, "Ending recovery (logdev: %s)",
+				mp->m_logname ? mp->m_logname : "internal");
+	} else {
+		xfs_info(mp, "Ending clean mount");
 	}
 	xfs_buftarg_drain(mp->m_ddev_targp);
 
+	log->l_flags &= ~XLOG_RECOVERY_NEEDED;
 	if (readonly)
 		mp->m_flags |= XFS_MOUNT_RDONLY;
 
 	/* Make sure the log is dead if we're returning failure. */
-	ASSERT(!error || (mp->m_log->l_flags & XLOG_IO_ERROR));
+	ASSERT(!error || (log->l_flags & XLOG_IO_ERROR));
 
 	return error;
 }
fs/xfs/xfs_log_recover.c +27 -46

89 lines not on this page · raw message

Signature · 1 line
2.31.1
#1 lore raw

PATCH 1/9 xfs: convert XLOG_FORCED_SHUTDOWN() to xlog_is_shutdown()

In reply to Dave Chinner

From: Dave Chinner <[email protected]>

Make it less shouty and a static inline before adding more calls through the log code.

---
 fs/xfs/xfs_log.c         | 32 ++++++++++++++++----------------
 fs/xfs/xfs_log_cil.c     | 10 +++++-----
 fs/xfs/xfs_log_priv.h    |  7 +++++--
 fs/xfs/xfs_log_recover.c |  9 +++------
 fs/xfs/xfs_trans.c       |  2 +-
 5 files changed, 30 insertions(+), 30 deletions(-)
fs/xfs/xfs_log.c +16 -16
index 71fd8c0cad92..5ae11e7bf2fd 100644
@@ -247,7 +247,7 @@ xlog_grant_head_wait(
 	list_add_tail(&tic->t_queue, &head->waiters);
 
 	do {
-		if (XLOG_FORCED_SHUTDOWN(log))
+		if (xlog_is_shutdown(log))
 			goto shutdown;
 		xlog_grant_push_ail(log, need_bytes);
 
@@ -261,7 +261,7 @@ xlog_grant_head_wait(
 		trace_xfs_log_grant_wake(log, tic);
 
 		spin_lock(&head->lock);
-		if (XLOG_FORCED_SHUTDOWN(log))
+		if (xlog_is_shutdown(log))
 			goto shutdown;
 	} while (xlog_space_left(log, &head->grant) < need_bytes);
 
@@ -366,7 +366,7 @@ xfs_log_writable(
 		return false;
 	if (xfs_readonly_buftarg(mp->m_log->l_targ))
 		return false;
-	if (XFS_FORCED_SHUTDOWN(mp))
+	if (xlog_is_shutdown(mp->m_log))
 		return false;
 	return true;
 }
@@ -383,7 +383,7 @@ xfs_log_regrant(
 	int			need_bytes;
 	int			error = 0;
 
-	if (XLOG_FORCED_SHUTDOWN(log))
+	if (xlog_is_shutdown(log))
 		return -EIO;
 
 	XFS_STATS_INC(mp, xs_try_logspace);
@@ -451,7 +451,7 @@ xfs_log_reserve(
 
 	ASSERT(client == XFS_TRANSACTION || client == XFS_LOG);
 
-	if (XLOG_FORCED_SHUTDOWN(log))
+	if (xlog_is_shutdown(log))
 		return -EIO;
 
 	XFS_STATS_INC(mp, xs_try_logspace);
@@ -787,7 +787,7 @@ xlog_wait_on_iclog(
 	struct xlog		*log = iclog->ic_log;
 
 	trace_xlog_iclog_wait_on(iclog, _RET_IP_);
-	if (!XLOG_FORCED_SHUTDOWN(log) &&
+	if (!xlog_is_shutdown(log) &&
 	    iclog->ic_state != XLOG_STATE_ACTIVE &&
 	    iclog->ic_state != XLOG_STATE_DIRTY) {
 		XFS_STATS_INC(log->l_mp, xs_log_force_sleep);
@@ -796,7 +796,7 @@ xlog_wait_on_iclog(
 		spin_unlock(&log->l_icloglock);
 	}
 
-	if (XLOG_FORCED_SHUTDOWN(log))
+	if (xlog_is_shutdown(log))
 		return -EIO;
 	return 0;
 }
@@ -915,7 +915,7 @@ xfs_log_unmount_write(
 
 	xfs_log_force(mp, XFS_LOG_SYNC);
 
-	if (XLOG_FORCED_SHUTDOWN(log))
+	if (xlog_is_shutdown(log))
 		return;
 
 	/*
@@ -1024,7 +1024,7 @@ xfs_log_space_wake(
 	struct xlog		*log = mp->m_log;
 	int			free_bytes;
 
-	if (XLOG_FORCED_SHUTDOWN(log))
+	if (xlog_is_shutdown(log))
 		return;
 
 	if (!list_empty_careful(&log->l_write_head.waiters)) {
@@ -1115,7 +1115,7 @@ xfs_log_cover(
 
 	ASSERT((xlog_cil_empty(mp->m_log) && xlog_iclogs_empty(mp->m_log) &&
 	        !xfs_ail_min_lsn(mp->m_log->l_ailp)) ||
-	       XFS_FORCED_SHUTDOWN(mp));
+		xlog_is_shutdown(mp->m_log));
 
 	if (!xfs_log_writable(mp))
 		return 0;
@@ -1546,7 +1546,7 @@ xlog_commit_record(
 	};
 	int	error;
 
-	if (XLOG_FORCED_SHUTDOWN(log))
+	if (xlog_is_shutdown(log))
 		return -EIO;
 
 	error = xlog_write(log, &vec, ticket, lsn, iclog, XLOG_COMMIT_TRANS);
@@ -1627,7 +1627,7 @@ xlog_grant_push_ail(
 	xfs_lsn_t	threshold_lsn;
 
 	threshold_lsn = xlog_grant_push_threshold(log, need_bytes);
-	if (threshold_lsn == NULLCOMMITLSN || XLOG_FORCED_SHUTDOWN(log))
+	if (threshold_lsn == NULLCOMMITLSN || xlog_is_shutdown(log))
 		return;
 
 	/*
@@ -2806,7 +2806,7 @@ xlog_state_do_callback(
 			cycled_icloglock = true;
 
 			spin_lock(&log->l_icloglock);
-			if (XLOG_FORCED_SHUTDOWN(log))
+			if (xlog_is_shutdown(log))
 				wake_up_all(&iclog->ic_force_wait);
 			else
 				xlog_state_clean_iclog(log, iclog);
@@ -2858,7 +2858,7 @@ xlog_state_done_syncing(
 	 * split log writes, on the second, we shut down the file system and
 	 * no iclogs should ever be attempted to be written to disk again.
 	 */
-	if (!XLOG_FORCED_SHUTDOWN(log)) {
+	if (!xlog_is_shutdown(log)) {
 		ASSERT(iclog->ic_state == XLOG_STATE_SYNCING);
 		iclog->ic_state = XLOG_STATE_DONE_SYNC;
 	}
@@ -2906,7 +2906,7 @@ xlog_state_get_iclog_space(
 
 restart:
 	spin_lock(&log->l_icloglock);
-	if (XLOG_FORCED_SHUTDOWN(log)) {
+	if (xlog_is_shutdown(log)) {
 		spin_unlock(&log->l_icloglock);
 		return -EIO;
 	}
@@ -3754,7 +3754,7 @@ xfs_log_force_umount(
 	 * No need to get locks for this.
 	 */
 	if (logerror && log->l_iclog->ic_state == XLOG_STATE_IOERROR) {
-		ASSERT(XLOG_FORCED_SHUTDOWN(log));
+		ASSERT(xlog_is_shutdown(log));
 		return 1;
 	}
 
fs/xfs/xfs_log_cil.c +5 -5
index d162e8b83e90..23eec4f76f19 100644
@@ -584,7 +584,7 @@ xlog_cil_committed(
 	struct xfs_cil_ctx	*ctx)
 {
 	struct xfs_mount	*mp = ctx->cil->xc_log->l_mp;
-	bool			abort = XLOG_FORCED_SHUTDOWN(ctx->cil->xc_log);
+	bool			abort = xlog_is_shutdown(ctx->cil->xc_log);
 
 	/*
 	 * If the I/O failed, we're aborting the commit and already shutdown.
@@ -853,7 +853,7 @@ xlog_cil_push_work(
 		 * shutdown, but then went back to sleep once already in the
 		 * shutdown state.
 		 */
-		if (XLOG_FORCED_SHUTDOWN(log)) {
+		if (xlog_is_shutdown(log)) {
 			spin_unlock(&cil->xc_push_lock);
 			goto out_abort_free_ticket;
 		}
@@ -962,7 +962,7 @@ xlog_cil_push_work(
 out_abort_free_ticket:
 	xfs_log_ticket_ungrant(log, tic);
 out_abort:
-	ASSERT(XLOG_FORCED_SHUTDOWN(log));
+	ASSERT(xlog_is_shutdown(log));
 	xlog_cil_committed(ctx);
 }
 
@@ -1115,7 +1115,7 @@ xlog_cil_commit(
 
 	xlog_cil_insert_items(log, tp);
 
-	if (regrant && !XLOG_FORCED_SHUTDOWN(log))
+	if (regrant && !xlog_is_shutdown(log))
 		xfs_log_ticket_regrant(log, tp->t_ticket);
 	else
 		xfs_log_ticket_ungrant(log, tp->t_ticket);
@@ -1188,7 +1188,7 @@ xlog_cil_force_seq(
 		 * shutdown, but then went back to sleep once already in the
 		 * shutdown state.
 		 */
-		if (XLOG_FORCED_SHUTDOWN(log))
+		if (xlog_is_shutdown(log))
 			goto out_shutdown;
 		if (ctx->sequence > sequence)
 			continue;
fs/xfs/xfs_log_priv.h +5 -2
index 4c41bbfa33b0..80d4e1325e1d 100644
@@ -454,8 +454,11 @@ struct xlog {
 #define XLOG_BUF_CANCEL_BUCKET(log, blkno) \
 	((log)->l_buf_cancel_table + ((uint64_t)blkno % XLOG_BC_TABLE_SIZE))
 
-#define XLOG_FORCED_SHUTDOWN(log) \
-	(unlikely((log)->l_flags & XLOG_IO_ERROR))
+static inline bool
+xlog_is_shutdown(struct xlog *log)
+{
+	return (log->l_flags & XLOG_IO_ERROR);
+}
 
 /* common routines */
 extern int
fs/xfs/xfs_log_recover.c +3 -6
index 1a291bf50776..f1b828dedb25 100644
@@ -144,7 +144,7 @@ xlog_do_io(
 
 	error = xfs_rw_bdev(log->l_targ->bt_bdev, log->l_logBBstart + blk_no,
 			BBTOB(nbblks), data, op);
-	if (error && !XFS_FORCED_SHUTDOWN(log->l_mp)) {
+	if (error && !xlog_is_shutdown(log)) {
 		xfs_alert(log->l_mp,
 			  "log recovery %s I/O error at daddr 0x%llx len %d error %d",
 			  op == REQ_OP_WRITE ? "write" : "read",
@@ -3278,10 +3278,7 @@ xlog_do_recover(
 	if (error)
 		return error;
 
-	/*
-	 * If IO errors happened during recovery, bail out.
-	 */
-	if (XFS_FORCED_SHUTDOWN(mp))
+	if (xlog_is_shutdown(log))
 		return -EIO;
 
 	/*
@@ -3303,7 +3300,7 @@ xlog_do_recover(
 	xfs_buf_hold(bp);
 	error = _xfs_buf_read(bp, XBF_READ);
 	if (error) {
-		if (!XFS_FORCED_SHUTDOWN(mp)) {
+		if (!xlog_is_shutdown(log)) {
 			xfs_buf_ioerror_alert(bp, __this_address);
 			ASSERT(0);
 		}
fs/xfs/xfs_trans.c +1 -1
index 87bffd12c20c..e26ade9fc630 100644
@@ -908,7 +908,7 @@ __xfs_trans_commit(
 	 */
 	xfs_trans_unreserve_and_mod_dquots(tp);
 	if (tp->t_ticket) {
-		if (regrant && !XLOG_FORCED_SHUTDOWN(mp->m_log))
+		if (regrant && !xlog_is_shutdown(mp->m_log))
 			xfs_log_ticket_regrant(mp->m_log, tp->t_ticket);
 		else
 			xfs_log_ticket_ungrant(mp->m_log, tp->t_ticket);
Signature · 1 line
2.31.1
#13 lore raw

PATCH 4/9 xfs: convert log flags to an operational state field

In reply to Dave Chinner

From: Dave Chinner <[email protected]>

log->l_flags doesn't actually contain "flags" as such, it contains operational state information that can change at runtime. For the shutdown state, this at least should be an atomic bit because it is read without holding locks in many places and so using atomic bitops for the state field modifications makes sense.

This allows us to use things like test_and_set_bit() on state changes (e.g. setting XLOG_TAIL_WARN) to avoid races in setting the state when we aren't holding locks.

---
 fs/xfs/xfs_log.c         | 60 ++++++++++++++++------------------------
 fs/xfs/xfs_log.h         |  1 -
 fs/xfs/xfs_log_priv.h    | 34 +++++++++++++++--------
 fs/xfs/xfs_log_recover.c |  6 ++--
 fs/xfs/xfs_super.c       |  2 +-
 5 files changed, 51 insertions(+), 52 deletions(-)
fs/xfs/xfs_log.c +24 -36
index 6e6f490a8ab5..049d6ac9cb4d 100644
@@ -299,7 +299,7 @@ xlog_grant_head_check(
 	int			free_bytes;
 	int			error = 0;
 
-	ASSERT(!(log->l_flags & XLOG_ACTIVE_RECOVERY));
+	ASSERT(!xlog_in_recovery(log));
 
 	/*
 	 * If there are other waiters on the queue then give them a chance at
@@ -552,6 +552,7 @@ xfs_log_mount(
 	xfs_daddr_t	blk_offset,
 	int		num_bblks)
 {
+	struct xlog	*log;
 	bool		fatal = xfs_sb_version_hascrc(&mp->m_sb);
 	int		error = 0;
 	int		min_logfsbs;
@@ -566,11 +567,12 @@ xfs_log_mount(
 		ASSERT(mp->m_flags & XFS_MOUNT_RDONLY);
 	}
 
-	mp->m_log = xlog_alloc_log(mp, log_target, blk_offset, num_bblks);
-	if (IS_ERR(mp->m_log)) {
-		error = PTR_ERR(mp->m_log);
+	log = xlog_alloc_log(mp, log_target, blk_offset, num_bblks);
+	if (IS_ERR(log)) {
+		error = PTR_ERR(log);
 		goto out;
 	}
+	mp->m_log = log;
 
 	/*
 	 * Validate the given log space and drop a critical message via syslog
@@ -635,7 +637,7 @@ xfs_log_mount(
 		xfs_warn(mp, "AIL initialisation failed: error %d", error);
 		goto out_free_log;
 	}
-	mp->m_log->l_ailp = mp->m_ail;
+	log->l_ailp = mp->m_ail;
 
 	/*
 	 * skip log recovery on a norecovery mount.  pretend it all
@@ -647,39 +649,39 @@ xfs_log_mount(
 		if (readonly)
 			mp->m_flags &= ~XFS_MOUNT_RDONLY;
 
-		error = xlog_recover(mp->m_log);
+		error = xlog_recover(log);
 
 		if (readonly)
 			mp->m_flags |= XFS_MOUNT_RDONLY;
 		if (error) {
 			xfs_warn(mp, "log mount/recovery failed: error %d",
 				error);
-			xlog_recover_cancel(mp->m_log);
+			xlog_recover_cancel(log);
 			goto out_destroy_ail;
 		}
 	}
 
-	error = xfs_sysfs_init(&mp->m_log->l_kobj, &xfs_log_ktype, &mp->m_kobj,
+	error = xfs_sysfs_init(&log->l_kobj, &xfs_log_ktype, &mp->m_kobj,
 			       "log");
 	if (error)
 		goto out_destroy_ail;
 
 	/* Normal transactions can now occur */
-	mp->m_log->l_flags &= ~XLOG_ACTIVE_RECOVERY;
+	clear_bit(XLOG_ACTIVE_RECOVERY, &log->l_opstate);
 
 	/*
 	 * Now the log has been fully initialised and we know were our
 	 * space grant counters are, we can initialise the permanent ticket
 	 * needed for delayed logging to work.
 	 */
-	xlog_cil_init_post_recovery(mp->m_log);
+	xlog_cil_init_post_recovery(log);
 
 	return 0;
 
 out_destroy_ail:
 	xfs_trans_ail_destroy(mp);
 out_free_log:
-	xlog_dealloc_log(mp->m_log);
+	xlog_dealloc_log(log);
 out:
 	return error;
 }
@@ -731,7 +733,7 @@ xfs_log_mount_finish(
 	 * mount failure occurs.
 	 */
 	mp->m_super->s_flags |= SB_ACTIVE;
-	if (log->l_flags & XLOG_RECOVERY_NEEDED)
+	if (xlog_recovery_needed(log))
 		error = xlog_recover_finish(log);
 	if (!error)
 		xfs_log_work_queue(mp);
@@ -747,7 +749,7 @@ xfs_log_mount_finish(
 	 * Don't push in the error case because the AIL may have pending intents
 	 * that aren't removed until recovery is cancelled.
 	 */
-	if (log->l_flags & XLOG_RECOVERY_NEEDED) {
+	if (xlog_recovery_needed(log)) {
 		if (!error) {
 			xfs_log_force(mp, XFS_LOG_SYNC);
 			xfs_ail_push_all_sync(mp->m_ail);
@@ -759,12 +761,12 @@ xfs_log_mount_finish(
 	}
 	xfs_buftarg_drain(mp->m_ddev_targp);
 
-	log->l_flags &= ~XLOG_RECOVERY_NEEDED;
+	clear_bit(XLOG_RECOVERY_NEEDED, &log->l_opstate);
 	if (readonly)
 		mp->m_flags |= XFS_MOUNT_RDONLY;
 
 	/* Make sure the log is dead if we're returning failure. */
-	ASSERT(!error || (log->l_flags & XLOG_IO_ERROR));
+	ASSERT(!error || xlog_is_shutdown(log));
 
 	return error;
 }
@@ -1036,7 +1038,7 @@ xfs_log_space_wake(
 		return;
 
 	if (!list_empty_careful(&log->l_write_head.waiters)) {
-		ASSERT(!(log->l_flags & XLOG_ACTIVE_RECOVERY));
+		ASSERT(!xlog_in_recovery(log));
 
 		spin_lock(&log->l_write_head.lock);
 		free_bytes = xlog_space_left(log, &log->l_write_head.grant);
@@ -1045,7 +1047,7 @@ xfs_log_space_wake(
 	}
 
 	if (!list_empty_careful(&log->l_reserve_head.waiters)) {
-		ASSERT(!(log->l_flags & XLOG_ACTIVE_RECOVERY));
+		ASSERT(!xlog_in_recovery(log));
 
 		spin_lock(&log->l_reserve_head.lock);
 		free_bytes = xlog_space_left(log, &log->l_reserve_head.grant);
@@ -1400,7 +1402,7 @@ xlog_alloc_log(
 	log->l_logBBstart  = blk_offset;
 	log->l_logBBsize   = num_bblks;
 	log->l_covered_state = XLOG_STATE_COVER_IDLE;
-	log->l_flags	   |= XLOG_ACTIVE_RECOVERY;
+	set_bit(XLOG_ACTIVE_RECOVERY, &log->l_opstate);
 	INIT_DELAYED_WORK(&log->l_work, xfs_log_worker);
 
 	log->l_prev_block  = -1;
@@ -3527,17 +3529,15 @@ xlog_verify_grant_tail(
 	xlog_crack_atomic_lsn(&log->l_tail_lsn, &tail_cycle, &tail_blocks);
 	if (tail_cycle != cycle) {
 		if (cycle - 1 != tail_cycle &&
-		    !(log->l_flags & XLOG_TAIL_WARN)) {
+		    !test_and_set_bit(XLOG_TAIL_WARN, &log->l_opstate)) {
 			xfs_alert_tag(log->l_mp, XFS_PTAG_LOGRES,
 				"%s: cycle - 1 != tail_cycle", __func__);
-			log->l_flags |= XLOG_TAIL_WARN;
 		}
 
 		if (space > BBTOB(tail_blocks) &&
-		    !(log->l_flags & XLOG_TAIL_WARN)) {
+		    !test_and_set_bit(XLOG_TAIL_WARN, &log->l_opstate)) {
 			xfs_alert_tag(log->l_mp, XFS_PTAG_LOGRES,
 				"%s: space > BBTOB(tail_blocks)", __func__);
-			log->l_flags |= XLOG_TAIL_WARN;
 		}
 	}
 }
@@ -3704,8 +3704,7 @@ xfs_log_force_umount(
 	 * If this happens during log recovery, don't worry about
 	 * locking; the log isn't open for business yet.
 	 */
-	if (!log ||
-	    log->l_flags & XLOG_ACTIVE_RECOVERY) {
+	if (!log || xlog_in_recovery(log)) {
 		mp->m_flags |= XFS_MOUNT_FS_SHUTDOWN;
 		if (mp->m_sb_bp)
 			mp->m_sb_bp->b_flags |= XBF_DONE;
@@ -3742,10 +3741,8 @@ xfs_log_force_umount(
 	 * Mark the log and the iclogs with IO error flags to prevent any
 	 * further log IO from being issued or completed.
 	 */
-	if (!(log->l_flags & XLOG_IO_ERROR)) {
-		log->l_flags |= XLOG_IO_ERROR;
+	if (!test_and_set_bit(XLOG_IO_ERROR, &log->l_opstate))
 		retval = 1;
-	}
 	spin_unlock(&log->l_icloglock);
 
 	/*
@@ -3832,12 +3829,3 @@ xfs_log_check_lsn(
 
 	return valid;
 }
-
-bool
-xfs_log_in_recovery(
-	struct xfs_mount	*mp)
-{
-	struct xlog		*log = mp->m_log;
-
-	return log->l_flags & XLOG_ACTIVE_RECOVERY;
-}
fs/xfs/xfs_log.h +0 -1
index 813b972e9788..4c5c8a7db1d9 100644
@@ -138,7 +138,6 @@ void	xfs_log_work_queue(struct xfs_mount *mp);
 int	xfs_log_quiesce(struct xfs_mount *mp);
 void	xfs_log_clean(struct xfs_mount *mp);
 bool	xfs_log_check_lsn(struct xfs_mount *, xfs_lsn_t);
-bool	xfs_log_in_recovery(struct xfs_mount *);
 
 xfs_lsn_t xlog_grant_push_threshold(struct xlog *log, int need_bytes);
 
fs/xfs/xfs_log_priv.h +23 -11
index bf05763ba8df..6c2f88e06ac3 100644
@@ -11,15 +11,6 @@ struct xlog;
 struct xlog_ticket;
 struct xfs_mount;
 
-/*
- * Flags for log structure
- */
-#define XLOG_ACTIVE_RECOVERY	0x2	/* in the middle of recovery */
-#define	XLOG_RECOVERY_NEEDED	0x4	/* log was recovered */
-#define XLOG_IO_ERROR		0x8	/* log hit an I/O error, and being
-					   shutdown */
-#define XLOG_TAIL_WARN		0x10	/* log tail verify warning issued */
-
 /*
  * get client id from packed copy.
  *
@@ -397,7 +388,7 @@ struct xlog {
 	struct xfs_buftarg	*l_targ;        /* buftarg of log */
 	struct workqueue_struct	*l_ioend_workqueue; /* for I/O completions */
 	struct delayed_work	l_work;		/* background flush work */
-	uint			l_flags;
+	long			l_opstate;	/* operational state */
 	uint			l_quotaoffs_flag; /* XFS_DQ_*, for QUOTAOFFs */
 	struct list_head	*l_buf_cancel_table;
 	int			l_iclog_hsize;  /* size of iclog header */
@@ -451,10 +442,31 @@ struct xlog {
 #define XLOG_BUF_CANCEL_BUCKET(log, blkno) \
 	((log)->l_buf_cancel_table + ((uint64_t)blkno % XLOG_BC_TABLE_SIZE))
 
+/*
+ * Bits for operational state
+ */
+#define XLOG_ACTIVE_RECOVERY	0	/* in the middle of recovery */
+#define XLOG_RECOVERY_NEEDED	1	/* log was recovered */
+#define XLOG_IO_ERROR		2	/* log hit an I/O error, and being
+				   shutdown */
+#define XLOG_TAIL_WARN		3	/* log tail verify warning issued */
+
+static inline bool
+xlog_recovery_needed(struct xlog *log)
+{
+	return test_bit(XLOG_RECOVERY_NEEDED, &log->l_opstate);
+}
+
+static inline bool
+xlog_in_recovery(struct xlog *log)
+{
+	return test_bit(XLOG_ACTIVE_RECOVERY, &log->l_opstate);
+}
+
 static inline bool
 xlog_is_shutdown(struct xlog *log)
 {
-	return (log->l_flags & XLOG_IO_ERROR);
+	return test_bit(XLOG_IO_ERROR, &log->l_opstate);
 }
 
 /* common routines */
fs/xfs/xfs_log_recover.c +3 -3
index aeaf4e7fc447..078b3f8bea84 100644
@@ -3324,7 +3324,7 @@ xlog_do_recover(
 	xlog_recover_check_summary(log);
 
 	/* Normal transactions can now occur */
-	log->l_flags &= ~XLOG_ACTIVE_RECOVERY;
+	clear_bit(XLOG_ACTIVE_RECOVERY, &log->l_opstate);
 	return 0;
 }
 
@@ -3408,7 +3408,7 @@ xlog_recover(
 						     : "internal");
 
 		error = xlog_do_recover(log, head_blk, tail_blk);
-		log->l_flags |= XLOG_RECOVERY_NEEDED;
+		set_bit(XLOG_RECOVERY_NEEDED, &log->l_opstate);
 	}
 	return error;
 }
@@ -3458,7 +3458,7 @@ void
 xlog_recover_cancel(
 	struct xlog	*log)
 {
-	if (log->l_flags & XLOG_RECOVERY_NEEDED)
+	if (xlog_recovery_needed(log))
 		xlog_recover_cancel_intents(log);
 }
 
fs/xfs/xfs_super.c +1 -1
index 2c9e26a44546..29bec1f6476e 100644
@@ -734,7 +734,7 @@ xfs_fs_drop_inode(
 	 * that.  See the comment for this inode flag.
 	 */
 	if (ip->i_flags & XFS_IRECOVERY) {
-		ASSERT(ip->i_mount->m_log->l_flags & XLOG_RECOVERY_NEEDED);
+		ASSERT(xlog_recovery_needed(ip->i_mount->m_log));
 		return 0;
 	}
 
Signature · 1 line
2.31.1
#16 lore raw

PATCH 5/9 xfs: make forced shutdown processing atomic

In reply to Dave Chinner

From: Dave Chinner <[email protected]>

The running of a forced shutdown is a bit of a mess. It does racy checks for XFS_MOUNT_SHUTDOWN in xfs_do_force_shutdown(), then does more racy checks in xfs_log_force_unmount() before finally setting XFS_MOUNT_SHUTDOWN and XLOG_IO_ERROR under the log->icloglock.

Move the checking and setting of XFS_MOUNT_SHUTDOWN into xfs_do_force_shutdown() so we only process a shutdown once and once only. Serialise this with the mp->m_sb_lock spinlock so that the state change is atomic and won't race. Move all the mount specific shutdown state changes from xfs_log_force_unmount() to xfs_do_force_shutdown() so they are done atomically with setting XFS_MOUNT_SHUTDOWN.

Then get rid of the racy xlog_is_shutdown() check from xlog_force_shutdown(), and gate the log shutdown on the test_and_set_bit(XLOG_IO_ERROR) test under the icloglock. This means that the log is shutdown once and once only, and code that needs to prevent races with shutdown can do so by holding the icloglock and checking the return value of xlog_is_shutdown().

This results in a predicable shutdown execution process - we set the shutdown flags once and process the shutdown once rather than the current "as many concurrent shutdowns as can race to the flag setting" situation we have now.

---
 fs/xfs/xfs_fsops.c |  64 ++++++++++++++---------------
 fs/xfs/xfs_log.c   | 100 ++++++++++++++++++++-------------------------
 fs/xfs/xfs_log.h   |   2 +-
 3 files changed, 77 insertions(+), 89 deletions(-)
fs/xfs/xfs_fsops.c +31 -33
index 6ed29b158312..caca391ce1a9 100644
@@ -511,6 +511,11 @@ xfs_fs_goingdown(
  * consistent. We don't do an unmount here; just shutdown the shop, make sure
  * that absolutely nothing persistent happens to this filesystem after this
  * point.
+ *
+ * The shutdown state change is atomic, resulting in the first and only the
+ * first shutdown call processing the shutdown. This means we only shutdown the
+ * log once as it requires, and we don't spam the logs when multiple concurrent
+ * shutdowns race to set the shutdown flags.
  */
 void
 xfs_do_force_shutdown(
@@ -519,48 +524,41 @@ xfs_do_force_shutdown(
 	char		*fname,
 	int		lnnum)
 {
-	bool		logerror = flags & SHUTDOWN_LOG_IO_ERROR;
-
-	/*
-	 * No need to duplicate efforts.
-	 */
-	if (XFS_FORCED_SHUTDOWN(mp) && !logerror)
-		return;
-
-	/*
-	 * This flags XFS_MOUNT_FS_SHUTDOWN, makes sure that we don't
-	 * queue up anybody new on the log reservations, and wakes up
-	 * everybody who's sleeping on log reservations to tell them
-	 * the bad news.
-	 */
-	if (xfs_log_force_umount(mp, logerror))
-		return;
+	int		tag;
+	const char	*why;
 
-	if (flags & SHUTDOWN_FORCE_UMOUNT) {
-		xfs_alert(mp,
-"User initiated shutdown (0x%x) received. Shutting down filesystem",
-				flags);
+	spin_lock(&mp->m_sb_lock);
+	if (XFS_FORCED_SHUTDOWN(mp)) {
+		spin_unlock(&mp->m_sb_lock);
 		return;
 	}
+	mp->m_flags |= XFS_MOUNT_FS_SHUTDOWN;
+	if (mp->m_sb_bp)
+		mp->m_sb_bp->b_flags |= XBF_DONE;
+	spin_unlock(&mp->m_sb_lock);
+
+	if (flags & SHUTDOWN_FORCE_UMOUNT)
+		xfs_alert(mp, "User initiated shutdown received.");
 
-	if (flags & SHUTDOWN_CORRUPT_INCORE) {
-		xfs_alert_tag(mp, XFS_PTAG_SHUTDOWN_CORRUPT,
-"Corruption of in-memory data (0x%x) detected at %pS (%s:%d).  Shutting down filesystem",
-				flags, __return_address, fname, lnnum);
-		if (XFS_ERRLEVEL_HIGH <= xfs_error_level)
-			xfs_stack_trace();
-	} else if (logerror) {
-		xfs_alert_tag(mp, XFS_PTAG_SHUTDOWN_LOGERROR,
-"Log I/O error (0x%x) detected at %pS (%s:%d). Shutting down filesystem",
-				flags, __return_address, fname, lnnum);
+	if (xlog_force_shutdown(mp->m_log, flags)) {
+		tag = XFS_PTAG_SHUTDOWN_LOGERROR;
+		why = "Log I/O Error";
+	} else if (flags & SHUTDOWN_CORRUPT_INCORE) {
+		tag = XFS_PTAG_SHUTDOWN_CORRUPT;
+		why = "Corruption of in-memory data";
 	} else {
-		xfs_alert_tag(mp, XFS_PTAG_SHUTDOWN_IOERROR,
-"I/O error (0x%x) detected at %pS (%s:%d). Shutting down filesystem",
-				flags, __return_address, fname, lnnum);
+		tag = XFS_PTAG_SHUTDOWN_IOERROR;
+		why = "Metadata I/O Error";
 	}
 
+	xfs_alert_tag(mp, tag,
+"%s (0x%x) detected at %pS (%s:%d).  Shutting down filesystem.",
+			why, flags, __return_address, fname, lnnum);
 	xfs_alert(mp,
 		"Please unmount the filesystem and rectify the problem(s)");
+	if (xfs_error_level >= XFS_ERRLEVEL_HIGH)
+		xfs_stack_trace();
+
 }
 
 /*
fs/xfs/xfs_log.c +45 -55

128 lines not on this page · raw message

fs/xfs/xfs_log.h +1 -1
index 4c5c8a7db1d9..3f680f0c9744 100644
@@ -125,7 +125,6 @@ int	  xfs_log_reserve(struct xfs_mount *mp,
 			  bool		   permanent);
 int	  xfs_log_regrant(struct xfs_mount *mp, struct xlog_ticket *tic);
 void      xfs_log_unmount(struct xfs_mount *mp);
-int	  xfs_log_force_umount(struct xfs_mount *mp, int logerror);
 bool	xfs_log_writable(struct xfs_mount *mp);
 
 struct xlog_ticket *xfs_log_ticket_get(struct xlog_ticket *ticket);
@@ -140,5 +139,6 @@ void	xfs_log_clean(struct xfs_mount *mp);
 bool	xfs_log_check_lsn(struct xfs_mount *, xfs_lsn_t);
 
 xfs_lsn_t xlog_grant_push_threshold(struct xlog *log, int need_bytes);
+bool	  xlog_force_shutdown(struct xlog *log, int shutdown_flags);
 
 #endif	/* __XFS_LOG_H__ */
Signature · 1 line
2.31.1
#22 lore raw

PATCH 6/9 xfs: reowrk up xlog_state_do_callback

In reply to Dave Chinner

From: Dave Chinner <[email protected]>

Clean it up a bit by factoring and rearranging some of the code.

---
 fs/xfs/xfs_log.c | 99 ++++++++++++++++++++++++++----------------------
 1 file changed, 53 insertions(+), 46 deletions(-)
fs/xfs/xfs_log.c +53 -46
index 6c7cfc052135..bb44dcfcae89 100644
@@ -2758,67 +2758,74 @@ xlog_state_iodone_process_iclog(
 	}
 }
 
+/*
+ * Loop over all the iclogs, running attached callbacks on them. Return true if
+ * we ran any callbacks, indicating that we dropped the icloglock.
+ */
+static bool
+xlog_state_do_iclog_callbacks(
+	struct xlog		*log)
+		__releases(&log->l_icloglock)
+		__acquires(&log->l_icloglock)
+{
+	struct xlog_in_core	*iclog = log->l_iclog;
+	struct xlog_in_core	*first_iclog = NULL;
+	bool			ran_callback = false;
+
+	for (; iclog != first_iclog; iclog = iclog->ic_next) {
+		LIST_HEAD(cb_list);
+
+		if (!first_iclog)
+			first_iclog = iclog;
+
+		if (!xlog_is_shutdown(log)) {
+			if (xlog_state_iodone_process_iclog(log, iclog))
+				break;
+			if (iclog->ic_state != XLOG_STATE_CALLBACK)
+				continue;
+		}
+		list_splice_init(&iclog->ic_callbacks, &cb_list);
+		spin_unlock(&log->l_icloglock);
+
+		trace_xlog_iclog_callbacks_start(iclog, _RET_IP_);
+		xlog_cil_process_committed(&cb_list);
+		trace_xlog_iclog_callbacks_done(iclog, _RET_IP_);
+		ran_callback = true;
+
+		spin_lock(&log->l_icloglock);
+		if (xlog_is_shutdown(log))
+			wake_up_all(&iclog->ic_force_wait);
+		else
+			xlog_state_clean_iclog(log, iclog);
+	};
+	return ran_callback;
+}
+
+
+/*
+ * Loop running iclog completion callbacks until there are no more iclogs in a
+ * state that can run callbacks.
+ */
 STATIC void
 xlog_state_do_callback(
 	struct xlog		*log)
 {
-	struct xlog_in_core	*iclog;
-	struct xlog_in_core	*first_iclog;
-	bool			cycled_icloglock;
 	int			flushcnt = 0;
 	int			repeats = 0;
 
 	spin_lock(&log->l_icloglock);
-	do {
-		repeats++;
+	while (xlog_state_do_iclog_callbacks(log)) {
+		if (xlog_is_shutdown(log))
+			break;
 
-		/*
-		 * Scan all iclogs starting with the one pointed to by the
-		 * log.  Reset this starting point each time the log is
-		 * unlocked (during callbacks).
-		 *
-		 * Keep looping through iclogs until one full pass is made
-		 * without running any callbacks.
-		 */
-		cycled_icloglock = false;
-		first_iclog = NULL;
-		for (iclog = log->l_iclog;
-		     iclog != first_iclog;
-		     iclog = iclog->ic_next) {
-			LIST_HEAD(cb_list);
-
-			if (!first_iclog)
-				first_iclog = iclog;
-
-			if (!xlog_is_shutdown(log)) {
-				if (xlog_state_iodone_process_iclog(log, iclog))
-					break;
-				if (iclog->ic_state != XLOG_STATE_CALLBACK)
-					continue;
-			}
-			list_splice_init(&iclog->ic_callbacks, &cb_list);
-			spin_unlock(&log->l_icloglock);
-
-			trace_xlog_iclog_callbacks_start(iclog, _RET_IP_);
-			xlog_cil_process_committed(&cb_list);
-			trace_xlog_iclog_callbacks_done(iclog, _RET_IP_);
-			cycled_icloglock = true;
-
-			spin_lock(&log->l_icloglock);
-			if (xlog_is_shutdown(log))
-				wake_up_all(&iclog->ic_force_wait);
-			else
-				xlog_state_clean_iclog(log, iclog);
-		};
-
-		if (repeats > 5000) {
+		if (repeats++ > 5000) {
 			flushcnt += repeats;
 			repeats = 0;
 			xfs_warn(log->l_mp,
 				"%s: possible infinite loop (%d iterations)",
 				__func__, flushcnt);
 		}
-	} while (!xlog_is_shutdown(log) && cycled_icloglock);
+	};
 
 	if (log->l_iclog->ic_state == XLOG_STATE_ACTIVE ||
 	    xlog_is_shutdown(log))
Signature · 1 line
2.31.1
#28 lore raw

PATCH 8/9 xfs: don't run shutdown callbacks on active iclogs

In reply to Dave Chinner

From: Dave Chinner <[email protected]>

When the log is shutdown, it currently walks all the iclogs and runs callbacks that are attached to the iclogs, regardless of whether the iclog is queued for IO completion or not. This creates a problem for contexts attaching callbacks to iclogs in that a racing shutdown can run the callbacks even before the attaching context has finished processing the iclog and releasing it for IO submission.

If the callback processing of the iclog frees the structure that is attached to the iclog, then this leads to an UAF scenario that can only be protected against by holding the icloglock from the point callbacks are attached through to the release of the iclog. While we currently do this, it is not practical or sustainable.

Hence we need to make shutdown processing the responsibility of the context that holds active references to the iclog. We know that the contexts attaching callbacks to the iclog must have active references to the iclog, and that means they must be in either ACTIVE or WANT_SYNC states. xlog_state_do_callback() will skip over iclogs in these states -except- when the log is shut down.

xlog_state_do_callback() checks the state of the iclogs while holding the icloglock, therefore the reference count/state change that occurs in xlog_state_release_iclog() after the callbacks are atomic w.r.t. shutdown processing.

We can't push the responsibility of callback cleanup onto the CIL context because we can have ACTIVE iclogs that have callbacks attached that have already been released. Hence we really need to internalise the cleanup of callbacks into xlog_state_release_iclog() processing.

Indeed, we already have that internalisation via:

xlog_state_release_iclog
drop last reference

    ->SYNCING

xlog_sync
xlog_write_iclog
if (log_is_shutdown)
xlog_state_done_syncing()
xlog_state_do_callback()
<process shutdown on iclog that is now in SYNCING state>

The problem is that xlog_state_release_iclog() aborts before doing anything if the log is already shut down. It assumes that the callbacks have already been cleaned up, and it doesn't need to do any cleanup.

Hence the fix is to remove the xlog_is_shutdown() check from xlog_state_release_iclog() so that reference counts are correctly released from the iclogs, and when the reference count is zero we always transition to SYNCING if the log is shut down. Hence we'll always enter the xlog_sync() path in a shutdown and eventually end up erroring out the iclog IO and running xlog_state_do_callback() to process the callbacks attached to the iclog.

This allows us to stop processing referenced ACTIVE/WANT_SYNC iclogs directly in the shutdown code, and in doing so gets rid of the UAF vector that currently exists. This then decouples the adding of callbacks to the iclogs from xlog_state_release_iclog() as we guarantee that xlog_state_release_iclog() will process the callbacks if the log has been shut down before xlog_state_release_iclog() has been called.

Signed-off-br: Dave Chinner <[email protected]>

---
 fs/xfs/xfs_log.c     | 42 +++++++++++++++++++++++++++++++++++-------
 fs/xfs/xfs_log_cil.c | 15 +++++++--------
 2 files changed, 42 insertions(+), 15 deletions(-)
fs/xfs/xfs_log.c +35 -7
index e9e16eeb99e3..caa07631b2e5 100644
@@ -41,6 +41,8 @@ xlog_dealloc_log(
 /* local state machine functions */
 STATIC void xlog_state_done_syncing(
 	struct xlog_in_core	*iclog);
+STATIC void xlog_state_do_callback(
+	struct xlog		*log);
 STATIC int
 xlog_state_get_iclog_space(
 	struct xlog		*log,
@@ -492,6 +494,11 @@ xfs_log_reserve(
  * space waiters so they can process the newly set shutdown state. We really
  * don't care what order we process callbacks here because the log is shut down
  * and so state cannot change on disk anymore.
+ *
+ * We avoid processing actively referenced iclogs so that we don't run callbacks
+ * while the iclog owner might still be preparing the iclog for IO submssion.
+ * These will be caught by xlog_state_iclog_release() and call this function
+ * again to process any callbacks that may have been added to that iclog.
  */
 static void
 xlog_state_shutdown_callbacks(
@@ -503,7 +510,12 @@ xlog_state_shutdown_callbacks(
 	spin_lock(&log->l_icloglock);
 	iclog = log->l_iclog;
 	do {
+		if (atomic_read(&iclog->ic_refcnt)) {
+			/* Reference holder will re-run iclog callbacks. */
+			continue;
+		}
 		list_splice_init(&iclog->ic_callbacks, &cb_list);
+		wake_up_all(&iclog->ic_write_wait);
 		wake_up_all(&iclog->ic_force_wait);
 	} while ((iclog = iclog->ic_next) != log->l_iclog);
 
@@ -514,7 +526,7 @@ xlog_state_shutdown_callbacks(
 }
 
 static bool
-__xlog_state_release_iclog(
+xlog_state_want_sync(
 	struct xlog		*log,
 	struct xlog_in_core	*iclog)
 {
@@ -537,27 +549,43 @@ __xlog_state_release_iclog(
 }
 
 /*
- * Flush iclog to disk if this is the last reference to the given iclog and the
- * it is in the WANT_SYNC state.
+ * Release the active reference to the iclog. If this is the last reference to
+ * the iclog being dropped, check if the caller wants to be synced to disk and
+ * initiate IO submission. If the log has been shut down, then we need to run
+ * callback processing on this iclog as shutdown callback processing skips
+ * actively referenced iclogs.
  */
 int
 xlog_state_release_iclog(
 	struct xlog		*log,
 	struct xlog_in_core	*iclog)
+		__releases(&log->l_icloglock)
+		__acquires(&log->l_icloglock)
 {
 	lockdep_assert_held(&log->l_icloglock);
 
 	trace_xlog_iclog_release(iclog, _RET_IP_);
-	if (xlog_is_shutdown(log))
-		return -EIO;
+	if (!atomic_dec_and_test(&iclog->ic_refcnt))
+		goto out_check_shutdown;
 
-	if (atomic_dec_and_test(&iclog->ic_refcnt) &&
-	    __xlog_state_release_iclog(log, iclog)) {
+	if (xlog_is_shutdown(log)) {
+		/*
+		 * No more references to this iclog, so process the pending
+		 * iclog callbacks that were waiting on the release of this
+		 * iclog.
+		 */
+		spin_unlock(&log->l_icloglock);
+		xlog_state_shutdown_callbacks(log);
+		spin_lock(&log->l_icloglock);
+	} else if (xlog_state_want_sync(log, iclog)) {
 		spin_unlock(&log->l_icloglock);
 		xlog_sync(log, iclog);
 		spin_lock(&log->l_icloglock);
 	}
 
+out_check_shutdown:
+	if (xlog_is_shutdown(log))
+		return -EIO;
 	return 0;
 }
 
fs/xfs/xfs_log_cil.c +7 -8
index 1616d0442cd9..6a36c720b365 100644
@@ -882,11 +882,10 @@ xlog_cil_push_work(
 	xfs_log_ticket_ungrant(log, tic);
 
 	/*
-	 * Once we attach the ctx to the iclog, a shutdown can process the
-	 * iclog, run the callbacks and free the ctx. The only thing preventing
-	 * this potential UAF situation here is that we are holding the
-	 * icloglock. Hence we cannot access the ctx once we have attached the
-	 * callbacks and dropped the icloglock.
+	 * Once we attach the ctx to the iclog, it is effectively owned by the
+	 * iclog and we can only use it while we still have an active reference
+	 * to the iclog. i.e. once we call xlog_state_release_iclog() we can no
+	 * longer safely reference the ctx.
 	 */
 	spin_lock(&log->l_icloglock);
 	if (xlog_is_shutdown(log)) {
@@ -918,9 +917,6 @@ xlog_cil_push_work(
 	 * wakeup until this commit_iclog is written to disk.  Hence we use the
 	 * iclog header lsn and compare it to the commit lsn to determine if we
 	 * need to wait on iclogs or not.
-	 *
-	 * NOTE: It is not safe to reference the ctx after this check as we drop
-	 * the icloglock if we have to wait for completion of other iclogs.
 	 */
 	if (ctx->start_lsn != commit_lsn) {
 		xfs_lsn_t	plsn;
@@ -950,6 +946,9 @@ xlog_cil_push_work(
 	 */
 	commit_iclog->ic_flags |= XLOG_ICL_NEED_FUA;
 	xlog_state_release_iclog(log, commit_iclog);
+
+	/* Not safe to reference ctx now! */
+
 	spin_unlock(&log->l_icloglock);
 	return;
 
Signature · 1 line
2.31.1
#0 lore raw

cover xfs: shutdown is a racy mess

Hi folks,

With the recent log problems we've uncovreed, it's clear that the way we shut down filesystems and the log is a chaotic mess. We can have multiple filesystem shutdown executions being in progress at once, all competing to run shutdown processing and emit log messages saying the filesystem has been shut down and why. Further, shutdown changes the log state and runs log IO completion callbacks without any co-ordination with ongoing log operations.

This results in shutdowns running unpredictably, running multiple times, racing with the iclog state machine transitions and exposing us to use-after-free situations and unexpected state changes within the log itself.

This patch series tries to address the chaotic nature of shutdowns by making shutdown execution consistent and predictable. THis is achieved by:

- making the mount shutdown state transistion atomic and not dependent on log state.
- making operational log state transitions atomic
- making the log shutdown check be based entirely on the operational XLOG_IO_ERROR log state rather than a combination of log flags and iclog XLOG_STATE_IOERROR checks.
- Getting rid of XLOG_STATE_IOERROR means shutdown doesn't perturb iclog state in the middle of operations that are expecting iclogs to be in specific state(s).
- shutdown doesn't process iclogs that are actively referenced. This avoids use-after-free situations where shutdown runs callbacks and frees objects that own the reference to the iclog and are still in use by the iclog reference owner.
- Run shutdown processing when the last active reference to an iclog goes away. This guarantees that shutdown processing occurs on all iclogs, but it only occurs when it is safe to do so.
- acknowledge that log state is not consistent once shutdown has been entered and so don't try to apply consistency checking during a shutdown...

At the end of this patch series, shutdown runs once and once only at the first trigger, iclog state is not modified by shutdown, and iclog callbacks and wakeups are not processed until all active references to the iclog(s) are dropped. Hence we now have deterministic shutdown behaviour for both the mount and the log and a consistent iclog lifecycle framework that we can build more complex functionality on top of safely.

Cheers,

Dave.

#6 lore raw

PATCH 2/9 xfs: XLOG_STATE_IOERROR must die

From: Dave Chinner <[email protected]>

We don't need an iclog state field to tell us the log has been shut down. We can just check the xlog_is_shutdown() instead. The avoids the need to shutdowns overwriting the current iclog state while being active used by the log code and so having to ensure that every iclog state check handles XLOG_STATE_IOERROR appropriately.

---
 fs/xfs/xfs_log.c      | 114 +++++++++++++-----------------------------
 fs/xfs/xfs_log_cil.c  |   2 +-
 fs/xfs/xfs_log_priv.h |   5 +-
 fs/xfs/xfs_trace.h    |   1 -
 4 files changed, 36 insertions(+), 86 deletions(-)
fs/xfs/xfs_log.c +34 -80
index 5ae11e7bf2fd..3ea67a90bcde 100644
@@ -522,7 +522,7 @@ xlog_state_release_iclog(
 	lockdep_assert_held(&log->l_icloglock);
 
 	trace_xlog_iclog_release(iclog, _RET_IP_);
-	if (iclog->ic_state == XLOG_STATE_IOERROR)
+	if (xlog_is_shutdown(log))
 		return -EIO;
 
 	if (atomic_dec_and_test(&iclog->ic_refcnt) &&
@@ -857,7 +857,7 @@ xlog_unmount_write(
 	error = xlog_write_unmount_record(log, tic);
 	/*
 	 * At this point, we're umounting anyway, so there's no point in
-	 * transitioning log state to IOERROR. Just continue...
+	 * transitioning log state to shutdown. Just continue...
 	 */
 out_err:
 	if (error)
@@ -870,7 +870,7 @@ xlog_unmount_write(
 		xlog_state_switch_iclogs(log, iclog, 0);
 	else
 		ASSERT(iclog->ic_state == XLOG_STATE_WANT_SYNC ||
-		       iclog->ic_state == XLOG_STATE_IOERROR);
+			xlog_is_shutdown(log));
 	/*
 	 * Ensure the journal is fully flushed and on stable storage once the
 	 * iclog containing the unmount record is written.
@@ -1769,7 +1769,7 @@ xlog_write_iclog(
 	 * across the log IO to archieve that.
 	 */
 	down(&iclog->ic_sema);
-	if (unlikely(iclog->ic_state == XLOG_STATE_IOERROR)) {
+	if (xlog_is_shutdown(log)) {
 		/*
 		 * It would seem logical to return EIO here, but we rely on
 		 * the log state machine to propagate I/O errors instead of
@@ -2298,7 +2298,7 @@ xlog_write_copy_finish(
 			xlog_state_switch_iclogs(log, iclog, 0);
 		else
 			ASSERT(iclog->ic_state == XLOG_STATE_WANT_SYNC ||
-			       iclog->ic_state == XLOG_STATE_IOERROR);
+				xlog_is_shutdown(log));
 		if (!commit_iclog)
 			goto release_iclog;
 		spin_unlock(&log->l_icloglock);
@@ -2713,8 +2713,7 @@ xlog_state_set_callback(
 static bool
 xlog_state_iodone_process_iclog(
 	struct xlog		*log,
-	struct xlog_in_core	*iclog,
-	bool			*ioerror)
+	struct xlog_in_core	*iclog)
 {
 	xfs_lsn_t		lowest_lsn;
 	xfs_lsn_t		header_lsn;
@@ -2726,15 +2725,6 @@ xlog_state_iodone_process_iclog(
 		 * Skip all iclogs in the ACTIVE & DIRTY states:
 		 */
 		return false;
-	case XLOG_STATE_IOERROR:
-		/*
-		 * Between marking a filesystem SHUTDOWN and stopping the log,
-		 * we do flush all iclogs to disk (if there wasn't a log I/O
-		 * error). So, we do want things to go smoothly in case of just
-		 * a SHUTDOWN w/o a LOG_IO_ERROR.
-		 */
-		*ioerror = true;
-		return false;
 	case XLOG_STATE_DONE_SYNC:
 		/*
 		 * Now that we have an iclog that is in the DONE_SYNC state, do
@@ -2765,12 +2755,13 @@ xlog_state_do_callback(
 	struct xlog_in_core	*iclog;
 	struct xlog_in_core	*first_iclog;
 	bool			cycled_icloglock;
-	bool			ioerror;
 	int			flushcnt = 0;
 	int			repeats = 0;
 
 	spin_lock(&log->l_icloglock);
 	do {
+		repeats++;
+
 		/*
 		 * Scan all iclogs starting with the one pointed to by the
 		 * log.  Reset this starting point each time the log is
@@ -2779,23 +2770,21 @@ xlog_state_do_callback(
 		 * Keep looping through iclogs until one full pass is made
 		 * without running any callbacks.
 		 */
-		first_iclog = log->l_iclog;
-		iclog = log->l_iclog;
 		cycled_icloglock = false;
-		ioerror = false;
-		repeats++;
-
-		do {
+		first_iclog = NULL;
+		for (iclog = log->l_iclog;
+		     iclog != first_iclog;
+		     iclog = iclog->ic_next) {
 			LIST_HEAD(cb_list);
 
-			if (xlog_state_iodone_process_iclog(log, iclog,
-							&ioerror))
-				break;
+			if (!first_iclog)
+				first_iclog = iclog;
 
-			if (iclog->ic_state != XLOG_STATE_CALLBACK &&
-			    iclog->ic_state != XLOG_STATE_IOERROR) {
-				iclog = iclog->ic_next;
-				continue;
+			if (!xlog_is_shutdown(log)) {
+				if (xlog_state_iodone_process_iclog(log, iclog))
+					break;
+				if (iclog->ic_state != XLOG_STATE_CALLBACK)
+					continue;
 			}
 			list_splice_init(&iclog->ic_callbacks, &cb_list);
 			spin_unlock(&log->l_icloglock);
@@ -2810,8 +2799,7 @@ xlog_state_do_callback(
 				wake_up_all(&iclog->ic_force_wait);
 			else
 				xlog_state_clean_iclog(log, iclog);
-			iclog = iclog->ic_next;
-		} while (first_iclog != iclog);
+		};
 
 		if (repeats > 5000) {
 			flushcnt += repeats;
@@ -2820,10 +2808,10 @@ xlog_state_do_callback(
 				"%s: possible infinite loop (%d iterations)",
 				__func__, flushcnt);
 		}
-	} while (!ioerror && cycled_icloglock);
+	} while (!xlog_is_shutdown(log) && cycled_icloglock);
 
 	if (log->l_iclog->ic_state == XLOG_STATE_ACTIVE ||
-	    log->l_iclog->ic_state == XLOG_STATE_IOERROR)
+	    xlog_is_shutdown(log))
 		wake_up_all(&log->l_flush_wait);
 
 	spin_unlock(&log->l_icloglock);
@@ -2833,13 +2821,6 @@ xlog_state_do_callback(
 /*
  * Finish transitioning this iclog to the dirty state.
  *
- * Make sure that we completely execute this routine only when this is
- * the last call to the iclog.  There is a good chance that iclog flushes,
- * when we reach the end of the physical log, get turned into 2 separate
- * calls to bwrite.  Hence, one iclog flush could generate two calls to this
- * routine.  By using the reference count bwritecnt, we guarantee that only
- * the second completion goes through.
- *
  * Callbacks could take time, so they are done outside the scope of the
  * global state machine log lock.
  */
@@ -3171,10 +3152,10 @@ xfs_log_force(
 	xlog_cil_force(log);
 
 	spin_lock(&log->l_icloglock);
-	iclog = log->l_iclog;
-	if (iclog->ic_state == XLOG_STATE_IOERROR)
+	if (xlog_is_shutdown(log))
 		goto out_error;
 
+	iclog = log->l_iclog;
 	trace_xlog_iclog_force(iclog, _RET_IP_);
 
 	if (iclog->ic_state == XLOG_STATE_DIRTY ||
@@ -3245,10 +3226,10 @@ xlog_force_lsn(
 	struct xlog_in_core	*iclog;
 
 	spin_lock(&log->l_icloglock);
-	iclog = log->l_iclog;
-	if (iclog->ic_state == XLOG_STATE_IOERROR)
+	if (xlog_is_shutdown(log))
 		goto out_error;
 
+	iclog = log->l_iclog;
 	while (be64_to_cpu(iclog->ic_header.h_lsn) != lsn) {
 		trace_xlog_iclog_force_lsn(iclog, _RET_IP_);
 		iclog = iclog->ic_next;
@@ -3683,34 +3664,6 @@ xlog_verify_iclog(
 }
 #endif
 
-/*
- * Mark all iclogs IOERROR. l_icloglock is held by the caller.
- */
-STATIC int
-xlog_state_ioerror(
-	struct xlog	*log)
-{
-	xlog_in_core_t	*iclog, *ic;
-
-	iclog = log->l_iclog;
-	if (iclog->ic_state != XLOG_STATE_IOERROR) {
-		/*
-		 * Mark all the incore logs IOERROR.
-		 * From now on, no log flushes will result.
-		 */
-		ic = iclog;
-		do {
-			ic->ic_state = XLOG_STATE_IOERROR;
-			ic = ic->ic_next;
-		} while (ic != iclog);
-		return 0;
-	}
-	/*
-	 * Return non-zero, if state transition has already happened.
-	 */
-	return 1;
-}
-
 /*
  * This is called from xfs_force_shutdown, when we're forcibly
  * shutting down the filesystem, typically because of an IO error.
@@ -3726,6 +3679,8 @@ xlog_state_ioerror(
  * Note: for the !logerror case we need to flush the regions held in memory out
  * to disk first. This needs to be done before the log is marked as shutdown,
  * otherwise the iclog writes will fail.
+ *
+ * Return non-zero if log shutdown transition had already happened.
  */
 int
 xfs_log_force_umount(
@@ -3733,7 +3688,7 @@ xfs_log_force_umount(
 	int			logerror)
 {
 	struct xlog	*log;
-	int		retval;
+	int		retval = 0;
 
 	log = mp->m_log;
 
@@ -3753,10 +3708,8 @@ xfs_log_force_umount(
 	 * Somebody could've already done the hard work for us.
 	 * No need to get locks for this.
 	 */
-	if (logerror && log->l_iclog->ic_state == XLOG_STATE_IOERROR) {
-		ASSERT(xlog_is_shutdown(log));
+	if (logerror && xlog_is_shutdown(log))
 		return 1;
-	}
 
 	/*
 	 * Flush all the completed transactions to disk before marking the log
@@ -3781,8 +3734,10 @@ xfs_log_force_umount(
 	 * Mark the log and the iclogs with IO error flags to prevent any
 	 * further log IO from being issued or completed.
 	 */
-	log->l_flags |= XLOG_IO_ERROR;
-	retval = xlog_state_ioerror(log);
+	if (!(log->l_flags & XLOG_IO_ERROR)) {
+		log->l_flags |= XLOG_IO_ERROR;
+		retval = 1;
+	}
 	spin_unlock(&log->l_icloglock);
 
 	/*
@@ -3806,7 +3761,6 @@ xfs_log_force_umount(
 	spin_unlock(&log->l_cilp->xc_push_lock);
 	xlog_state_do_callback(log);
 
-	/* return non-zero if log IOERROR transition had already happened */
 	return retval;
 }
 
fs/xfs/xfs_log_cil.c +1 -1
index 23eec4f76f19..1616d0442cd9 100644
@@ -889,7 +889,7 @@ xlog_cil_push_work(
 	 * callbacks and dropped the icloglock.
 	 */
 	spin_lock(&log->l_icloglock);
-	if (commit_iclog->ic_state == XLOG_STATE_IOERROR) {
+	if (xlog_is_shutdown(log)) {
 		spin_unlock(&log->l_icloglock);
 		goto out_abort;
 	}
fs/xfs/xfs_log_priv.h +1 -4
index 80d4e1325e1d..bf05763ba8df 100644
@@ -47,7 +47,6 @@ enum xlog_iclog_state {
 	XLOG_STATE_DONE_SYNC,	/* Done syncing to disk */
 	XLOG_STATE_CALLBACK,	/* Callback functions now */
 	XLOG_STATE_DIRTY,	/* Dirty IC log, not ready for ACTIVE status */
-	XLOG_STATE_IOERROR,	/* IO error happened in sync'ing log */
 };
 
 #define XLOG_STATE_STRINGS \
@@ -56,9 +55,7 @@ enum xlog_iclog_state {
 	{ XLOG_STATE_SYNCING,	"XLOG_STATE_SYNCING" }, \
 	{ XLOG_STATE_DONE_SYNC,	"XLOG_STATE_DONE_SYNC" }, \
 	{ XLOG_STATE_CALLBACK,	"XLOG_STATE_CALLBACK" }, \
-	{ XLOG_STATE_DIRTY,	"XLOG_STATE_DIRTY" }, \
-	{ XLOG_STATE_IOERROR,	"XLOG_STATE_IOERROR" }
-
+	{ XLOG_STATE_DIRTY,	"XLOG_STATE_DIRTY" }
 
 /*
  * Log ticket flags
fs/xfs/xfs_trace.h +0 -1
index 38f2f67303f7..da49db486b90 100644
@@ -3932,7 +3932,6 @@ TRACE_DEFINE_ENUM(XLOG_STATE_SYNCING);
 TRACE_DEFINE_ENUM(XLOG_STATE_DONE_SYNC);
 TRACE_DEFINE_ENUM(XLOG_STATE_CALLBACK);
 TRACE_DEFINE_ENUM(XLOG_STATE_DIRTY);
-TRACE_DEFINE_ENUM(XLOG_STATE_IOERROR);
 
 DECLARE_EVENT_CLASS(xlog_iclog_class,
 	TP_PROTO(struct xlog_in_core *iclog, unsigned long caller_ip),
Signature · 1 line
2.31.1
#25 lore raw

PATCH 7/9 xfs: separate out log shutdown callback processing

In reply to Dave Chinner

From: Dave Chinner <[email protected]>

The iclog callback processing done during a forced log shutdown has different logic to normal runtime IO completion callback processing. Separate out eh shutdown callbacks into their own function and call that from the shutdown code instead.

We don't need this shutdown specific logic in the normal runtime completion code - we'll always run the shutdown version on shutdown, and it will do what shutdown needs regardless of whether there are racing IO completion callbacks scheduled or in progress. Hence we can also simplify the normal IO completion callpath and only abort if shutdown occurred while we actively were processing callbacks.

---
 fs/xfs/xfs_log.c | 51 +++++++++++++++++++++++++++++++++++-------------
 1 file changed, 37 insertions(+), 14 deletions(-)
fs/xfs/xfs_log.c +37 -14
index bb44dcfcae89..e9e16eeb99e3 100644
@@ -487,6 +487,32 @@ xfs_log_reserve(
 	return error;
 }
 
+/*
+ * Run all the pending iclog callbacks and wake log force waiters and iclog
+ * space waiters so they can process the newly set shutdown state. We really
+ * don't care what order we process callbacks here because the log is shut down
+ * and so state cannot change on disk anymore.
+ */
+static void
+xlog_state_shutdown_callbacks(
+	struct xlog		*log)
+{
+	struct xlog_in_core	*iclog;
+	LIST_HEAD(cb_list);
+
+	spin_lock(&log->l_icloglock);
+	iclog = log->l_iclog;
+	do {
+		list_splice_init(&iclog->ic_callbacks, &cb_list);
+		wake_up_all(&iclog->ic_force_wait);
+	} while ((iclog = iclog->ic_next) != log->l_iclog);
+
+	wake_up_all(&log->l_flush_wait);
+	spin_unlock(&log->l_icloglock);
+
+	xlog_cil_process_committed(&cb_list);
+}
+
 static bool
 __xlog_state_release_iclog(
 	struct xlog		*log,
@@ -2760,7 +2786,10 @@ xlog_state_iodone_process_iclog(
 
 /*
  * Loop over all the iclogs, running attached callbacks on them. Return true if
- * we ran any callbacks, indicating that we dropped the icloglock.
+ * we ran any callbacks, indicating that we dropped the icloglock. We don't need
+ * to handle transient shutdown state here at all because
+ * xlog_state_shutdown_callbacks() will be run to do the necessary shutdown
+ * cleanup of the callbacks.
  */
 static bool
 xlog_state_do_iclog_callbacks(
@@ -2778,12 +2807,10 @@ xlog_state_do_iclog_callbacks(
 		if (!first_iclog)
 			first_iclog = iclog;
 
-		if (!xlog_is_shutdown(log)) {
-			if (xlog_state_iodone_process_iclog(log, iclog))
-				break;
-			if (iclog->ic_state != XLOG_STATE_CALLBACK)
-				continue;
-		}
+		if (xlog_state_iodone_process_iclog(log, iclog))
+			break;
+		if (iclog->ic_state != XLOG_STATE_CALLBACK)
+			continue;
 		list_splice_init(&iclog->ic_callbacks, &cb_list);
 		spin_unlock(&log->l_icloglock);
 
@@ -2793,10 +2820,7 @@ xlog_state_do_iclog_callbacks(
 		ran_callback = true;
 
 		spin_lock(&log->l_icloglock);
-		if (xlog_is_shutdown(log))
-			wake_up_all(&iclog->ic_force_wait);
-		else
-			xlog_state_clean_iclog(log, iclog);
+		xlog_state_clean_iclog(log, iclog);
 	};
 	return ran_callback;
 }
@@ -2827,8 +2851,7 @@ xlog_state_do_callback(
 		}
 	};
 
-	if (log->l_iclog->ic_state == XLOG_STATE_ACTIVE ||
-	    xlog_is_shutdown(log))
+	if (log->l_iclog->ic_state == XLOG_STATE_ACTIVE)
 		wake_up_all(&log->l_flush_wait);
 
 	spin_unlock(&log->l_icloglock);
@@ -3761,7 +3784,7 @@ xlog_force_shutdown(
 	spin_lock(&log->l_cilp->xc_push_lock);
 	wake_up_all(&log->l_cilp->xc_commit_wait);
 	spin_unlock(&log->l_cilp->xc_push_lock);
-	xlog_state_do_callback(log);
+	xlog_state_shutdown_callbacks(log);
 
 	return log_error;
 }
Signature · 1 line
2.31.1
#31 lore raw

PATCH 9/9 xfs: log head and tail aren't reliable during shutdown

In reply to Dave Chinner

From: Dave Chinner <[email protected]>

I'm seeing assert failures from xlog_space_left() after a shutdown has begun that look like:

XFS (dm-0): log I/O error -5
XFS (dm-0): xfs_do_force_shutdown(0x2) called from line 1338 of file fs/xfs/xfs_log.c. Return address = xlog_ioend_work+0x64/0xc0
XFS (dm-0): Log I/O Error Detected.
XFS (dm-0): Shutting down filesystem. Please unmount the filesystem and rectify the problem(s)
XFS (dm-0): xlog_space_left: head behind tail
XFS (dm-0):   tail_cycle = 6, tail_bytes = 2706944
XFS (dm-0):   GH   cycle = 6, GH   bytes = 1633867
XFS: Assertion failed: 0, file: fs/xfs/xfs_log.c, line: 1310
------------[ cut here ]------------
Call Trace:
 xlog_space_left+0xc3/0x110
 xlog_grant_push_threshold+0x3f/0xf0
 xlog_grant_push_ail+0x12/0x40
 xfs_log_reserve+0xd2/0x270
 ? __might_sleep+0x4b/0x80
 xfs_trans_reserve+0x18b/0x260

.....

There are two things here. Firstly, after a shutdown, the log head and tail can be out of whack as things abort and release (or don't release) resources, so checking them for sanity doesn't make much sense. Secondly, xfs_log_reserve() can race with shutdown and so it can still fail like this even though it has already checked for a log shutdown before calling xlog_grant_push_ail().

So, before ASSERT failing in xlog_space_left(), make sure we haven't already shut down....

---
 fs/xfs/xfs_log.c | 51 ++++++++++++++++++++++++------------------------
 1 file changed, 26 insertions(+), 25 deletions(-)
fs/xfs/xfs_log.c +26 -25
index caa07631b2e5..be066a8f90b8 100644
@@ -1269,16 +1269,18 @@ xlog_assign_tail_lsn(
  * wrap the tail, we should blow up.  Rather than catch this case here,
  * we depend on other ASSERTions in other parts of the code.   XXXmiken
  *
- * This code also handles the case where the reservation head is behind
- * the tail.  The details of this case are described below, but the end
- * result is that we return the size of the log as the amount of space left.
+ * If reservation head is behind the tail, we have a problem. Warn about it,
+ * but then treat it as if the log is empty.
+ *
+ * If the log is shut down, the head and tail may be invalid or out of whack, so
+ * shortcut invalidity asserts in this case so that we don't trigger them
+ * falsely.
  */
 STATIC int
 xlog_space_left(
 	struct xlog	*log,
 	atomic64_t	*head)
 {
-	int		free_bytes;
 	int		tail_bytes;
 	int		tail_cycle;
 	int		head_cycle;
@@ -1287,30 +1289,29 @@ xlog_space_left(
 	xlog_crack_grant_head(head, &head_cycle, &head_bytes);
 	xlog_crack_atomic_lsn(&log->l_tail_lsn, &tail_cycle, &tail_bytes);
 	tail_bytes = BBTOB(tail_bytes);
-	if (tail_cycle == head_cycle && head_bytes >= tail_bytes)
-		free_bytes = log->l_logsize - (head_bytes - tail_bytes);
-	else if (tail_cycle + 1 < head_cycle)
+	if (tail_cycle == head_cycle && head_bytes >= tail_bytes) {
+		return log->l_logsize - (head_bytes - tail_bytes);
+	} else if (tail_cycle + 1 < head_cycle) {
 		return 0;
-	else if (tail_cycle < head_cycle) {
+	} else if (xlog_is_shutdown(log)) {
+		/* Ignore potential inconsistency when shutdown. */
+		return log->l_logsize;
+	} else if (tail_cycle < head_cycle) {
 		ASSERT(tail_cycle == (head_cycle - 1));
-		free_bytes = tail_bytes - head_bytes;
-	} else {
-		/*
-		 * The reservation head is behind the tail.
-		 * In this case we just want to return the size of the
-		 * log as the amount of space left.
-		 */
-		xfs_alert(log->l_mp, "xlog_space_left: head behind tail");
-		xfs_alert(log->l_mp,
-			  "  tail_cycle = %d, tail_bytes = %d",
-			  tail_cycle, tail_bytes);
-		xfs_alert(log->l_mp,
-			  "  GH   cycle = %d, GH   bytes = %d",
-			  head_cycle, head_bytes);
-		ASSERT(0);
-		free_bytes = log->l_logsize;
+		return tail_bytes - head_bytes;
 	}
-	return free_bytes;
+
+	/*
+	 * The reservation head is behind the tail. In this case we just want to
+	 * return the size of the log as the amount of space left.
+	 */
+	xfs_alert(log->l_mp, "xlog_space_left: head behind tail");
+	xfs_alert(log->l_mp, "  tail_cycle = %d, tail_bytes = %d",
+		  tail_cycle, tail_bytes);
+	xfs_alert(log->l_mp, "  GH   cycle = %d, GH   bytes = %d",
+		  head_cycle, head_bytes);
+	ASSERT(0);
+	return log->l_logsize;
 }
 
 
Signature · 1 line
2.31.1
#7 lore raw

Re: [PATCH 2/9] xfs: XLOG_STATE_IOERROR must die

In reply to Dave Chinner
Thank you for the patch! Perhaps something to improve:

Hi Dave,

Thank you for the patch! Perhaps something to improve:

[auto build test WARNING on xfs-linux/for-next]
[also build test WARNING on next-20210630]
[cannot apply to v5.13]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Dave-Chinner/xfs-shutdown-is-a-racy-mess/20210630-144028
base:   https://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git for-next
config: openrisc-randconfig-c023-20210630 (attached as .config)
compiler: or1k-linux-gcc (GCC) 9.3.0

If you fix the issue, kindly add following tag as appropriate

cocci warnings: (new ones prefixed by >>)

>> fs/xfs/xfs_log.c:2803:3-4: Unneeded semicolon

Please review and possibly fold the followup patch.

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

#8 lore raw

[PATCH] xfs: fix semicolon.cocci warnings

In reply to Dave Chinner
fs/xfs/xfs_log.c:2803:3-4: Unneeded semicolon

From: kernel test robot <[email protected]>

fs/xfs/xfs_log.c:2803:3-4: Unneeded semicolon

Remove unneeded semicolon.

Generated by: scripts/coccinelle/misc/semicolon.cocci

---

url:    https://github.com/0day-ci/linux/commits/Dave-Chinner/xfs-shutdown-is-a-racy-mess/20210630-144028
base:   https://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git for-next

:::::: branch date: 9 hours ago
:::::: commit date: 9 hours ago

 xfs_log.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
fs/xfs/xfs_log.c +1 -1
@@ -2800,7 +2800,7 @@ xlog_state_do_callback(
 				wake_up_all(&iclog->ic_force_wait);
 			else
 				xlog_state_clean_iclog(log, iclog);
-		};
+		}
 
 		if (repeats > 5000) {
 			flushcnt += repeats;
Reviewed-by on v1 1/9
#2 lore raw

Re: [PATCH 1/9] xfs: convert XLOG_FORCED_SHUTDOWN() to xlog_is_shutdown()

In reply to Dave Chinner
Quoted 6 lines · Dave Chinner

From: Dave Chinner <[email protected]>

Make it less shouty and a static inline before adding more calls through the log code.

Easy peasy,

--D

Quoted 273 lines
---
 fs/xfs/xfs_log.c         | 32 ++++++++++++++++----------------
 fs/xfs/xfs_log_cil.c     | 10 +++++-----
 fs/xfs/xfs_log_priv.h    |  7 +++++--
 fs/xfs/xfs_log_recover.c |  9 +++------
 fs/xfs/xfs_trans.c       |  2 +-
 5 files changed, 30 insertions(+), 30 deletions(-)
fs/xfs/xfs_log.c +16 -16
index 71fd8c0cad92..5ae11e7bf2fd 100644
@@ -247,7 +247,7 @@ xlog_grant_head_wait(
 	list_add_tail(&tic->t_queue, &head->waiters);
 
 	do {
-		if (XLOG_FORCED_SHUTDOWN(log))
+		if (xlog_is_shutdown(log))
 			goto shutdown;
 		xlog_grant_push_ail(log, need_bytes);
 
@@ -261,7 +261,7 @@ xlog_grant_head_wait(
 		trace_xfs_log_grant_wake(log, tic);
 
 		spin_lock(&head->lock);
-		if (XLOG_FORCED_SHUTDOWN(log))
+		if (xlog_is_shutdown(log))
 			goto shutdown;
 	} while (xlog_space_left(log, &head->grant) < need_bytes);
 
@@ -366,7 +366,7 @@ xfs_log_writable(
 		return false;
 	if (xfs_readonly_buftarg(mp->m_log->l_targ))
 		return false;
-	if (XFS_FORCED_SHUTDOWN(mp))
+	if (xlog_is_shutdown(mp->m_log))
 		return false;
 	return true;
 }
@@ -383,7 +383,7 @@ xfs_log_regrant(
 	int			need_bytes;
 	int			error = 0;
 
-	if (XLOG_FORCED_SHUTDOWN(log))
+	if (xlog_is_shutdown(log))
 		return -EIO;
 
 	XFS_STATS_INC(mp, xs_try_logspace);
@@ -451,7 +451,7 @@ xfs_log_reserve(
 
 	ASSERT(client == XFS_TRANSACTION || client == XFS_LOG);
 
-	if (XLOG_FORCED_SHUTDOWN(log))
+	if (xlog_is_shutdown(log))
 		return -EIO;
 
 	XFS_STATS_INC(mp, xs_try_logspace);
@@ -787,7 +787,7 @@ xlog_wait_on_iclog(
 	struct xlog		*log = iclog->ic_log;
 
 	trace_xlog_iclog_wait_on(iclog, _RET_IP_);
-	if (!XLOG_FORCED_SHUTDOWN(log) &&
+	if (!xlog_is_shutdown(log) &&
 	    iclog->ic_state != XLOG_STATE_ACTIVE &&
 	    iclog->ic_state != XLOG_STATE_DIRTY) {
 		XFS_STATS_INC(log->l_mp, xs_log_force_sleep);
@@ -796,7 +796,7 @@ xlog_wait_on_iclog(
 		spin_unlock(&log->l_icloglock);
 	}
 
-	if (XLOG_FORCED_SHUTDOWN(log))
+	if (xlog_is_shutdown(log))
 		return -EIO;
 	return 0;
 }
@@ -915,7 +915,7 @@ xfs_log_unmount_write(
 
 	xfs_log_force(mp, XFS_LOG_SYNC);
 
-	if (XLOG_FORCED_SHUTDOWN(log))
+	if (xlog_is_shutdown(log))
 		return;
 
 	/*
@@ -1024,7 +1024,7 @@ xfs_log_space_wake(
 	struct xlog		*log = mp->m_log;
 	int			free_bytes;
 
-	if (XLOG_FORCED_SHUTDOWN(log))
+	if (xlog_is_shutdown(log))
 		return;
 
 	if (!list_empty_careful(&log->l_write_head.waiters)) {
@@ -1115,7 +1115,7 @@ xfs_log_cover(
 
 	ASSERT((xlog_cil_empty(mp->m_log) && xlog_iclogs_empty(mp->m_log) &&
 	        !xfs_ail_min_lsn(mp->m_log->l_ailp)) ||
-	       XFS_FORCED_SHUTDOWN(mp));
+		xlog_is_shutdown(mp->m_log));
 
 	if (!xfs_log_writable(mp))
 		return 0;
@@ -1546,7 +1546,7 @@ xlog_commit_record(
 	};
 	int	error;
 
-	if (XLOG_FORCED_SHUTDOWN(log))
+	if (xlog_is_shutdown(log))
 		return -EIO;
 
 	error = xlog_write(log, &vec, ticket, lsn, iclog, XLOG_COMMIT_TRANS);
@@ -1627,7 +1627,7 @@ xlog_grant_push_ail(
 	xfs_lsn_t	threshold_lsn;
 
 	threshold_lsn = xlog_grant_push_threshold(log, need_bytes);
-	if (threshold_lsn == NULLCOMMITLSN || XLOG_FORCED_SHUTDOWN(log))
+	if (threshold_lsn == NULLCOMMITLSN || xlog_is_shutdown(log))
 		return;
 
 	/*
@@ -2806,7 +2806,7 @@ xlog_state_do_callback(
 			cycled_icloglock = true;
 
 			spin_lock(&log->l_icloglock);
-			if (XLOG_FORCED_SHUTDOWN(log))
+			if (xlog_is_shutdown(log))
 				wake_up_all(&iclog->ic_force_wait);
 			else
 				xlog_state_clean_iclog(log, iclog);
@@ -2858,7 +2858,7 @@ xlog_state_done_syncing(
 	 * split log writes, on the second, we shut down the file system and
 	 * no iclogs should ever be attempted to be written to disk again.
 	 */
-	if (!XLOG_FORCED_SHUTDOWN(log)) {
+	if (!xlog_is_shutdown(log)) {
 		ASSERT(iclog->ic_state == XLOG_STATE_SYNCING);
 		iclog->ic_state = XLOG_STATE_DONE_SYNC;
 	}
@@ -2906,7 +2906,7 @@ xlog_state_get_iclog_space(
 
 restart:
 	spin_lock(&log->l_icloglock);
-	if (XLOG_FORCED_SHUTDOWN(log)) {
+	if (xlog_is_shutdown(log)) {
 		spin_unlock(&log->l_icloglock);
 		return -EIO;
 	}
@@ -3754,7 +3754,7 @@ xfs_log_force_umount(
 	 * No need to get locks for this.
 	 */
 	if (logerror && log->l_iclog->ic_state == XLOG_STATE_IOERROR) {
-		ASSERT(XLOG_FORCED_SHUTDOWN(log));
+		ASSERT(xlog_is_shutdown(log));
 		return 1;
 	}
 
fs/xfs/xfs_log_cil.c +5 -5
index d162e8b83e90..23eec4f76f19 100644
@@ -584,7 +584,7 @@ xlog_cil_committed(
 	struct xfs_cil_ctx	*ctx)
 {
 	struct xfs_mount	*mp = ctx->cil->xc_log->l_mp;
-	bool			abort = XLOG_FORCED_SHUTDOWN(ctx->cil->xc_log);
+	bool			abort = xlog_is_shutdown(ctx->cil->xc_log);
 
 	/*
 	 * If the I/O failed, we're aborting the commit and already shutdown.
@@ -853,7 +853,7 @@ xlog_cil_push_work(
 		 * shutdown, but then went back to sleep once already in the
 		 * shutdown state.
 		 */
-		if (XLOG_FORCED_SHUTDOWN(log)) {
+		if (xlog_is_shutdown(log)) {
 			spin_unlock(&cil->xc_push_lock);
 			goto out_abort_free_ticket;
 		}
@@ -962,7 +962,7 @@ xlog_cil_push_work(
 out_abort_free_ticket:
 	xfs_log_ticket_ungrant(log, tic);
 out_abort:
-	ASSERT(XLOG_FORCED_SHUTDOWN(log));
+	ASSERT(xlog_is_shutdown(log));
 	xlog_cil_committed(ctx);
 }
 
@@ -1115,7 +1115,7 @@ xlog_cil_commit(
 
 	xlog_cil_insert_items(log, tp);
 
-	if (regrant && !XLOG_FORCED_SHUTDOWN(log))
+	if (regrant && !xlog_is_shutdown(log))
 		xfs_log_ticket_regrant(log, tp->t_ticket);
 	else
 		xfs_log_ticket_ungrant(log, tp->t_ticket);
@@ -1188,7 +1188,7 @@ xlog_cil_force_seq(
 		 * shutdown, but then went back to sleep once already in the
 		 * shutdown state.
 		 */
-		if (XLOG_FORCED_SHUTDOWN(log))
+		if (xlog_is_shutdown(log))
 			goto out_shutdown;
 		if (ctx->sequence > sequence)
 			continue;
fs/xfs/xfs_log_priv.h +5 -2
index 4c41bbfa33b0..80d4e1325e1d 100644
@@ -454,8 +454,11 @@ struct xlog {
 #define XLOG_BUF_CANCEL_BUCKET(log, blkno) \
 	((log)->l_buf_cancel_table + ((uint64_t)blkno % XLOG_BC_TABLE_SIZE))
 
-#define XLOG_FORCED_SHUTDOWN(log) \
-	(unlikely((log)->l_flags & XLOG_IO_ERROR))
+static inline bool
+xlog_is_shutdown(struct xlog *log)
+{
+	return (log->l_flags & XLOG_IO_ERROR);
+}
 
 /* common routines */
 extern int
fs/xfs/xfs_log_recover.c +3 -6
index 1a291bf50776..f1b828dedb25 100644
@@ -144,7 +144,7 @@ xlog_do_io(
 
 	error = xfs_rw_bdev(log->l_targ->bt_bdev, log->l_logBBstart + blk_no,
 			BBTOB(nbblks), data, op);
-	if (error && !XFS_FORCED_SHUTDOWN(log->l_mp)) {
+	if (error && !xlog_is_shutdown(log)) {
 		xfs_alert(log->l_mp,
 			  "log recovery %s I/O error at daddr 0x%llx len %d error %d",
 			  op == REQ_OP_WRITE ? "write" : "read",
@@ -3278,10 +3278,7 @@ xlog_do_recover(
 	if (error)
 		return error;
 
-	/*
-	 * If IO errors happened during recovery, bail out.
-	 */
-	if (XFS_FORCED_SHUTDOWN(mp))
+	if (xlog_is_shutdown(log))
 		return -EIO;
 
 	/*
@@ -3303,7 +3300,7 @@ xlog_do_recover(
 	xfs_buf_hold(bp);
 	error = _xfs_buf_read(bp, XBF_READ);
 	if (error) {
-		if (!XFS_FORCED_SHUTDOWN(mp)) {
+		if (!xlog_is_shutdown(log)) {
 			xfs_buf_ioerror_alert(bp, __this_address);
 			ASSERT(0);
 		}
fs/xfs/xfs_trans.c +1 -1
index 87bffd12c20c..e26ade9fc630 100644
@@ -908,7 +908,7 @@ __xfs_trans_commit(
 	 */
 	xfs_trans_unreserve_and_mod_dquots(tp);
 	if (tp->t_ticket) {
-		if (regrant && !XLOG_FORCED_SHUTDOWN(mp->m_log))
+		if (regrant && !xlog_is_shutdown(mp->m_log))
 			xfs_log_ticket_regrant(mp->m_log, tp->t_ticket);
 		else
 			xfs_log_ticket_ungrant(mp->m_log, tp->t_ticket);
Signature · 1 line
2.31.1
#3 lore raw

Re: [PATCH 1/9] xfs: convert XLOG_FORCED_SHUTDOWN() to xlog_is_shutdown()

In reply to Dave Chinner

fs/xfs/xfs_log.c xfs_log_writable( · line 369

Quoted 6 lines
fs/xfs/xfs_log.c +1 -1
@@ -366,7 +366,7 @@ xfs_log_writable(
 		return false;
 	if (xfs_readonly_buftarg(mp->m_log->l_targ))
 		return false;
-	if (XFS_FORCED_SHUTDOWN(mp))
+	if (xlog_is_shutdown(mp->m_log))

This wasn't XLOG_FORCED_SHUTDOWN to start with. Same for a few more spots.

#9 lore raw

Re: [PATCH 2/9] xfs: XLOG_STATE_IOERROR must die

In reply to Dave Chinner

fs/xfs/xfs_log.c xlog_unmount_write( · line 873

Quoted 4 lines
fs/xfs/xfs_log.c +1 -1
 	else
 		ASSERT(iclog->ic_state == XLOG_STATE_WANT_SYNC ||
-		       iclog->ic_state == XLOG_STATE_IOERROR);
+			xlog_is_shutdown(log));

Nit: think doing this as:

	else if (!xlog_is_shutdown(log))
		ASSERT(iclog->ic_state == XLOG_STATE_WANT_SYNC);

would be a tad cleaner.

fs/xfs/xfs_log.c xlog_unmount_write( · line 873

Quoted 4 lines
fs/xfs/xfs_log.c +1 -1
 		else
 			ASSERT(iclog->ic_state == XLOG_STATE_WANT_SYNC ||
-			       iclog->ic_state == XLOG_STATE_IOERROR);
+				xlog_is_shutdown(log));

Same here.

Quoted 51 lines
fs/xfs/xfs_log.c +13 -14
@@ -2765,12 +2755,13 @@ xlog_state_do_callback(
 	struct xlog_in_core	*iclog;
 	struct xlog_in_core	*first_iclog;
 	bool			cycled_icloglock;
-	bool			ioerror;
 	int			flushcnt = 0;
 	int			repeats = 0;
 
 	spin_lock(&log->l_icloglock);
 	do {
+		repeats++;
+
 		/*
 		 * Scan all iclogs starting with the one pointed to by the
 		 * log.  Reset this starting point each time the log is
@@ -2779,23 +2770,21 @@ xlog_state_do_callback(
 		 * Keep looping through iclogs until one full pass is made
 		 * without running any callbacks.
 		 */
-		first_iclog = log->l_iclog;
-		iclog = log->l_iclog;
 		cycled_icloglock = false;
-		ioerror = false;
-		repeats++;
-
-		do {
+		first_iclog = NULL;
+		for (iclog = log->l_iclog;
+		     iclog != first_iclog;
+		     iclog = iclog->ic_next) {
 			LIST_HEAD(cb_list);
 
-			if (xlog_state_iodone_process_iclog(log, iclog,
-							&ioerror))
-				break;
+			if (!first_iclog)
+				first_iclog = iclog;
 
-			if (iclog->ic_state != XLOG_STATE_CALLBACK &&
-			    iclog->ic_state != XLOG_STATE_IOERROR) {
-				iclog = iclog->ic_next;
-				continue;
+			if (!xlog_is_shutdown(log)) {
+				if (xlog_state_iodone_process_iclog(log, iclog))
+					break;
+				if (iclog->ic_state != XLOG_STATE_CALLBACK)
+					continue;
 			}
 			list_splice_init(&iclog->ic_callbacks, &cb_list);
 			spin_unlock(&log->l_icloglock);

fs/xfs/xfs_log.c xlog_state_do_callback( · line 2802

Quoted 6 lines
fs/xfs/xfs_log.c +1 -2
@@ -2810,8 +2799,7 @@ xlog_state_do_callback(
 				wake_up_all(&iclog->ic_force_wait);
 			else
 				xlog_state_clean_iclog(log, iclog);
-			iclog = iclog->ic_next;
-		} while (first_iclog != iclog);
+		};

I don't think swiching to a random other iteration style here helps to review. Please keep the old in in this otherwise pretty mechnical change and clean it up later if you strongly prefer the new style. (which I'm really not sold on, btw - yes it makes the continiue easier but otherwise isn't all that great either).

Otherwise looks fine and long overdue.

Reviewed-by on v1 4/9
#14 lore raw

Re: [PATCH 4/9] xfs: convert log flags to an operational state field

In reply to Dave Chinner
Quoted 11 lines
fs/xfs/xfs_log.c +1 -0
@@ -552,6 +552,7 @@ xfs_log_mount(
 	xfs_daddr_t	blk_offset,
 
 {
+	struct xlog	*log;
 	bool		fatal = xfs_sb_version_hascrc(&mp->m_sb);
 	int		error = 0;
 	int		min_logfsbs;
@@ -566,11 +567,12 @@ xfs_log_mount(
 		ASSERT(mp->m_flags & XFS_MOUNT_RDONLY);
 	}

fs/xfs/xfs_log.c xfs_log_mount( · line 575

Quoted 10 lines
fs/xfs/xfs_log.c +4 -3
 
-	mp->m_log = xlog_alloc_log(mp, log_target, blk_offset, num_bblks);
-	if (IS_ERR(mp->m_log)) {
-		error = PTR_ERR(mp->m_log);
+	log = xlog_alloc_log(mp, log_target, blk_offset, num_bblks);
+	if (IS_ERR(log)) {
+		error = PTR_ERR(log);
 		goto out;
 	}
+	mp->m_log = log;

Additition of the local variable here looks rather unrelated, given that the log is only touched twice in relation to the flags.

Otherwise looks good:

#17 lore raw

Re: [PATCH 5/9] xfs: make forced shutdown processing atomic

In reply to Dave Chinner

fs/xfs/xfs_fsops.c xfs_do_force_shutdown( · line 538

Quoted 9 lines
fs/xfs/xfs_fsops.c +7 -0
+	spin_lock(&mp->m_sb_lock);
+	if (XFS_FORCED_SHUTDOWN(mp)) {
+		spin_unlock(&mp->m_sb_lock);
 		return;
 	}
+	mp->m_flags |= XFS_MOUNT_FS_SHUTDOWN;
+	if (mp->m_sb_bp)
+		mp->m_sb_bp->b_flags |= XBF_DONE;
+	spin_unlock(&mp->m_sb_lock);

Any particular reason for picking m_sb_lock which so far doesn't seem to be related to mp->m_flags at all? (On which we probably have a few other races, most notably remount).

fs/xfs/xfs_fsops.c xfs_do_force_shutdown( · line 560

Quoted 2 lines
fs/xfs/xfs_fsops.c +2 -0
+	if (xfs_error_level >= XFS_ERRLEVEL_HIGH)
+		xfs_stack_trace();

This seems new and unrelated?

Quoted 1 line

+

Spurious empty line at the end of the function.

#26 lore raw

Re: [PATCH 7/9] xfs: separate out log shutdown callback processing

In reply to Dave Chinner
Quoted 7 lines · Dave Chinner

From: Dave Chinner <[email protected]>

The iclog callback processing done during a forced log shutdown has different logic to normal runtime IO completion callback processing. Separate out eh shutdown callbacks into their own function and call that from the shutdown code instead.

Quoted 6 lines

We don't need this shutdown specific logic in the normal runtime completion code - we'll always run the shutdown version on shutdown, and it will do what shutdown needs regardless of whether there are racing IO completion callbacks scheduled or in progress. Hence we can also simplify the normal IO completion callpath and only abort if shutdown occurred while we actively were processing callbacks.

What prevents a log shutdown from coming in during the callback processing? Or is there a reason why we simply don't care for that case?

#4 lore raw

Re: [PATCH 1/9] xfs: convert XLOG_FORCED_SHUTDOWN() to xlog_is_shutdown()

In reply to Christoph Hellwig
Quoted 9 lines · Christoph Hellwig

fs/xfs/xfs_log.c xfs_log_writable( · line 369

Quoted 6 lines
fs/xfs/xfs_log.c +1 -1
@@ -366,7 +366,7 @@ xfs_log_writable(
 		return false;
 	if (xfs_readonly_buftarg(mp->m_log->l_targ))
 		return false;
-	if (XFS_FORCED_SHUTDOWN(mp))
+	if (xlog_is_shutdown(mp->m_log))

This wasn't XLOG_FORCED_SHUTDOWN to start with. Same for a few more spots.

Yup, but in the places where we are working on the log, we should be checking the log state for shutdown, not the mount. They currently mean the same thing, but that doesn't mean we should use mount based checks in the log and vice versa.

Cheers,

Dave.

Signature · 2 lines
Dave Chinner
[email protected]
Reviewed-by on v1 8/9
#29 lore raw

Re: [PATCH 8/9] xfs: don't run shutdown callbacks on active iclogs

In reply to Dave Chinner
Quoted 5 lines · Dave Chinner

The problem is that xlog_state_release_iclog() aborts before doing anything if the log is already shut down. It assumes that the callbacks have already been cleaned up, and it doesn't need to do any cleanup.

Quoted 7 lines

Hence the fix is to remove the xlog_is_shutdown() check from xlog_state_release_iclog() so that reference counts are correctly released from the iclogs, and when the reference count is zero we always transition to SYNCING if the log is shut down. Hence we'll always enter the xlog_sync() path in a shutdown and eventually end up erroring out the iclog IO and running xlog_state_do_callback() to process the callbacks attached to the iclog.

Ok, this answers my question to the previous patch. Maybe add a little blurb there?

Quoted 9 lines
fs/xfs/xfs_log.c +9 -0
+	if (xlog_is_shutdown(log)) {
+		/*
+		 * No more references to this iclog, so process the pending
+		 * iclog callbacks that were waiting on the release of this
+		 * iclog.
+		 */
+		spin_unlock(&log->l_icloglock);
+		xlog_state_shutdown_callbacks(log);
+		spin_lock(&log->l_icloglock);

fs/xfs/xfs_log.c __xlog_state_release_iclog(

Quoted 10 lines
fs/xfs/xfs_log.c +4 -0
+	} else if (xlog_state_want_sync(log, iclog)) {
 		spin_unlock(&log->l_icloglock);
 		xlog_sync(log, iclog);
 		spin_lock(&log->l_icloglock);
 
 
+out_check_shutdown:
+	if (xlog_is_shutdown(log))
+		return -EIO;
 	return 0;

Nit: we can just return -EIO directly in the first xlog_is_shutdown block.. It's not going to make any difference for the CPU, but makes the code a little easier to follow.

Otherwise looks good:

Reviewed-by on v1 9/9
#32 lore raw

Re: [PATCH 9/9] xfs: log head and tail aren't reliable during shutdown

In reply to Dave Chinner

fs/xfs/xfs_log.c xlog_space_left( · line 1302

Quoted 11 lines
fs/xfs/xfs_log.c +8 -0
+	if (tail_cycle == head_cycle && head_bytes >= tail_bytes) {
+		return log->l_logsize - (head_bytes - tail_bytes);
+	} else if (tail_cycle + 1 < head_cycle) {
 		return 0;
+	} else if (xlog_is_shutdown(log)) {
+		/* Ignore potential inconsistency when shutdown. */
+		return log->l_logsize;
+	} else if (tail_cycle < head_cycle) {
 		ASSERT(tail_cycle == (head_cycle - 1));
+		return tail_bytes - head_bytes;
 	}

Drop the else after the returns to make this a little easier to follow:

	if (tail_cycle == head_cycle && head_bytes >= tail_bytes)
		return log->l_logsize - (head_bytes - tail_bytes);
	if (tail_cycle + 1 < head_cycle)
		return 0;

	/* Ignore potential inconsistency when shutdown. */
	if (xlog_is_shutdown(log)) {
		return log->l_logsize;

	if (tail_cycle < head_cycle) {
		ASSERT(tail_cycle == (head_cycle - 1));
		return tail_bytes - head_bytes;
	}

Otherwise looks good:

#10 lore raw

Re: [PATCH 2/9] xfs: XLOG_STATE_IOERROR must die

In reply to Christoph Hellwig
Quoted 11 lines · Christoph Hellwig

fs/xfs/xfs_log.c xlog_unmount_write( · line 873

Quoted 4 lines
fs/xfs/xfs_log.c +1 -1
 	else
 		ASSERT(iclog->ic_state == XLOG_STATE_WANT_SYNC ||
-		       iclog->ic_state == XLOG_STATE_IOERROR);
+			xlog_is_shutdown(log));

Nit: think doing this as:

	else if (!xlog_is_shutdown(log))
		ASSERT(iclog->ic_state == XLOG_STATE_WANT_SYNC);

would be a tad cleaner.

I kill a lot of these checks in the near future. Once the log shutdown doesn't change the iclog state, we no longer need to check if the log is shutdown when checking state like this because shutdown doesn't change the iclog state when we have active references to the iclog....

IOWs, these are all figments of the horrible racy log shutdown that changes the iclog state regardless of who is using the iclog at the time. This pattern sucks, it's broken and it goes away soon so there's not much point in rearranging these asserts...

Cheers,

Dave.

Signature · 2 lines
Dave Chinner
[email protected]
#5 lore raw

Re: [PATCH 1/9] xfs: convert XLOG_FORCED_SHUTDOWN() to xlog_is_shutdown()

In reply to Dave Chinner
Quoted 11 lines · Dave Chinner
Quoted 9 lines · Christoph Hellwig

fs/xfs/xfs_log.c xfs_log_writable( · line 369

Quoted 6 lines
fs/xfs/xfs_log.c +1 -1
@@ -366,7 +366,7 @@ xfs_log_writable(
 		return false;
 	if (xfs_readonly_buftarg(mp->m_log->l_targ))
 		return false;
-	if (XFS_FORCED_SHUTDOWN(mp))
+	if (xlog_is_shutdown(mp->m_log))

This wasn't XLOG_FORCED_SHUTDOWN to start with. Same for a few more spots.

Quoted 4 lines

Yup, but in the places where we are working on the log, we should be checking the log state for shutdown, not the mount. They currently mean the same thing, but that doesn't mean we should use mount based checks in the log and vice versa.

Need documentation in the changelog, or even better a separate commit.

#18 lore raw

Re: [PATCH 5/9] xfs: make forced shutdown processing atomic

In reply to Christoph Hellwig
Quoted 5 lines · Christoph Hellwig
Quoted 5 lines
fs/xfs/xfs_fsops.c +3 -0
+	spin_lock(&mp->m_sb_lock);
+	if (XFS_FORCED_SHUTDOWN(mp)) {
+		spin_unlock(&mp->m_sb_lock);
 		return;
 	}
Quoted 8 lines

fs/xfs/xfs_fsops.c xfs_do_force_shutdown( · line 538

Quoted 4 lines
fs/xfs/xfs_fsops.c +4 -0
+	mp->m_flags |= XFS_MOUNT_FS_SHUTDOWN;
+	if (mp->m_sb_bp)
+		mp->m_sb_bp->b_flags |= XBF_DONE;
+	spin_unlock(&mp->m_sb_lock);

Any particular reason for picking m_sb_lock which so far doesn't seem to be related to mp->m_flags at all? (On which we probably have a few other races, most notably remount).

I was just reusing an existing lock rather than having to add yet another global scope spinlock just for the shutdown. I can add another lock but...

... as you point out, m_flags is a mess when it comes to races. I've got a series somewhere that addresses this... Yeah:

https://lore.kernel.org/linux-xfs/[email protected]/

And that does similar things bit making state changes atomic as this patchset does, in which case the lock around this shutdown state change goes away...

I need to rebase that series and get it moving again, because we really do need to split m_flags up into features and atomic operational state, too. In the mean time, I considered using the m_sb_lock largely harmless...

Quoted 4 lines

fs/xfs/xfs_fsops.c xfs_do_force_shutdown( · line 560

Quoted 2 lines
fs/xfs/xfs_fsops.c +2 -0
+	if (xfs_error_level >= XFS_ERRLEVEL_HIGH)
+		xfs_stack_trace();

This seems new and unrelated?

It's run on the majority of shutdowns already, but not all. It makes no sense to have an error level triggered stack dump on shutdown and not actually use it multiple shutdown vectors - that has been problematic in the past when trying to diagnose shutdown causes in the field. I'll add a comment to the commit description.

Quoted 3 lines
Quoted 1 line

+

Spurious empty line at the end of the function.

Removed.

-Dave.

Signature · 2 lines
Dave Chinner
[email protected]
#27 lore raw

Re: [PATCH 7/9] xfs: separate out log shutdown callback processing

In reply to Christoph Hellwig
Quoted 15 lines · Christoph Hellwig
Quoted 7 lines · Dave Chinner

From: Dave Chinner <[email protected]>

The iclog callback processing done during a forced log shutdown has different logic to normal runtime IO completion callback processing. Separate out eh shutdown callbacks into their own function and call that from the shutdown code instead.

Quoted 6 lines

We don't need this shutdown specific logic in the normal runtime completion code - we'll always run the shutdown version on shutdown, and it will do what shutdown needs regardless of whether there are racing IO completion callbacks scheduled or in progress. Hence we can also simplify the normal IO completion callpath and only abort if shutdown occurred while we actively were processing callbacks.

Quoted 3 lines

What prevents a log shutdown from coming in during the callback processing? Or is there a reason why we simply don't care for that case?

We simpy don't care. IO completion based callbacks can already race with shutdown driven callbacks. RIght now, both cases will process all iclogs, so it just depends on which one gets to the iclog first as to which one runs the callbacks. We don't actually pass the shutdown state to the callbacks, so the callbacks are none-the-wiser for whether they are being called from shutdown or IO completion when they see the shutdown state.

With the code as per this patch, a racing shutdown will result in the callbacks from IO completion seeing the shutdown state, but IO completion will now avoid processing iclogs out of order/statei because the shutdown state is set. Hence by taking out the shutdown check from IO completion, we avoid having IO completion based callbacks racing with referenced iclogs that have just attached callbacks to the iclog but haven't yet released their reference and submitted the iclog for IO...

IOWs, it's not just the callbacks running from shutdown that trigger the UAF problems with callbacks, it can also occur from IO completion, too. Hence we really need to separate out the shutdown case from the IO completion path to avoid it from having the same problem(s) as the shutdown path...

Cheers,

Dave.

Signature · 2 lines
Dave Chinner
[email protected]
#30 lore raw

Re: [PATCH 8/9] xfs: don't run shutdown callbacks on active iclogs

In reply to Christoph Hellwig
Quoted 14 lines · Christoph Hellwig
Quoted 5 lines · Dave Chinner

The problem is that xlog_state_release_iclog() aborts before doing anything if the log is already shut down. It assumes that the callbacks have already been cleaned up, and it doesn't need to do any cleanup.

Quoted 7 lines

Hence the fix is to remove the xlog_is_shutdown() check from xlog_state_release_iclog() so that reference counts are correctly released from the iclogs, and when the reference count is zero we always transition to SYNCING if the log is shut down. Hence we'll always enter the xlog_sync() path in a shutdown and eventually end up erroring out the iclog IO and running xlog_state_do_callback() to process the callbacks attached to the iclog.

Quoted 2 lines

Ok, this answers my question to the previous patch. Maybe add a little blurb there?

Done.

Quoted 15 lines
Quoted 15 lines
fs/xfs/xfs_log.c +10 -0
+	if (xlog_is_shutdown(log)) {
+		/*
+		 * No more references to this iclog, so process the pending
+		 * iclog callbacks that were waiting on the release of this
+		 * iclog.
+		 */
+		spin_unlock(&log->l_icloglock);
+		xlog_state_shutdown_callbacks(log);
+		spin_lock(&log->l_icloglock);
+	} else if (xlog_state_want_sync(log, iclog)) {
 		spin_unlock(&log->l_icloglock);
 		xlog_sync(log, iclog);
 		spin_lock(&log->l_icloglock);
Quoted 8 lines

fs/xfs/xfs_log.c __xlog_state_release_iclog( · line 589

Quoted 4 lines
fs/xfs/xfs_log.c +3 -0
+out_check_shutdown:
+	if (xlog_is_shutdown(log))
+		return -EIO;
 	return 0;

Nit: we can just return -EIO directly in the first xlog_is_shutdown block.. It's not going to make any difference for the CPU, but makes the code a little easier to follow.

Fixed.

Quoted 1 line

Thanks.

-Dave.

Signature · 2 lines
Dave Chinner
[email protected]
#33 lore raw

Re: [PATCH 9/9] xfs: log head and tail aren't reliable during shutdown

In reply to Christoph Hellwig
Quoted 14 lines · Christoph Hellwig

fs/xfs/xfs_log.c xlog_space_left( · line 1302

Quoted 11 lines
fs/xfs/xfs_log.c +8 -0
+	if (tail_cycle == head_cycle && head_bytes >= tail_bytes) {
+		return log->l_logsize - (head_bytes - tail_bytes);
+	} else if (tail_cycle + 1 < head_cycle) {
 		return 0;
+	} else if (xlog_is_shutdown(log)) {
+		/* Ignore potential inconsistency when shutdown. */
+		return log->l_logsize;
+	} else if (tail_cycle < head_cycle) {
 		ASSERT(tail_cycle == (head_cycle - 1));
+		return tail_bytes - head_bytes;
 	}

Drop the else after the returns to make this a little easier to follow:

Quoted 13 lines
	if (tail_cycle == head_cycle && head_bytes >= tail_bytes)
		return log->l_logsize - (head_bytes - tail_bytes);
	if (tail_cycle + 1 < head_cycle)
		return 0;

	/* Ignore potential inconsistency when shutdown. */
	if (xlog_is_shutdown(log)) {
		return log->l_logsize;

	if (tail_cycle < head_cycle) {
		ASSERT(tail_cycle == (head_cycle - 1));
		return tail_bytes - head_bytes;
	}

Yup, that's better. I'll fix it.

-Dave.

Signature · 2 lines
Dave Chinner
[email protected]
Reviewed-by
#15 lore raw

Re: [PATCH 4/9] xfs: convert log flags to an operational state field

In reply to Christoph Hellwig
Quoted 12 lines · Christoph Hellwig
Quoted 12 lines
fs/xfs/xfs_log.c +1 -0
@@ -552,6 +552,7 @@ xfs_log_mount(
 	xfs_daddr_t	blk_offset,
 
 {
+	struct xlog	*log;
 	bool		fatal = xfs_sb_version_hascrc(&mp->m_sb);
 	int		error = 0;
 	int		min_logfsbs;
@@ -566,11 +567,12 @@ xfs_log_mount(
 		ASSERT(mp->m_flags & XFS_MOUNT_RDONLY);
 	}
 
Quoted 12 lines

fs/xfs/xfs_log.c xfs_log_mount( · line 575

Quoted 9 lines
fs/xfs/xfs_log.c +4 -3
-	mp->m_log = xlog_alloc_log(mp, log_target, blk_offset, num_bblks);
-	if (IS_ERR(mp->m_log)) {
-		error = PTR_ERR(mp->m_log);
+	log = xlog_alloc_log(mp, log_target, blk_offset, num_bblks);
+	if (IS_ERR(log)) {
+		error = PTR_ERR(log);
 		goto out;
 	}
+	mp->m_log = log;

Additition of the local variable here looks rather unrelated, given that the log is only touched twice in relation to the flags.

Same comment as Christoph -- if you wanted to split the variable cutout into a separate patch, that's fine. Don't really care that much though.

--D

Quoted 4 lines

Otherwise looks good:

#19 lore raw

Re: [PATCH 5/9] xfs: make forced shutdown processing atomic

In reply to Dave Chinner
Quoted 8 lines · Dave Chinner

From: Dave Chinner <[email protected]>

The running of a forced shutdown is a bit of a mess. It does racy checks for XFS_MOUNT_SHUTDOWN in xfs_do_force_shutdown(), then does more racy checks in xfs_log_force_unmount() before finally setting XFS_MOUNT_SHUTDOWN and XLOG_IO_ERROR under the log->icloglock.

Quoted 4 lines

Move the checking and setting of XFS_MOUNT_SHUTDOWN into xfs_do_force_shutdown() so we only process a shutdown once and once only. Serialise this with the mp->m_sb_lock spinlock so that the state change is atomic and won't race. Move all the mount specific

Assuming you're working on cleaning /that/ up too, I'll let that go...

Quoted 4 lines

shutdown state changes from xfs_log_force_unmount() to xfs_do_force_shutdown() so they are done atomically with setting XFS_MOUNT_SHUTDOWN.

Quoted 8 lines

Then get rid of the racy xlog_is_shutdown() check from xlog_force_shutdown(), and gate the log shutdown on the test_and_set_bit(XLOG_IO_ERROR) test under the icloglock. This means that the log is shutdown once and once only, and code that needs to prevent races with shutdown can do so by holding the icloglock and checking the return value of xlog_is_shutdown().

This results in a predicable shutdown execution process - we set the

s/predicable/predictable/

Quoted 88 lines

shutdown flags once and process the shutdown once rather than the current "as many concurrent shutdowns as can race to the flag setting" situation we have now.

---
 fs/xfs/xfs_fsops.c |  64 ++++++++++++++---------------
 fs/xfs/xfs_log.c   | 100 ++++++++++++++++++++-------------------------
 fs/xfs/xfs_log.h   |   2 +-
 3 files changed, 77 insertions(+), 89 deletions(-)
fs/xfs/xfs_fsops.c +24 -33
index 6ed29b158312..caca391ce1a9 100644
@@ -511,6 +511,11 @@ xfs_fs_goingdown(
  * consistent. We don't do an unmount here; just shutdown the shop, make sure
  * that absolutely nothing persistent happens to this filesystem after this
  * point.
+ *
+ * The shutdown state change is atomic, resulting in the first and only the
+ * first shutdown call processing the shutdown. This means we only shutdown the
+ * log once as it requires, and we don't spam the logs when multiple concurrent
+ * shutdowns race to set the shutdown flags.
  */
 void
 xfs_do_force_shutdown(
@@ -519,48 +524,41 @@ xfs_do_force_shutdown(
 	char		*fname,
 	int		lnnum)
 {
-	bool		logerror = flags & SHUTDOWN_LOG_IO_ERROR;
-
-	/*
-	 * No need to duplicate efforts.
-	 */
-	if (XFS_FORCED_SHUTDOWN(mp) && !logerror)
-		return;
-
-	/*
-	 * This flags XFS_MOUNT_FS_SHUTDOWN, makes sure that we don't
-	 * queue up anybody new on the log reservations, and wakes up
-	 * everybody who's sleeping on log reservations to tell them
-	 * the bad news.
-	 */
-	if (xfs_log_force_umount(mp, logerror))
-		return;
+	int		tag;
+	const char	*why;
 
-	if (flags & SHUTDOWN_FORCE_UMOUNT) {
-		xfs_alert(mp,
-"User initiated shutdown (0x%x) received. Shutting down filesystem",
-				flags);
+	spin_lock(&mp->m_sb_lock);
+	if (XFS_FORCED_SHUTDOWN(mp)) {
+		spin_unlock(&mp->m_sb_lock);
 		return;
 	}
+	mp->m_flags |= XFS_MOUNT_FS_SHUTDOWN;
+	if (mp->m_sb_bp)
+		mp->m_sb_bp->b_flags |= XBF_DONE;
+	spin_unlock(&mp->m_sb_lock);
+
+	if (flags & SHUTDOWN_FORCE_UMOUNT)
+		xfs_alert(mp, "User initiated shutdown received.");
 
-	if (flags & SHUTDOWN_CORRUPT_INCORE) {
-		xfs_alert_tag(mp, XFS_PTAG_SHUTDOWN_CORRUPT,
-"Corruption of in-memory data (0x%x) detected at %pS (%s:%d).  Shutting down filesystem",
-				flags, __return_address, fname, lnnum);
-		if (XFS_ERRLEVEL_HIGH <= xfs_error_level)
-			xfs_stack_trace();
-	} else if (logerror) {
-		xfs_alert_tag(mp, XFS_PTAG_SHUTDOWN_LOGERROR,
-"Log I/O error (0x%x) detected at %pS (%s:%d). Shutting down filesystem",
-				flags, __return_address, fname, lnnum);
+	if (xlog_force_shutdown(mp->m_log, flags)) {
+		tag = XFS_PTAG_SHUTDOWN_LOGERROR;
+		why = "Log I/O Error";
+	} else if (flags & SHUTDOWN_CORRUPT_INCORE) {
+		tag = XFS_PTAG_SHUTDOWN_CORRUPT;
+		why = "Corruption of in-memory data";
 	} else {
-		xfs_alert_tag(mp, XFS_PTAG_SHUTDOWN_IOERROR,
-"I/O error (0x%x) detected at %pS (%s:%d). Shutting down filesystem",
-				flags, __return_address, fname, lnnum);
+		tag = XFS_PTAG_SHUTDOWN_IOERROR;

fs/xfs/xfs_fsops.c xfs_do_force_shutdown( · line 560

Quoted 10 lines
fs/xfs/xfs_fsops.c +6 -0
+		why = "Metadata I/O Error";
 	}
 
+	xfs_alert_tag(mp, tag,
+"%s (0x%x) detected at %pS (%s:%d).  Shutting down filesystem.",
+			why, flags, __return_address, fname, lnnum);
 	xfs_alert(mp,
 		"Please unmount the filesystem and rectify the problem(s)");
+	if (xfs_error_level >= XFS_ERRLEVEL_HIGH)
+		xfs_stack_trace();

Doesn't xfs_alert already drop a stack trace for xfs_error_level >= XFS_ERRLEVEL_HIGH ?

--D

Quoted 160 lines
fs/xfs/xfs_fsops.c +1 -0
+
 }
 
 /*
fs/xfs/xfs_log.c +45 -55
index 049d6ac9cb4d..6c7cfc052135 100644
@@ -3673,76 +3673,66 @@ xlog_verify_iclog(
 #endif
 
 /*
- * This is called from xfs_force_shutdown, when we're forcibly
- * shutting down the filesystem, typically because of an IO error.
- * Our main objectives here are to make sure that:
- *	a. if !logerror, flush the logs to disk. Anything modified
- *	   after this is ignored.
- *	b. the filesystem gets marked 'SHUTDOWN' for all interested
- *	   parties to find out, 'atomically'.
- *	c. those who're sleeping on log reservations, pinned objects and
- *	    other resources get woken up, and be told the bad news.
- *	d. nothing new gets queued up after (b) and (c) are done.
+ * Perform a forced shutdown on the log. This should be called once and once
+ * only by the high level filesystem shutdown code to shut the log subsystem
+ * down cleanly.
  *
- * Note: for the !logerror case we need to flush the regions held in memory out
- * to disk first. This needs to be done before the log is marked as shutdown,
- * otherwise the iclog writes will fail.
+ * Our main objectives here are to make sure that:
+ *	a. if the shutdown was not due to a log IO error, flush the logs to
+ *	   disk. Anything modified after this is ignored.
+ *	b. the log gets atomically marked 'XLOG_IO_ERROR' for all interested
+ *	   parties to find out. Nothing new gets queued after this is done.
+ *	c. Tasks sleeping on log reservations, pinned objects and
+ *	   other resources get woken up.
  *
- * Return non-zero if log shutdown transition had already happened.
+ * Return true if the shutdown cause was a log IO error and we actually shut the
+ * log down.
  */
-int
-xfs_log_force_umount(
-	struct xfs_mount	*mp,
-	int			logerror)
+bool
+xlog_force_shutdown(
+	struct xlog	*log,
+	int		shutdown_flags)
 {
-	struct xlog	*log;
-	int		retval = 0;
-
-	log = mp->m_log;
+	bool		log_error = (shutdown_flags & SHUTDOWN_LOG_IO_ERROR);
 
 	/*
-	 * If this happens during log recovery, don't worry about
-	 * locking; the log isn't open for business yet.
+	 * If this happens during log recovery then we aren't using the runtime
+	 * log mechanisms yet so there's nothing to shut down.
 	 */
-	if (!log || xlog_in_recovery(log)) {
-		mp->m_flags |= XFS_MOUNT_FS_SHUTDOWN;
-		if (mp->m_sb_bp)
-			mp->m_sb_bp->b_flags |= XBF_DONE;
-		return 0;
-	}
+	if (!log || xlog_in_recovery(log))
+		return false;
 
-	/*
-	 * Somebody could've already done the hard work for us.
-	 * No need to get locks for this.
-	 */
-	if (logerror && xlog_is_shutdown(log))
-		return 1;
+	ASSERT(!xlog_is_shutdown(log));
 
 	/*
 	 * Flush all the completed transactions to disk before marking the log
-	 * being shut down. We need to do it in this order to ensure that
-	 * completed operations are safely on disk before we shut down, and that
-	 * we don't have to issue any buffer IO after the shutdown flags are set
-	 * to guarantee this.
+	 * being shut down. We need to do this first as shutting down the log
+	 * before the force will prevent the log force from flushing the iclogs
+	 * to disk.
+	 *
+	 * Re-entry due to a log IO error shutdown during the log force is
+	 * prevented by the atomicity of higher level shutdown code.
 	 */
-	if (!logerror)
-		xfs_log_force(mp, XFS_LOG_SYNC);
+	if (!log_error)
+		xfs_log_force(log->l_mp, XFS_LOG_SYNC);
 
 	/*
-	 * mark the filesystem and the as in a shutdown state and wake
-	 * everybody up to tell them the bad news.
+	 * Atomically set the shutdown state. If the shutdown state is already
+	 * set, there someone else is performing the shutdown and so we are done
+	 * here. This should never happen because we should only ever get called
+	 * once by the first shutdown caller.
+	 *
+	 * Much of the log state machine transitions assume that shutdown state
+	 * cannot change once they hold the log->l_icloglock. Hence we need to
+	 * hold that lock here, even though we use the atomic test_and_set_bit()
+	 * operation to set the shutdown state.
 	 */
 	spin_lock(&log->l_icloglock);
-	mp->m_flags |= XFS_MOUNT_FS_SHUTDOWN;
-	if (mp->m_sb_bp)
-		mp->m_sb_bp->b_flags |= XBF_DONE;
-
-	/*
-	 * Mark the log and the iclogs with IO error flags to prevent any
-	 * further log IO from being issued or completed.
-	 */
-	if (!test_and_set_bit(XLOG_IO_ERROR, &log->l_opstate))
-		retval = 1;
+	if (test_and_set_bit(XLOG_IO_ERROR, &log->l_opstate)) {
+		spin_unlock(&log->l_icloglock);
+		ASSERT(0);
+		return false;
+	}
 	spin_unlock(&log->l_icloglock);
 
 	/*
@@ -3766,7 +3756,7 @@ xfs_log_force_umount(
 	spin_unlock(&log->l_cilp->xc_push_lock);
 	xlog_state_do_callback(log);
 
-	return retval;
+	return log_error;
 }
 
 STATIC int
fs/xfs/xfs_log.h +1 -1
index 4c5c8a7db1d9..3f680f0c9744 100644
@@ -125,7 +125,6 @@ int	  xfs_log_reserve(struct xfs_mount *mp,
 			  bool		   permanent);
 int	  xfs_log_regrant(struct xfs_mount *mp, struct xlog_ticket *tic);
 void      xfs_log_unmount(struct xfs_mount *mp);
-int	  xfs_log_force_umount(struct xfs_mount *mp, int logerror);
 bool	xfs_log_writable(struct xfs_mount *mp);
 
 struct xlog_ticket *xfs_log_ticket_get(struct xlog_ticket *ticket);
@@ -140,5 +139,6 @@ void	xfs_log_clean(struct xfs_mount *mp);
 bool	xfs_log_check_lsn(struct xfs_mount *, xfs_lsn_t);
 
 xfs_lsn_t xlog_grant_push_threshold(struct xlog *log, int need_bytes);
+bool	  xlog_force_shutdown(struct xlog *log, int shutdown_flags);
 
 #endif	/* __XFS_LOG_H__ */
Signature · 1 line
2.31.1
Reviewed-by on v1 6/9
#24 lore raw

Re: [PATCH 6/9] xfs: reowrk up xlog_state_do_callback

In reply to Dave Chinner
Quoted 47 lines · Dave Chinner

From: Dave Chinner <[email protected]>

Clean it up a bit by factoring and rearranging some of the code.

---
 fs/xfs/xfs_log.c | 99 ++++++++++++++++++++++++++----------------------
 1 file changed, 53 insertions(+), 46 deletions(-)
fs/xfs/xfs_log.c +30 -0
index 6c7cfc052135..bb44dcfcae89 100644
@@ -2758,67 +2758,74 @@ xlog_state_iodone_process_iclog(
 	}
 }
 
+/*
+ * Loop over all the iclogs, running attached callbacks on them. Return true if
+ * we ran any callbacks, indicating that we dropped the icloglock.
+ */
+static bool
+xlog_state_do_iclog_callbacks(
+	struct xlog		*log)
+		__releases(&log->l_icloglock)
+		__acquires(&log->l_icloglock)
+{
+	struct xlog_in_core	*iclog = log->l_iclog;
+	struct xlog_in_core	*first_iclog = NULL;
+	bool			ran_callback = false;
+
+	for (; iclog != first_iclog; iclog = iclog->ic_next) {
+		LIST_HEAD(cb_list);
+
+		if (!first_iclog)
+			first_iclog = iclog;
+
+		if (!xlog_is_shutdown(log)) {
+			if (xlog_state_iodone_process_iclog(log, iclog))
+				break;
+			if (iclog->ic_state != XLOG_STATE_CALLBACK)
+				continue;
+		}
+		list_splice_init(&iclog->ic_callbacks, &cb_list);
+		spin_unlock(&log->l_icloglock);
+
+		trace_xlog_iclog_callbacks_start(iclog, _RET_IP_);

fs/xfs/xfs_log.c xlog_state_iodone_process_iclog( · line 2800

Quoted 10 lines
fs/xfs/xfs_log.c +10 -0
+		xlog_cil_process_committed(&cb_list);
+		trace_xlog_iclog_callbacks_done(iclog, _RET_IP_);
+		ran_callback = true;
+
+		spin_lock(&log->l_icloglock);
+		if (xlog_is_shutdown(log))
+			wake_up_all(&iclog->ic_force_wait);
+		else
+			xlog_state_clean_iclog(log, iclog);
+	};

Aside from the unnecessary semicolon here...

Quoted 64 lines
fs/xfs/xfs_log.c +11 -44
+	return ran_callback;
+}
+
+
+/*
+ * Loop running iclog completion callbacks until there are no more iclogs in a
+ * state that can run callbacks.
+ */
 STATIC void
 xlog_state_do_callback(
 	struct xlog		*log)
 {
-	struct xlog_in_core	*iclog;
-	struct xlog_in_core	*first_iclog;
-	bool			cycled_icloglock;
 	int			flushcnt = 0;
 	int			repeats = 0;
 
 	spin_lock(&log->l_icloglock);
-	do {
-		repeats++;
+	while (xlog_state_do_iclog_callbacks(log)) {
+		if (xlog_is_shutdown(log))
+			break;
 
-		/*
-		 * Scan all iclogs starting with the one pointed to by the
-		 * log.  Reset this starting point each time the log is
-		 * unlocked (during callbacks).
-		 *
-		 * Keep looping through iclogs until one full pass is made
-		 * without running any callbacks.
-		 */
-		cycled_icloglock = false;
-		first_iclog = NULL;
-		for (iclog = log->l_iclog;
-		     iclog != first_iclog;
-		     iclog = iclog->ic_next) {
-			LIST_HEAD(cb_list);
-
-			if (!first_iclog)
-				first_iclog = iclog;
-
-			if (!xlog_is_shutdown(log)) {
-				if (xlog_state_iodone_process_iclog(log, iclog))
-					break;
-				if (iclog->ic_state != XLOG_STATE_CALLBACK)
-					continue;
-			}
-			list_splice_init(&iclog->ic_callbacks, &cb_list);
-			spin_unlock(&log->l_icloglock);
-
-			trace_xlog_iclog_callbacks_start(iclog, _RET_IP_);
-			xlog_cil_process_committed(&cb_list);
-			trace_xlog_iclog_callbacks_done(iclog, _RET_IP_);
-			cycled_icloglock = true;
-
-			spin_lock(&log->l_icloglock);
-			if (xlog_is_shutdown(log))
-				wake_up_all(&iclog->ic_force_wait);
-			else
-				xlog_state_clean_iclog(log, iclog);
-		};
-

fs/xfs/xfs_log.c xlog_state_iodone_process_iclog( · line 2828

Quoted 10 lines
fs/xfs/xfs_log.c +2 -2
-		if (repeats > 5000) {
+		if (repeats++ > 5000) {
 			flushcnt += repeats;
 			repeats = 0;
 			xfs_warn(log->l_mp,
 				"%s: possible infinite loop (%d iterations)",
 				__func__, flushcnt);
 		}
-	} while (!xlog_is_shutdown(log) && cycled_icloglock);
+	};

...and here, this looks like a simple hoist.

With that fixed,

--D

Quoted 6 lines
fs/xfs/xfs_log.c +0 -0
 	if (log->l_iclog->ic_state == XLOG_STATE_ACTIVE ||
 	    xlog_is_shutdown(log))
Signature · 1 line
2.31.1
#20 lore raw

Re: [PATCH 5/9] xfs: make forced shutdown processing atomic

In reply to Darrick J. Wong
Quoted 9 lines · Darrick J. Wong
Quoted 8 lines · Dave Chinner

From: Dave Chinner <[email protected]>

The running of a forced shutdown is a bit of a mess. It does racy checks for XFS_MOUNT_SHUTDOWN in xfs_do_force_shutdown(), then does more racy checks in xfs_log_force_unmount() before finally setting XFS_MOUNT_SHUTDOWN and XLOG_IO_ERROR under the log->icloglock.

Quoted 6 lines
Quoted 4 lines

Move the checking and setting of XFS_MOUNT_SHUTDOWN into xfs_do_force_shutdown() so we only process a shutdown once and once only. Serialise this with the mp->m_sb_lock spinlock so that the state change is atomic and won't race. Move all the mount specific

Assuming you're working on cleaning /that/ up too, I'll let that go...

Yes, a forward ported patch set that does this will be posted soon.

Quoted 10 lines

fs/xfs/xfs_fsops.c xfs_do_force_shutdown( · line 560

Quoted 7 lines
fs/xfs/xfs_fsops.c +5 -0
+	xfs_alert_tag(mp, tag,
+"%s (0x%x) detected at %pS (%s:%d).  Shutting down filesystem.",
+			why, flags, __return_address, fname, lnnum);
 	xfs_alert(mp,
 		"Please unmount the filesystem and rectify the problem(s)");
+	if (xfs_error_level >= XFS_ERRLEVEL_HIGH)
+		xfs_stack_trace();

Doesn't xfs_alert already drop a stack trace for xfs_error_level >= XFS_ERRLEVEL_HIGH ?

It does? I've never seen it do that, and the existing code implies it doesn't do this, either, and that's the logic was looking at here:

        if (flags & SHUTDOWN_CORRUPT_INCORE) {
                xfs_alert_tag(mp, XFS_PTAG_SHUTDOWN_CORRUPT,
"Corruption of in-memory data (0x%x) detected at %pS (%s:%d).  Shutting down filesystem",
                                flags, __return_address, fname, lnnum);
                if (XFS_ERRLEVEL_HIGH <= xfs_error_level)
                        xfs_stack_trace();
	} else if (....)

Yes, xfs_alert_tag() does not trigger a stack trace at all, but there's an unconditional xfs_alert() call after this so if that issues stack traces then we'd get a double stack trace on all SHUTDOWN_CORRUPT_INCORE incidents. AFAICT, that doesn't actually happen....

This pattern is repeated in several places - look at xfs_inode_verifier_error(), xfs_buf_verifier_error(), and xfs_buf_corruption_error(). They all have xfs_alert() calls, then follow it up with a specific error level check for a stack dump.

Hmmm, it looks like xfs_alert() was intended to dump stacks, but I don't think it works:

        if (!kstrtoint(kern_level, 0, &level) &&                \
            level <= LOGLEVEL_ERR &&                            \
            xfs_error_level >= XFS_ERRLEVEL_HIGH)               \
                xfs_stack_trace();                              \

And kern_level is KERN_ALERT, which is:

#define KERN_SOH "\001"
....
#define KERN_ALERT KERN_SOH "1"

And:

#define LOGLEVEL_ERR 3 /* error conditions */

So what does kstrtoint() return when passed the string "\0011"? It's not actually an integer string...

Hmmm, I think it returns -EINVAL, which means it then uses level uninitialised, and the result is .... unpredictable it is likely no stack trace is emitted....

Fixing this mess is out of scope for this patchset. The changes in this patchset don't change the existing pattern of the function of unconditionally calling xfs_alert() and conditionally and explicitly dumping stack traces manually. I'll add it to my ever growing list of cleanups that need to be done...

Cheers,

Dave.

Signature · 2 lines
Dave Chinner
[email protected]
#21 lore raw

Re: [PATCH 5/9] xfs: make forced shutdown processing atomic

Quoted 10 lines · Dave Chinner
Quoted 9 lines · Darrick J. Wong
Quoted 8 lines · Dave Chinner

From: Dave Chinner <[email protected]>

The running of a forced shutdown is a bit of a mess. It does racy checks for XFS_MOUNT_SHUTDOWN in xfs_do_force_shutdown(), then does more racy checks in xfs_log_force_unmount() before finally setting XFS_MOUNT_SHUTDOWN and XLOG_IO_ERROR under the log->icloglock.

Quoted 8 lines
Quoted 6 lines
Quoted 4 lines

Move the checking and setting of XFS_MOUNT_SHUTDOWN into xfs_do_force_shutdown() so we only process a shutdown once and once only. Serialise this with the mp->m_sb_lock spinlock so that the state change is atomic and won't race. Move all the mount specific

Assuming you're working on cleaning /that/ up too, I'll let that go...

Yes, a forward ported patch set that does this will be posted soon.

Ok.

Quoted 59 lines
Quoted 10 lines

fs/xfs/xfs_fsops.c xfs_do_force_shutdown( · line 560

Quoted 7 lines
fs/xfs/xfs_fsops.c +5 -0
+	xfs_alert_tag(mp, tag,
+"%s (0x%x) detected at %pS (%s:%d).  Shutting down filesystem.",
+			why, flags, __return_address, fname, lnnum);
 	xfs_alert(mp,
 		"Please unmount the filesystem and rectify the problem(s)");
+	if (xfs_error_level >= XFS_ERRLEVEL_HIGH)
+		xfs_stack_trace();

Doesn't xfs_alert already drop a stack trace for xfs_error_level >= XFS_ERRLEVEL_HIGH ?

It does? I've never seen it do that, and the existing code implies it doesn't do this, either, and that's the logic was looking at here:

        if (flags & SHUTDOWN_CORRUPT_INCORE) {
                xfs_alert_tag(mp, XFS_PTAG_SHUTDOWN_CORRUPT,
"Corruption of in-memory data (0x%x) detected at %pS (%s:%d).  Shutting down filesystem",
                                flags, __return_address, fname, lnnum);
                if (XFS_ERRLEVEL_HIGH <= xfs_error_level)
                        xfs_stack_trace();
	} else if (....)

Yes, xfs_alert_tag() does not trigger a stack trace at all, but there's an unconditional xfs_alert() call after this so if that issues stack traces then we'd get a double stack trace on all SHUTDOWN_CORRUPT_INCORE incidents. AFAICT, that doesn't actually happen....

This pattern is repeated in several places - look at xfs_inode_verifier_error(), xfs_buf_verifier_error(), and xfs_buf_corruption_error(). They all have xfs_alert() calls, then follow it up with a specific error level check for a stack dump.

Hmmm, it looks like xfs_alert() was intended to dump stacks, but I don't think it works:

        if (!kstrtoint(kern_level, 0, &level) &&                \
            level <= LOGLEVEL_ERR &&                            \
            xfs_error_level >= XFS_ERRLEVEL_HIGH)               \
                xfs_stack_trace();                              \

And kern_level is KERN_ALERT, which is:

#define KERN_SOH "\001"
....
#define KERN_ALERT KERN_SOH "1"

And:

#define LOGLEVEL_ERR 3 /* error conditions */

So what does kstrtoint() return when passed the string "\0011"? It's not actually an integer string...

Hmmm, I think it returns -EINVAL, which means it then uses level uninitialised, and the result is .... unpredictable it is likely no stack trace is emitted....

Quoted 5 lines

Fixing this mess is out of scope for this patchset. The changes in this patchset don't change the existing pattern of the function of unconditionally calling xfs_alert() and conditionally and explicitly dumping stack traces manually. I'll add it to my ever growing list of cleanups that need to be done...

AHA! That's why that's never seemed to work right for me.

Well, at least the good news is that we each have enough outstanding patchsets to keep the other busy reviewing until 2028 or so. ;)

--D

Quoted 6 lines

Cheers,

Dave.

Signature · 2 lines
Dave Chinner
[email protected]