Skip to contentSkip to conversation

xfs: prevent NMI timeouts in cmn_err

PATCH In review Dave Chinner 2 participants 4 messages Dec 3, 2010 – Dec 13, 2010 XFS filesystem

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 xfs: prevent NMI timeouts in cmn_err

From: Dave Chinner <[email protected]>

We currently have a global error message buffer in cmn_err that is protected by a spin lock that disables interrupts. Recently there have been reports of NMI timeouts occurring when the console is being flooded by SCSI error reports due to cmn_err() getting stuck trying to print to the console while holding this lock (i.e. with interrupts disabled). The NMI watchdog is seeing this CPU as non-responding and so is triggering a panic. While the trigger for the reported case is SCSI errors, pretty much anything that spams the kernel log could cause this to occur.

Realistically the only reason that we have the intemediate message buffer is to prepend the correct kernel log level prefix to the log message. The only reason we have the lock is to protect the global message buffer and the only reason the message buffer is global is to keep it off the stack. Hence if we can avoid needing a global message buffer we avoid needing the lock, and we can do this with a small amount of cleanup and some preprocessor tricks:

1. clean up xfs_cmn_err() panic mask functionality to avoid needing debug code in xfs_cmn_err()
2. remove the couple of "!" message prefixes that still exist that the existing cmn_err() code steps over.
3. redefine CE_* levels directly to KERN_*
4. redefine cmn_err() and friends to use printk() directly via variable argument length macros.

By doing this, we can completely remove the cmn_err() code and the lock that is causing the problems, and rely solely on printk() serialisation to ensure that we don't get garbled messages.

A series of followup patches is really needed to clean up all the cmn_err() calls and related messages properly, but that results in a series that is not easily back portable to enterprise kernels. Hence this initial fix is only to address the direct problem in the lowest impact way possible.

---
 fs/xfs/linux-2.6/xfs_sysctl.c |   22 +++++++++++-
 fs/xfs/support/debug.c        |   76 -----------------------------------------
 fs/xfs/support/debug.h        |   40 +++++++++++++++++-----
 fs/xfs/xfs_error.c            |   31 -----------------
 fs/xfs/xfs_error.h            |   14 ++-----
 fs/xfs/xfs_log.c              |    2 +-
 fs/xfs/xfs_log_recover.c      |    2 +-
 7 files changed, 58 insertions(+), 129 deletions(-)
fs/xfs/linux-2.6/xfs_sysctl.c +21 -1
index 7bb5092..768acd8 100644
@@ -51,6 +51,26 @@ xfs_stats_clear_proc_handler(
 
 	return ret;
 }
+
+STATIC int
+xfs_panic_mask_proc_handler(
+	ctl_table	*ctl,
+	int		write,
+	void		__user *buffer,
+	size_t		*lenp,
+	loff_t		*ppos)
+{
+	int		ret, *valp = ctl->data;
+
+	ret = proc_dointvec_minmax(ctl, write, buffer, lenp, ppos);
+	if (!ret && write) {
+		xfs_panic_mask = *valp;
+#ifdef DEBUG
+		xfs_panic_mask |= (XFS_PTAG_SHUTDOWN_CORRUPT | XFS_PTAG_LOGRES);
+#endif
+	}
+	return ret;
+}
 #endif /* CONFIG_PROC_FS */
 
 static ctl_table xfs_table[] = {
@@ -77,7 +97,7 @@ static ctl_table xfs_table[] = {
 		.data		= &xfs_params.panic_mask.val,
 		.maxlen		= sizeof(int),
 		.mode		= 0644,
-		.proc_handler	= proc_dointvec_minmax,
+		.proc_handler	= xfs_panic_mask_proc_handler,
 		.extra1		= &xfs_params.panic_mask.min,
 		.extra2		= &xfs_params.panic_mask.max
 	},
fs/xfs/support/debug.c +0 -76

82 lines not on this page · raw message

fs/xfs/support/debug.h +31 -9
index d2d2046..f659bd0 100644
@@ -20,15 +20,37 @@
 
 #include <stdarg.h>
 
-#define CE_DEBUG        7               /* debug        */
-#define CE_CONT         6               /* continuation */
-#define CE_NOTE         5               /* notice       */
-#define CE_WARN         4               /* warning      */
-#define CE_ALERT        1               /* alert        */
-#define CE_PANIC        0               /* panic        */
-
-extern void cmn_err(int, char *, ...)
-	__attribute__ ((format (printf, 2, 3)));
+#define CE_DEBUG        KERN_DEBUG
+#define CE_CONT         KERN_INFO
+#define CE_NOTE         KERN_NOTICE
+#define CE_WARN         KERN_WARNING
+#define CE_ALERT        KERN_ALERT
+#define CE_PANIC        KERN_EMERG
+
+#define cmn_err(lvl, fmt, args...)	\
+	do { \
+		printk(lvl fmt, ## args); \
+		BUG_ON(strncmp(lvl, KERN_EMERG, strlen(KERN_EMERG)) == 0); \
+	} while (0)
+
+#define xfs_fs_cmn_err(lvl, mp, fmt, args...)	\
+	do { \
+		printk(lvl "Filesystem %s: " fmt, (mp)->m_fsname, ## args); \
+		BUG_ON(strncmp(lvl, KERN_EMERG, strlen(KERN_EMERG)) == 0); \
+	} while (0)
+
+/* All callers to xfs_cmn_err use CE_ALERT, so don't bother testing lvl */
+#define xfs_cmn_err(panic_tag, lvl, mp, fmt, args...)	\
+	do { \
+		int	panic = 0; \
+		if (xfs_panic_mask && (xfs_panic_mask & panic_tag)) { \
+			printk(KERN_ALERT "XFS: Transforming an alert into a BUG."); \
+			panic = 1; \
+		} \
+		printk(KERN_ALERT "Filesystem %s: " fmt, (mp)->m_fsname, ## args); \
+		BUG_ON(panic); \
+	} while (0)
+
 extern void assfail(char *expr, char *f, int l);
 
 #define ASSERT_ALWAYS(expr)	\
fs/xfs/xfs_error.c +0 -31
index c78cc6a..4c7db74 100644
@@ -152,37 +152,6 @@ xfs_errortag_clearall(xfs_mount_t *mp, int loud)
 }
 #endif /* DEBUG */
 
-
-void
-xfs_fs_cmn_err(int level, xfs_mount_t *mp, char *fmt, ...)
-{
-	va_list ap;
-
-	va_start(ap, fmt);
-	xfs_fs_vcmn_err(level, mp, fmt, ap);
-	va_end(ap);
-}
-
-void
-xfs_cmn_err(int panic_tag, int level, xfs_mount_t *mp, char *fmt, ...)
-{
-	va_list ap;
-
-#ifdef DEBUG
-	xfs_panic_mask |= (XFS_PTAG_SHUTDOWN_CORRUPT | XFS_PTAG_LOGRES);
-#endif
-
-	if (xfs_panic_mask && (xfs_panic_mask & panic_tag)
-	    && (level & CE_ALERT)) {
-		level &= ~CE_ALERT;
-		level |= CE_PANIC;
-		cmn_err(CE_ALERT, "XFS: Transforming an alert into a BUG.");
-	}
-	va_start(ap, fmt);
-	xfs_fs_vcmn_err(level, mp, fmt, ap);
-	va_end(ap);
-}
-
 void
 xfs_error_report(
 	const char		*tag,
fs/xfs/xfs_error.h +4 -10
index f338847..6b518d2 100644
@@ -162,21 +162,15 @@ extern int xfs_errortag_clearall(xfs_mount_t *mp, int loud);
 
 struct xfs_mount;
 
-extern void xfs_fs_vcmn_err(int level, struct xfs_mount *mp,
-		char *fmt, va_list ap)
-	__attribute__ ((format (printf, 3, 0)));
-extern void xfs_cmn_err(int panic_tag, int level, struct xfs_mount *mp,
-			char *fmt, ...)
-	__attribute__ ((format (printf, 4, 5)));
-extern void xfs_fs_cmn_err(int level, struct xfs_mount *mp, char *fmt, ...)
-	__attribute__ ((format (printf, 3, 4)));
-
 extern void xfs_hex_dump(void *p, int length);
 
 #define xfs_fs_repair_cmn_err(level, mp, fmt, args...) \
 	xfs_fs_cmn_err(level, mp, fmt "  Unmount and run xfs_repair.", ## args)
 
 #define xfs_fs_mount_cmn_err(f, fmt, args...) \
-	((f & XFS_MFSI_QUIET)? (void)0 : cmn_err(CE_WARN, "XFS: " fmt, ## args))
+	do { \
+		if (!(f & XFS_MFSI_QUIET)) 	\
+			cmn_err(CE_WARN, "XFS: " fmt, ## args); \
+	} while (0)
 
 #endif	/* __XFS_ERROR_H__ */
fs/xfs/xfs_log.c +1 -1
index abdcbd0..2209035 100644
@@ -431,7 +431,7 @@ xfs_log_mount(
 		cmn_err(CE_NOTE, "XFS mounting filesystem %s", mp->m_fsname);
 	else {
 		cmn_err(CE_NOTE,
-			"!Mounting filesystem \"%s\" in no-recovery mode.  Filesystem will be inconsistent.",
+			"Mounting filesystem \"%s\" in no-recovery mode.  Filesystem will be inconsistent.",
 			mp->m_fsname);
 		ASSERT(mp->m_flags & XFS_MOUNT_RDONLY);
 	}
fs/xfs/xfs_log_recover.c +1 -1
index 6e7dfbb..411f3a9 100644
@@ -3934,7 +3934,7 @@ xlog_recover_finish(
 		log->l_flags &= ~XLOG_RECOVERY_NEEDED;
 	} else {
 		cmn_err(CE_DEBUG,
-			"!Ending clean XFS mount for filesystem: %s\n",
+			"Ending clean XFS mount for filesystem: %s\n",
 			log->l_mp->m_fsname);
 	}
 	return 0;
Signature · 6 lines
1.7.2.3

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
#1 lore raw
Quoted 32 lines · Dave Chinner

From: Dave Chinner <[email protected]>

We currently have a global error message buffer in cmn_err that is protected by a spin lock that disables interrupts. Recently there have been reports of NMI timeouts occurring when the console is being flooded by SCSI error reports due to cmn_err() getting stuck trying to print to the console while holding this lock (i.e. with interrupts disabled). The NMI watchdog is seeing this CPU as non-responding and so is triggering a panic. While the trigger for the reported case is SCSI errors, pretty much anything that spams the kernel log could cause this to occur.

Realistically the only reason that we have the intemediate message buffer is to prepend the correct kernel log level prefix to the log message. The only reason we have the lock is to protect the global message buffer and the only reason the message buffer is global is to keep it off the stack. Hence if we can avoid needing a global message buffer we avoid needing the lock, and we can do this with a small amount of cleanup and some preprocessor tricks:

1. clean up xfs_cmn_err() panic mask functionality to avoid needing debug code in xfs_cmn_err()
2. remove the couple of "!" message prefixes that still exist that the existing cmn_err() code steps over.
3. redefine CE_* levels directly to KERN_*
4. redefine cmn_err() and friends to use printk() directly via variable argument length macros.

By doing this, we can completely remove the cmn_err() code and the lock that is causing the problems, and rely solely on printk() serialisation to ensure that we don't get garbled messages.

Quoted 5 lines

A series of followup patches is really needed to clean up all the cmn_err() calls and related messages properly, but that results in a series that is not easily back portable to enterprise kernels. Hence this initial fix is only to address the direct problem in the lowest impact way possible.

FWIW, while these macros are the best way to make a simple backport is possible, I just discovered that mainline has a %pV format operator that allows an implementation like:

void
xfs_fs_cmn_err(

	const char              *lvl,
	struct xfs_mount        *mp,
	const char              *fmt,
	...)
{
	struct va_format        vaf;
	va_list                 args;

	va_start(args, fmt);
	vaf.fmt = fmt;
	vaf.va = &args;

	printk("%sFilesystem %s: %pV", lvl, mp->m_fsname, &vaf);
	va_end(args);

	BUG_ON(strncmp(lvl, KERN_EMERG, strlen(KERN_EMERG)) == 0);
}

Would this be a preferable method for replacing the existing implementations, or are the macros good enough as the first step of a mainline cleanup?

Cheers,,,

Dave.

Signature · 7 lines
Dave Chinner
[email protected]

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
#2 lore raw
Quoted 15 lines · Dave Chinner

FWIW, while these macros are the best way to make a simple backport is possible, I just discovered that mainline has a %pV format operator that allows an implementation like:

void
xfs_fs_cmn_err(

	const char              *lvl,
	struct xfs_mount        *mp,
	const char              *fmt,
	...)
{
	struct va_format        vaf;
	va_list                 args;

	va_start(args, fmt);
Quoted 8 lines
	vaf.fmt = fmt;
	vaf.va = &args;

	printk("%sFilesystem %s: %pV", lvl, mp->m_fsname, &vaf);
	va_end(args);

	BUG_ON(strncmp(lvl, KERN_EMERG, strlen(KERN_EMERG)) == 0);
}

With this we can also keep the existing integer-based CE_ values and do trivial array lookup. That also avoids having to do a strcmp for every message printed.

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs

#3 lore raw
Quoted 22 lines · Christoph Hellwig
Quoted 21 lines · Dave Chinner

FWIW, while these macros are the best way to make a simple backport is possible, I just discovered that mainline has a %pV format operator that allows an implementation like:

void
xfs_fs_cmn_err(

	const char              *lvl,
	struct xfs_mount        *mp,
	const char              *fmt,
	...)
{
	struct va_format        vaf;
	va_list                 args;

	va_start(args, fmt);
	vaf.fmt = fmt;
	vaf.va = &args;

	printk("%sFilesystem %s: %pV", lvl, mp->m_fsname, &vaf);
	va_end(args);
Quoted 6 lines
Quoted 2 lines
	BUG_ON(strncmp(lvl, KERN_EMERG, strlen(KERN_EMERG)) == 0);
}

With this we can also keep the existing integer-based CE_ values and do trivial array lookup. That also avoids having to do a strcmp for every message printed.

Very true.

Ok, so how do we want to process for this? I'm happy to drop the macro-ised patch and only use it as a backportable fix for old kernels - mainline appears to have a lot more functionality in this area than even recent distro kernels. Lachlan - is that an acceptable approach for you?

It seems to me that the above is a much better way to start cleaning up the mainline code base, in the following way:

1. the above patch to remove the local message buffer + lock.
2. rationalise all the differences in error reporting down to a common interface
3. convert the common interface to use kernel log levels directly.

The current trend seems to be to move towards logging interfaces with the following template:

	{sub_sys}_pr_{level}(priv, fmt, ...)

Which in this case would give us:

	xfs_pr_debug(struct xfs_mount *mp, char *fmt, ...);
	xfs_pr_note(struct xfs_mount *mp, char *fmt, ...);
	....
	xfs_pr_alert(struct xfs_mount *mp, char *fmt, ...);
	xfs_pr_emergency(struct xfs_mount *mp, char *fmt, ...);

And a variant for the panic mask tagged version of xfs_pr_alert() of:

	xfs_pr_alert_tag(struct xfs_mount *mp, int tag, char *fmt, ...);

I can't see any particular reason for needing to keep the separate parameter for the log level, nor for keeping all the different logging variants we currently have.

I'm open to other ideas on the best way to rationalise all the logging we currently have - the above is just my current thoughts on what to do. I'm pushing this out as a proposal for discussion before I change anything as I don't relish the idea of iterating different logging APIs through patches...

Cheers,

Dave.

Quoted 2 lines
Signature · 7 lines
Dave Chinner
[email protected]

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs