Skip to contentSkip to conversation

xfs log recovery delay instrumentation

PATCH In review Brian Foster 2 participants 4 messages Sep 2, 2014 – Sep 9, 2014 XFS filesystem

lore
Share & tools
lore
#0 lore raw

v2 cover xfs log recovery delay instrumentation

Here's v2 of the log recovery delay instrumentation set. v1 is available here:

http://oss.sgi.com/archives/xfs/2014-08/msg00276.html

After a brief conversation with Dave, we thought it best to tie in this functionality with DEBUG mode to express/ensure that these attributes are for debugging/hacking purposes only. Everything else should be equivalent to v1.

Brian

v2:
- Enable /sys/fs/xfs/debug for DEBUG mode kernels only.
v1: http://oss.sgi.com/archives/xfs/2014-08/msg00276.html

Brian Foster (2):
xfs: add debug sysfs attribute set
xfs: export log_recovery_delay to delay mount time log recovery

 fs/xfs/xfs_globals.c     |  4 +++
 fs/xfs/xfs_log_recover.c | 12 ++++++++
 fs/xfs/xfs_super.c       | 23 +++++++++++++--
 fs/xfs/xfs_sysctl.h      |  5 ++++
 fs/xfs/xfs_sysfs.c       | 74 ++++++++++++++++++++++++++++++++++++++++++++++++
 fs/xfs/xfs_sysfs.h       |  1 +
 6 files changed, 117 insertions(+), 2 deletions(-)
Signature · 6 lines
1.8.3.1

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
#1 lore raw

PATCH v2 1/2 xfs: add debug sysfs attribute set

Create a top-level debug directory for global debug sysfs attributes. This directory is added and removed on XFS module initialization and removal respectively for DEBUG mode kernels only. It typically resides at /sys/fs/xfs/debug. It is located at the top level of the xfs sysfs hierarchy as attributes might define global behavior or behavior that must be configured before an xfs mount is available (e.g., log recovery behavior).

Define the global debug kobject that represents the debug sysfs directory and add generic attribute show/store helpers to support future attributes. No debug attributes are exported as of yet.

---
 fs/xfs/xfs_super.c | 23 +++++++++++++++++++++--
 fs/xfs/xfs_sysfs.c | 43 +++++++++++++++++++++++++++++++++++++++++++
 fs/xfs/xfs_sysfs.h |  1 +
 3 files changed, 65 insertions(+), 2 deletions(-)
fs/xfs/xfs_super.c +21 -2
index b194652..919a64c 100644
@@ -47,6 +47,7 @@
 #include "xfs_dinode.h"
 #include "xfs_filestream.h"
 #include "xfs_quota.h"
+#include "xfs_sysfs.h"
 
 #include <linux/namei.h>
 #include <linux/init.h>
@@ -61,7 +62,11 @@
 static const struct super_operations xfs_super_operations;
 static kmem_zone_t *xfs_ioend_zone;
 mempool_t *xfs_ioend_pool;
-struct kset *xfs_kset;
+
+struct kset *xfs_kset;			/* top-level xfs sysfs dir */
+#ifdef DEBUG
+static struct xfs_kobj xfs_dbg_kobj;	/* global debug sysfs attrs */
+#endif
 
 #define MNTOPT_LOGBUFS	"logbufs"	/* number of XFS log buffers */
 #define MNTOPT_LOGBSIZE	"logbsize"	/* size of XFS log buffers */
@@ -1768,9 +1773,16 @@ init_xfs_fs(void)
 		goto out_sysctl_unregister;;
 	}
 
-	error = xfs_qm_init();
+#ifdef DEBUG
+	xfs_dbg_kobj.kobject.kset = xfs_kset;
+	error = xfs_sysfs_init(&xfs_dbg_kobj, &xfs_dbg_ktype, NULL, "debug");
 	if (error)
 		goto out_kset_unregister;
+#endif
+
+	error = xfs_qm_init();
+	if (error)
+		goto out_remove_kobj;
 
 	error = register_filesystem(&xfs_fs_type);
 	if (error)
@@ -1779,7 +1791,11 @@ init_xfs_fs(void)
 
  out_qm_exit:
 	xfs_qm_exit();
+ out_remove_kobj:
+#ifdef DEBUG
+	xfs_sysfs_del(&xfs_dbg_kobj);
  out_kset_unregister:
+#endif
 	kset_unregister(xfs_kset);
  out_sysctl_unregister:
 	xfs_sysctl_unregister();
@@ -1802,6 +1818,9 @@ exit_xfs_fs(void)
 {
 	xfs_qm_exit();
 	unregister_filesystem(&xfs_fs_type);
+#ifdef DEBUG
+	xfs_sysfs_del(&xfs_dbg_kobj);
+#endif
 	kset_unregister(xfs_kset);
 	xfs_sysctl_unregister();
 	xfs_cleanup_procfs();
fs/xfs/xfs_sysfs.c +43 -0

49 lines not on this page · raw message

fs/xfs/xfs_sysfs.h +1 -0
index 54a2091..240eee3 100644
@@ -20,6 +20,7 @@
 #define __XFS_SYSFS_H__
 
 extern struct kobj_type xfs_mp_ktype;	/* xfs_mount */
+extern struct kobj_type xfs_dbg_ktype;	/* debug */
 extern struct kobj_type xfs_log_ktype;	/* xlog */
 
 static inline struct xfs_kobj *
Signature · 6 lines
1.8.3.1

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
#2 lore raw

PATCH v2 2/2 xfs: export log_recovery_delay to delay mount time log recovery

In reply to Brian Foster

XFS log recovery has been discovered to have race conditions with buffers when I/O errors occur. External tools are available to simulate I/O errors to XFS, but this alone is not sufficient for testing log recovery. XFS unconditionally resets the inactive region of the log prior to log recovery to avoid confusion over processing any partially written log records that might have been written before an unclean shutdown. Therefore, unconditional write I/O failures at mount time are caught by the reset sequence rather than log recovery and hinder the ability to test the latter.

The device-mapper dm-flakey module uses an up/down timer to define a cycle for when to fail I/Os. Create a pre log recovery delay tunable that can be used to coordinate XFS log recovery with I/O errors simulated by dm-flakey. This facilitates coordination in userspace that allows the reset of stale log blocks to succeed and writes due to log recovery to fail. For example, define a dm-flakey instance with an uptime long enough to allow log reset to succeed and a log recovery delay long enough to allow the dm-flakey uptime to expire.

The 'log_recovery_delay' sysfs tunable is exported under /sys/fs/xfs/debug and is only enabled for kernels compiled in XFS debug mode. The value is exported in units of seconds and allows for a delay of up to 60 seconds. Note that this is for XFS debug and test instrumentation purposes only and should not be used by applications. No delay is enabled by default.

---
 fs/xfs/xfs_globals.c     |  4 ++++
 fs/xfs/xfs_log_recover.c | 12 ++++++++++++
 fs/xfs/xfs_sysctl.h      |  5 +++++
 fs/xfs/xfs_sysfs.c       | 31 +++++++++++++++++++++++++++++++
 4 files changed, 52 insertions(+)
fs/xfs/xfs_globals.c +4 -0
index 5399ef2..4d41b24 100644
@@ -43,3 +43,7 @@ xfs_param_t xfs_params = {
 	.fstrm_timer	= {	1,		30*100,		3600*100},
 	.eofb_timer	= {	1,		300,		3600*24},
 };
+
+struct xfs_globals xfs_globals = {
+	.log_recovery_delay	=	0,	/* no delay by default */
+};
fs/xfs/xfs_log_recover.c +12 -0
index 1fd5787..176c4b3 100644
@@ -4509,6 +4509,18 @@ xlog_recover(
 			return -EINVAL;
 		}
 
+		/*
+		 * Delay log recovery if the debug hook is set. This is debug
+		 * instrumention to coordinate simulation of I/O failures with
+		 * log recovery.
+		 */
+		if (xfs_globals.log_recovery_delay) {
+			xfs_notice(log->l_mp,
+				"Delaying log recovery for %d seconds.",
+				xfs_globals.log_recovery_delay);
+			msleep(xfs_globals.log_recovery_delay * 1000);
+		}
+
 		xfs_notice(log->l_mp, "Starting recovery (logdev: %s)",
 				log->l_mp->m_logname ? log->l_mp->m_logname
 						     : "internal");
fs/xfs/xfs_sysctl.h +5 -0
index bd8e157..ffef453 100644
@@ -92,6 +92,11 @@ enum {
 
 extern xfs_param_t	xfs_params;
 
+struct xfs_globals {
+	int	log_recovery_delay;	/* log recovery delay (secs) */
+};
+extern struct xfs_globals	xfs_globals;
+
 #ifdef CONFIG_SYSCTL
 extern int xfs_sysctl_register(void);
 extern void xfs_sysctl_unregister(void);
fs/xfs/xfs_sysfs.c +31 -0
index 32ddf0c..aa03670 100644
@@ -54,7 +54,38 @@ struct kobj_type xfs_mp_ktype = {
 #ifdef DEBUG
 /* debug */
 
+STATIC ssize_t
+log_recovery_delay_store(
+	const char	*buf,
+	size_t		count,
+	void		*data)
+{
+	int		ret;
+	int		val;
+
+	ret = kstrtoint(buf, 0, &val);
+	if (ret)
+		return ret;
+
+	if (val < 0 || val > 60)
+		return -EINVAL;
+
+	xfs_globals.log_recovery_delay = val;
+
+	return count;
+}
+
+STATIC ssize_t
+log_recovery_delay_show(
+	char	*buf,
+	void	*data)
+{
+	return snprintf(buf, PAGE_SIZE, "%d\n", xfs_globals.log_recovery_delay);
+}
+XFS_SYSFS_ATTR_RW(log_recovery_delay);
+
 static struct attribute *xfs_dbg_attrs[] = {
+	ATTR_LIST(log_recovery_delay),
 	NULL,
 };
 
Signature · 6 lines
1.8.3.1

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs
Reviewed-by on the v2 series
#3 lore raw
In reply to Brian Foster
Quoted 12 lines · Brian Foster

Here's v2 of the log recovery delay instrumentation set. v1 is available here:

http://oss.sgi.com/archives/xfs/2014-08/msg00276.html

After a brief conversation with Dave, we thought it best to tie in this functionality with DEBUG mode to express/ensure that these attributes are for debugging/hacking purposes only. Everything else should be equivalent to v1.

Brian

Quoted 7 lines

v2:
- Enable /sys/fs/xfs/debug for DEBUG mode kernels only.
v1: http://oss.sgi.com/archives/xfs/2014-08/msg00276.html

Brian Foster (2):
xfs: add debug sysfs attribute set
xfs: export log_recovery_delay to delay mount time log recovery

These look good. I'm sure that as we add more attributes there will be opportunity for factoring/streamlining this code more, but right now I don't think that is necessary.

Signature · 7 lines
Dave Chinner
[email protected]

_______________________________________________
xfs mailing list
[email protected]
http://oss.sgi.com/mailman/listinfo/xfs