Skip to contentSkip to conversation

documentation and scripts

PATCH New Akinobu Mita 1 participant 7 messages Nov 8, 2006 1 message not in the archive

Subsystems (8)
lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

Akinobu Mita

author maintainer
#1 lore raw

PATCH 1/7 documentation and scripts

From: Akinobu Mita <[email protected]>

This patch set provides some fault-injection capabilities.

- kmalloc() failures

- alloc_pages() failures

- disk IO errors

We can see what really happens if those failures happen.

In order to enable these fault-injection capabilities:

1. Enable relevant config options (CONFIG_FAILSLAB, CONFIG_PAGE_ALLOC, CONFIG_MAKE_REQUEST) and if you want to configure them via debugfs, enable CONFIG_FAULT_INJECTION_DEBUG_FS.

2. Build and boot with this kernel

3. Configure fault-injection capabilities behavior by boot option or debugfs

- Boot option

failslab=
fail_page_alloc=
fail_make_request=

- Debugfs

/debug/failslab/*
/debug/fail_page_alloc/*
/debug/fail_make_request/*

Please refer to the Documentation/fault-injection/fault-injection.txt for details.

4. See what really happens.

 Documentation/fault-injection/failcmd.sh          |    4 
 Documentation/fault-injection/failmodule.sh       |   31 +++
 Documentation/fault-injection/fault-injection.txt |  223 ++++++++++++++++++++++
 Documentation/kernel-parameters.txt               |    7 
 4 files changed, 265 insertions(+)
Documentation/fault-injection/failcmd.sh new file +4 -0
Index: 2.6-rc/Documentation/fault-injection/failcmd.sh
===================================================================
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+echo 1 > /proc/self/make-it-fail
+exec $*
Documentation/fault-injection/failmodule.sh new file +31 -0
Index: 2.6-rc/Documentation/fault-injection/failmodule.sh
===================================================================
@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Usage: failmodule <failname> <modulename> [stacktrace-depth]
+#
+#	<failname>: "failslab", "fail_alloc_page", or "fail_make_request"
+#
+#	<modulename>: module name that you want to inject faults.
+#
+#	[stacktrace-depth]: the maximum number of stacktrace walking allowed
+#
+
+STACKTRACE_DEPTH=5
+if [ $# -gt 2 ]; then
+	STACKTRACE_DEPTH=$3
+fi
+
+if [ ! -d /debug/$1 ]; then
+	echo "Fault-injection $1 does not exist" >&2
+	exit 1
+fi
+if [ ! -d /sys/module/$2 ]; then
+	echo "Module $2 does not exist" >&2
+	exit 1
+fi
+
+# Disable any fault injection
+echo 0 > /debug/$1/stacktrace-depth
+
+echo `cat /sys/module/$2/sections/.text` > /debug/$1/address-start
+echo `cat /sys/module/$2/sections/.exit.text` > /debug/$1/address-end
+echo $STACKTRACE_DEPTH > /debug/$1/stacktrace-depth
Documentation/fault-injection/fault-injection.txt new file +223 -0

223 lines not on this page · raw message

Documentation/kernel-parameters.txt +7 -0
Index: 2.6-rc/Documentation/kernel-parameters.txt
===================================================================
@@ -544,6 +544,13 @@ and is between 256 and 4096 characters. 
 	eurwdt=		[HW,WDT] Eurotech CPU-1220/1410 onboard watchdog.
 			Format: <io>[,<irq>]
 
+	failslab=
+	fail_page_alloc=
+	fail_make_request=[KNL]
+			General fault injection mechanism.
+			Format: <interval>,<probability>,<space>,<times>
+			See also /Documentation/fault-injection/.
+
 	fd_mcs=		[HW,SCSI]
 			See header of drivers/scsi/fd_mcs.c.
 

--

author maintainer
#2 lore raw

PATCH 2/7 fault-injection capabilities infrastructure

In reply to a message not in the archive

From: Akinobu Mita <[email protected]>

This patch provides base functions implement to fault-injection capabilities.

- The function should_fail() is taken from failmalloc-1.0
(http://www.nongnu.org/failmalloc/)

 include/linux/fault-inject.h |   69 ++++++++++++++++
 lib/Kconfig.debug            |   12 ++
 lib/Makefile                 |    1 
 lib/fault-inject.c           |  179 +++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 261 insertions(+)
lib/Kconfig.debug +12 -0
Index: 2.6-rc/lib/Kconfig.debug
===================================================================
@@ -412,3 +412,15 @@ config LKDTM
 
 	Documentation on how to use the module can be found in
 	drivers/misc/lkdtm.c
+
+config FAULT_INJECTION
+	bool
+
+config FAULT_INJECTION_DEBUG_FS
+	bool "Debugfs entries for fault-injection capabilities"
+	depends on FAULT_INJECTION && SYSFS
+	select DEBUG_FS
+	help
+	  This option enables to configure fault-injection capabilities via
+	  debugfs entries.
+
lib/Makefile +1 -0
Index: 2.6-rc/lib/Makefile
===================================================================
@@ -54,6 +54,7 @@ obj-$(CONFIG_SMP) += percpu_counter.o
 obj-$(CONFIG_AUDIT_GENERIC) += audit.o
 
 obj-$(CONFIG_SWIOTLB) += swiotlb.o
+obj-$(CONFIG_FAULT_INJECTION) += fault-inject.o
 
 hostprogs-y	:= gen_crc32table
 clean-files	:= crc32table.h
include/linux/fault-inject.h new file +69 -0

69 lines not on this page · raw message

lib/fault-inject.c new file +179 -0

179 lines not on this page · raw message

--

author maintainer
#3 lore raw

PATCH 3/7 fault-injection capability for kmalloc

In reply to a message not in the archive

From: Akinobu Mita <[email protected]>

This patch provides fault-injection capability for kmalloc.

Boot option:

failslab=<interval>,<probability>,<space>,<times>

<interval> -- specifies the interval of failures.

<probability> -- specifies how often it should fail in percent.

<space> -- specifies the size of free space where memory can be
allocated safely in bytes.

<times> -- specifies how many times failures may happen at most.

Debugfs:

/debug/failslab/interval
/debug/failslab/probability
/debug/failslab/specifies
/debug/failslab/times
/debug/failslab/ignore-gfp-highmem
/debug/failslab/ignore-gfp-wait

Example:

	failslab=10,100,0,-1

slab allocation (kmalloc(), kmem_cache_alloc(),..) fails once per 10 times.

 lib/Kconfig.debug |    7 ++++
 mm/slab.c         |   90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 97 insertions(+)
mm/slab.c +90 -0
Index: 2.6-rc/mm/slab.c
===================================================================
@@ -107,6 +107,7 @@
 #include	<linux/mempolicy.h>
 #include	<linux/mutex.h>
 #include	<linux/rtmutex.h>
+#include	<linux/fault-inject.h>
 
 #include	<asm/uaccess.h>
 #include	<asm/cacheflush.h>
@@ -3069,12 +3070,101 @@ static void *cache_alloc_debugcheck_afte
 #define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
 #endif
 
+#ifdef CONFIG_FAILSLAB
+
+static struct failslab_attr {
+
+	struct fault_attr attr;
+
+	u32 ignore_gfp_highmem;
+	u32 ignore_gfp_wait;
+
+#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
+
+	struct dentry *ignore_gfp_highmem_file;
+	struct dentry *ignore_gfp_wait_file;
+
+#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
+
+} failslab = {
+	.attr = FAULT_ATTR_INITIALIZER,
+};
+
+static int __init setup_failslab(char *str)
+{
+	return setup_fault_attr(&failslab.attr, str);
+}
+__setup("failslab=", setup_failslab);
+
+static int should_failslab(struct kmem_cache *cachep, gfp_t flags)
+{
+	if (cachep == &cache_cache)
+		return 0;
+	if (flags & __GFP_NOFAIL)
+		return 0;
+	if (failslab.ignore_gfp_highmem && (flags & __GFP_HIGHMEM))
+		return 0;
+	if (failslab.ignore_gfp_wait && (flags & __GFP_WAIT))
+		return 0;
+
+	return should_fail(&failslab.attr, obj_size(cachep));
+}
+
+#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
+
+static int __init failslab_debugfs(void)
+{
+	mode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
+	struct dentry *dir;
+	int err;
+
+       	err = init_fault_attr_dentries(&failslab.attr, "failslab");
+	if (err)
+		return err;
+	dir = failslab.attr.dentries.dir;
+
+	failslab.ignore_gfp_wait_file =
+		debugfs_create_bool("ignore-gfp-wait", mode, dir,
+				      &failslab.ignore_gfp_wait);
+
+	failslab.ignore_gfp_highmem_file =
+		debugfs_create_bool("ignore-gfp-highmem", mode, dir,
+				      &failslab.ignore_gfp_highmem);
+
+	if (!failslab.ignore_gfp_wait_file ||
+			!failslab.ignore_gfp_highmem_file) {
+		err = -ENOMEM;
+		debugfs_remove(failslab.ignore_gfp_wait_file);
+		debugfs_remove(failslab.ignore_gfp_highmem_file);
+		cleanup_fault_attr_dentries(&failslab.attr);
+	}
+
+	return err;
+}
+
+late_initcall(failslab_debugfs);
+
+#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
+
+#else /* CONFIG_FAILSLAB */
+
+static inline int should_failslab(struct kmem_cache *cachep, gfp_t flags)
+{
+	return 0;
+}
+
+#endif /* CONFIG_FAILSLAB */
+
 static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
 {
 	void *objp;
 	struct array_cache *ac;
 
 	check_irq_off();
+
+	if (should_failslab(cachep, flags))
+		return NULL;
+
 	ac = cpu_cache_get(cachep);
 	if (likely(ac->avail)) {
 		STATS_INC_ALLOCHIT(cachep);
lib/Kconfig.debug +7 -0
Index: 2.6-rc/lib/Kconfig.debug
===================================================================
@@ -416,6 +416,13 @@ config LKDTM
 config FAULT_INJECTION
 	bool
 
+config FAILSLAB
+	bool "Fault-injection capabilitiy for kmalloc"
+	depends on DEBUG_KERNEL
+	select FAULT_INJECTION
+	help
+	  This option provides fault-injection capabilitiy for kmalloc.
+
 config FAULT_INJECTION_DEBUG_FS
 	bool "Debugfs entries for fault-injection capabilities"
 	depends on FAULT_INJECTION && SYSFS

--

author maintainer
#4 lore raw

PATCH 4/7 fault-injection capability for alloc_pages()

In reply to a message not in the archive

From: Akinobu Mita <[email protected]>

This patch provides fault-injection capability for alloc_pages()

Boot option:

fail_page_alloc=<interval>,<probability>,<space>,<times>

<interval> -- specifies the interval of failures.

<probability> -- specifies how often it should fail in percent.

<space> -- specifies the size of free space where memory can be
allocated safely in pages.

<times> -- specifies how many times failures may happen at most.

Debugfs:

/debug/fail_page_alloc/interval
/debug/fail_page_alloc/probability
/debug/fail_page_alloc/specifies
/debug/fail_page_alloc/times
/debug/fail_page_alloc/ignore-gfp-highmem
/debug/fail_page_alloc/ignore-gfp-wait

Example:

	fail_page_alloc=10,100,0,-1

The page allocation (alloc_pages(), ...) fails once per 10 times.

 lib/Kconfig.debug |    7 ++++
 mm/page_alloc.c   |   87 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 94 insertions(+)
lib/Kconfig.debug +7 -0
Index: 2.6-rc/lib/Kconfig.debug
===================================================================
@@ -423,6 +423,13 @@ config FAILSLAB
 	help
 	  This option provides fault-injection capabilitiy for kmalloc.
 
+config FAIL_PAGE_ALLOC
+	bool "Fault-injection capabilitiy for alloc_pages()"
+	depends on DEBUG_KERNEL
+	select FAULT_INJECTION
+	help
+	  This option provides fault-injection capabilitiy for alloc_pages().
+
 config FAULT_INJECTION_DEBUG_FS
 	bool "Debugfs entries for fault-injection capabilities"
 	depends on FAULT_INJECTION && SYSFS
mm/page_alloc.c +87 -0

105 lines not on this page · raw message

--

author maintainer
#5 lore raw

PATCH 5/7 fault-injection capability for disk IO

In reply to a message not in the archive

From: Akinobu Mita <[email protected]>

This patch provides fault-injection capability for disk IO.

Boot option:

fail_make_request=<probability>,<interval>,<space>,<times>

<interval> -- specifies the interval of failures.

<probability> -- specifies how often it should fail in percent.

<space> -- specifies the size of free space where disk IO can be issued
safely in bytes.

<times> -- specifies how many times failures may happen at most.

Debugfs:

/debug/fail_make_request/interval
/debug/fail_make_request/probability
/debug/fail_make_request/specifies
/debug/fail_make_request/times

Example:

	fail_make_request=10,100,0,-1
	echo 1 > /sys/blocks/hda/hda1/make-it-fail

generic_make_request() on /dev/hda1 fails once per 10 times.

 block/genhd.c         |   31 +++++++++++++++++++++++++++++++
 block/ll_rw_blk.c     |   40 ++++++++++++++++++++++++++++++++++++++++
 fs/partitions/check.c |   27 +++++++++++++++++++++++++++
 include/linux/genhd.h |    4 ++++
 lib/Kconfig.debug     |    7 +++++++
 5 files changed, 109 insertions(+)
block/ll_rw_blk.c +40 -0
Index: 2.6-rc/block/ll_rw_blk.c
===================================================================
@@ -28,6 +28,7 @@
 #include <linux/interrupt.h>
 #include <linux/cpu.h>
 #include <linux/blktrace_api.h>
+#include <linux/fault-inject.h>
 
 /*
  * for max sense size
@@ -2971,6 +2972,42 @@ static void handle_bad_sector(struct bio
 	set_bit(BIO_EOF, &bio->bi_flags);
 }
 
+#ifdef CONFIG_FAIL_MAKE_REQUEST
+
+static DECLARE_FAULT_ATTR(fail_make_request);
+
+static int __init setup_fail_make_request(char *str)
+{
+	return setup_fault_attr(&fail_make_request, str);
+}
+__setup("fail_make_request=", setup_fail_make_request);
+
+static int should_fail_request(struct bio *bio)
+{
+	if ((bio->bi_bdev->bd_disk->flags & GENHD_FL_FAIL) ||
+	    (bio->bi_bdev->bd_part && bio->bi_bdev->bd_part->make_it_fail))
+		return should_fail(&fail_make_request, bio->bi_size);
+
+	return 0;
+}
+
+static int __init fail_make_request_debugfs(void)
+{
+	return init_fault_attr_dentries(&fail_make_request,
+					"fail_make_request");
+}
+
+late_initcall(fail_make_request_debugfs);
+
+#else /* CONFIG_FAIL_MAKE_REQUEST */
+
+static inline int should_fail_request(struct bio *bio)
+{
+	return 0;
+}
+
+#endif /* CONFIG_FAIL_MAKE_REQUEST */
+
 /**
  * generic_make_request: hand a buffer to its device driver for I/O
  * @bio:  The bio describing the location in memory and on the device.
@@ -3056,6 +3093,9 @@ end_io:
 		if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)))
 			goto end_io;
 
+		if (should_fail_request(bio))
+			goto end_io;
+
 		/*
 		 * If this device has partitions, remap block n
 		 * of partition p to block n+start(p) of the disk.
lib/Kconfig.debug +7 -0
Index: 2.6-rc/lib/Kconfig.debug
===================================================================
@@ -430,6 +430,13 @@ config FAIL_PAGE_ALLOC
 	help
 	  This option provides fault-injection capabilitiy for alloc_pages().
 
+config FAIL_MAKE_REQUEST
+	bool "Fault-injection capabilitiy for disk IO"
+	depends on DEBUG_KERNEL
+	select FAULT_INJECTION 
+	help
+	  This option provides fault-injection capabilitiy to disk IO.
+
 config FAULT_INJECTION_DEBUG_FS
 	bool "Debugfs entries for fault-injection capabilities"
 	depends on FAULT_INJECTION && SYSFS
block/genhd.c +31 -0
Index: 2.6-rc/block/genhd.c
===================================================================
@@ -417,6 +417,34 @@ static struct disk_attribute disk_attr_s
 	.show	= disk_stats_read
 };
 
+#ifdef CONFIG_FAIL_MAKE_REQUEST
+
+static ssize_t disk_fail_store(struct gendisk * disk,
+			       const char *buf, size_t count)
+{
+	int i;
+
+	if (count > 0 && sscanf(buf, "%d", &i) > 0) {
+		if (i == 0)
+			disk->flags &= ~GENHD_FL_FAIL;
+		else
+			disk->flags |= GENHD_FL_FAIL;
+	}
+
+	return count;
+}
+static ssize_t disk_fail_read(struct gendisk * disk, char *page)
+{
+	return sprintf(page, "%d\n", disk->flags & GENHD_FL_FAIL ? 1 : 0);
+}
+static struct disk_attribute disk_attr_fail = {
+	.attr = {.name = "make-it-fail", .mode = S_IRUGO | S_IWUSR },
+	.store	= disk_fail_store,
+	.show	= disk_fail_read
+};
+
+#endif
+
 static struct attribute * default_attrs[] = {
 	&disk_attr_uevent.attr,
 	&disk_attr_dev.attr,
@@ -424,6 +452,9 @@ static struct attribute * default_attrs[
 	&disk_attr_removable.attr,
 	&disk_attr_size.attr,
 	&disk_attr_stat.attr,
+#ifdef CONFIG_FAIL_MAKE_REQUEST
+	&disk_attr_fail.attr,
+#endif
 	NULL,
 };
 
include/linux/genhd.h +4 -0
Index: 2.6-rc/include/linux/genhd.h
===================================================================
@@ -83,6 +83,9 @@ struct hd_struct {
 	struct kobject *holder_dir;
 	unsigned ios[2], sectors[2];	/* READs and WRITEs */
 	int policy, partno;
+#ifdef CONFIG_FAIL_MAKE_REQUEST
+	int make_it_fail;
+#endif
 };
 
 #define GENHD_FL_REMOVABLE			1
@@ -90,6 +93,7 @@ struct hd_struct {
 #define GENHD_FL_CD				8
 #define GENHD_FL_UP				16
 #define GENHD_FL_SUPPRESS_PARTITION_INFO	32
+#define GENHD_FL_FAIL				64
 
 struct disk_stats {
 	unsigned long sectors[2];	/* READs and WRITEs */
fs/partitions/check.c +27 -0
Index: 2.6-rc/fs/partitions/check.c
===================================================================
@@ -265,12 +265,39 @@ static struct part_attribute part_attr_s
 	.show	= part_stat_read
 };
 
+#ifdef CONFIG_FAIL_MAKE_REQUEST
+
+static ssize_t part_fail_store(struct hd_struct * p,
+			       const char *buf, size_t count)
+{
+	int i;
+
+	if (count > 0 && sscanf(buf, "%d", &i) > 0)
+		p->make_it_fail = (i == 0) ? 0 : 1;
+
+	return count;
+}
+static ssize_t part_fail_read(struct hd_struct * p, char *page)
+{
+	return sprintf(page, "%d\n", p->make_it_fail);
+}
+static struct part_attribute part_attr_fail = {
+	.attr = {.name = "make-it-fail", .mode = S_IRUGO | S_IWUSR },
+	.store	= part_fail_store,
+	.show	= part_fail_read
+};
+
+#endif
+
 static struct attribute * default_attrs[] = {
 	&part_attr_uevent.attr,
 	&part_attr_dev.attr,
 	&part_attr_start.attr,
 	&part_attr_size.attr,
 	&part_attr_stat.attr,
+#ifdef CONFIG_FAIL_MAKE_REQUEST
+	&part_attr_fail.attr,
+#endif
 	NULL,
 };
 

--

author maintainer
#6 lore raw

PATCH 6/7 process filtering for fault-injection capabilities

In reply to a message not in the archive

From: Akinobu Mita <[email protected]>

This patch provides process filtering feature. The process filter allows failing only permitted processes by /proc/<pid>/make-it-fail

Please see the example that demostrates how to inject slab allocation failures into module init/cleanup code
in Documentation/fault-injection/fault-injection.txt

 fs/proc/base.c               |   65 +++++++++++++++++++++++++++++++++++++++++++
 include/linux/fault-inject.h |    2 +
 include/linux/sched.h        |    3 +
 lib/fault-inject.c           |   17 ++++++++++-
 4 files changed, 86 insertions(+), 1 deletion(-)
fs/proc/base.c +65 -0
Index: 2.6-rc/fs/proc/base.c
===================================================================
@@ -850,6 +850,65 @@ static struct file_operations proc_secco
 };
 #endif /* CONFIG_SECCOMP */
 
+#ifdef CONFIG_FAULT_INJECTION
+static ssize_t proc_fault_inject_read(struct file * file, char __user * buf,
+				      size_t count, loff_t *ppos)
+{
+	struct task_struct *task = get_proc_task(file->f_dentry->d_inode);
+	char buffer[PROC_NUMBUF];
+	size_t len;
+	int make_it_fail;
+	loff_t __ppos = *ppos;
+
+	if (!task)
+		return -ESRCH;
+	make_it_fail = task->make_it_fail;
+	put_task_struct(task);
+
+	len = snprintf(buffer, sizeof(buffer), "%i\n", make_it_fail);
+	if (__ppos >= len)
+		return 0;
+	if (count > len-__ppos)
+		count = len-__ppos;
+	if (copy_to_user(buf, buffer + __ppos, count))
+		return -EFAULT;
+	*ppos = __ppos + count;
+	return count;
+}
+
+static ssize_t proc_fault_inject_write(struct file * file,
+			const char __user * buf, size_t count, loff_t *ppos)
+{
+	struct task_struct *task;
+	char buffer[PROC_NUMBUF], *end;
+	int make_it_fail;
+
+	if (!capable(CAP_SYS_RESOURCE))
+		return -EPERM;
+	memset(buffer, 0, sizeof(buffer));
+	if (count > sizeof(buffer) - 1)
+		count = sizeof(buffer) - 1;
+	if (copy_from_user(buffer, buf, count))
+		return -EFAULT;
+	make_it_fail = simple_strtol(buffer, &end, 0);
+	if (*end == '\n')
+		end++;
+	task = get_proc_task(file->f_dentry->d_inode);
+	if (!task)
+		return -ESRCH;
+	task->make_it_fail = make_it_fail;
+	put_task_struct(task);
+	if (end - buffer == 0)
+		return -EIO;
+	return end - buffer;
+}
+
+static struct file_operations proc_fault_inject_operations = {
+	.read		= proc_fault_inject_read,
+	.write		= proc_fault_inject_write,
+};
+#endif
+
 static void *proc_pid_follow_link(struct dentry *dentry, struct nameidata *nd)
 {
 	struct inode *inode = dentry->d_inode;
@@ -1790,6 +1849,9 @@ static struct pid_entry tgid_base_stuff[
 #ifdef CONFIG_AUDITSYSCALL
 	REG("loginuid",   S_IWUSR|S_IRUGO, loginuid),
 #endif
+#ifdef CONFIG_FAULT_INJECTION
+	REG("make-it-fail", S_IRUGO|S_IWUSR, fault_inject),
+#endif
 };
 
 static int proc_tgid_base_readdir(struct file * filp,
@@ -2064,6 +2126,9 @@ static struct pid_entry tid_base_stuff[]
 #ifdef CONFIG_AUDITSYSCALL
 	REG("loginuid",  S_IWUSR|S_IRUGO, loginuid),
 #endif
+#ifdef CONFIG_FAULT_INJECTION
+	REG("make-it-fail", S_IRUGO|S_IWUSR, fault_inject),
+#endif
 };
 
 static int proc_tid_base_readdir(struct file * filp,
include/linux/sched.h +3 -0
Index: 2.6-rc/include/linux/sched.h
===================================================================
@@ -1023,6 +1023,9 @@ struct task_struct {
 #ifdef	CONFIG_TASK_DELAY_ACCT
 	struct task_delay_info *delays;
 #endif
+#ifdef CONFIG_FAULT_INJECTION
+	int make_it_fail;
+#endif
 };
 
 static inline pid_t process_group(struct task_struct *tsk)
include/linux/fault-inject.h +2 -0
Index: 2.6-rc/include/linux/fault-inject.h
===================================================================
@@ -17,6 +17,7 @@ struct fault_attr {
 	atomic_t times;
 	atomic_t space;
 	unsigned long verbose;
+	u32 task_filter;
 
 	unsigned long count;
 
@@ -30,6 +31,7 @@ struct fault_attr {
 		struct dentry *times_file;
 		struct dentry *space_file;
 		struct dentry *verbose_file;
+		struct dentry *task_filter_file;
 	} dentries;
 
 #endif
lib/fault-inject.c +16 -1
Index: 2.6-rc/lib/fault-inject.c
===================================================================
@@ -5,6 +5,7 @@
 #include <linux/types.h>
 #include <linux/fs.h>
 #include <linux/module.h>
+#include <linux/interrupt.h>
 #include <linux/fault-inject.h>
 
 int setup_fault_attr(struct fault_attr *attr, char *str)
@@ -40,6 +41,11 @@ static void fail_dump(struct fault_attr 
 
 #define atomic_dec_not_zero(v)		atomic_add_unless((v), -1, 0)
 
+static int fail_task(struct fault_attr *attr, struct task_struct *task)
+{
+	return !in_interrupt() && task->make_it_fail;
+}
+
 /*
  * This code is stolen from failmalloc-1.0
  * http://www.nongnu.org/failmalloc/
@@ -47,6 +53,9 @@ static void fail_dump(struct fault_attr 
 
 int should_fail(struct fault_attr *attr, ssize_t size)
 {
+	if (attr->task_filter && !fail_task(attr, current))
+		return 0;
+
 	if (atomic_read(&attr->times) == 0)
 		return 0;
 
@@ -131,6 +140,9 @@ void cleanup_fault_attr_dentries(struct 
 	debugfs_remove(attr->dentries.verbose_file);
 	attr->dentries.verbose_file = NULL;
 
+	debugfs_remove(attr->dentries.task_filter_file);
+	attr->dentries.task_filter_file = NULL;
+
 	if (attr->dentries.dir)
 		WARN_ON(!simple_empty(attr->dentries.dir));
 
@@ -165,9 +177,12 @@ int init_fault_attr_dentries(struct faul
 	attr->dentries.verbose_file =
 		debugfs_create_ul("verbose", mode, dir, &attr->verbose);
 
+	attr->dentries.task_filter_file = debugfs_create_bool("task-filter",
+						mode, dir, &attr->task_filter);
+
 	if (!attr->dentries.probability_file || !attr->dentries.interval_file
 	    || !attr->dentries.times_file || !attr->dentries.space_file
-	    || !attr->dentries.verbose_file)
+	    || !attr->dentries.verbose_file || !attr->dentries.task_filter_file)
 		goto fail;
 
 	return 0;

--

author maintainer
#7 lore raw

PATCH 7/7 stacktrace filtering

In reply to a message not in the archive

From: Akinobu Mita <[email protected]>

This patch provides stacktrace filtering feature.

The stacktrace filter allows failing only for the caller you are interested in.

For example someone may want to inject kmalloc() failures into only e100 module. they want to inject not only direct kmalloc() call, but also indirect allocation, too.

- e100_poll --> netif_receive_skb --> packet_rcv_spkt --> skb_clone --> kmem_cache_alloc

This patch enables to detect function calls like this by stacktrace and inject failures. The script Documentaion/fault-injection/failmodule.sh helps it.

The range of text section of loaded e100 is expected to be [/sys/module/e100/sections/.text, /sys/module/e100/sections/.exit.text)

So failmodule.sh stores these values into /debug/failslab/address-start and /debug/failslab/address-end. The maximum stacktrace depth is specified by /debug/failslab/stacktrace-depth.

Please see the example that demonstrates how to inject slab allocation failures only for a specific module
in Documentation/fault-injection/fault-injection.txt

 include/linux/fault-inject.h |    7 ++
 lib/Kconfig.debug            |    2 
 lib/fault-inject.c           |  107 ++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 115 insertions(+), 1 deletion(-)
lib/fault-inject.c +106 -1
Index: 2.6-rc/lib/fault-inject.c
===================================================================
@@ -6,6 +6,9 @@
 #include <linux/fs.h>
 #include <linux/module.h>
 #include <linux/interrupt.h>
+#include <linux/unwind.h>
+#include <linux/stacktrace.h>
+#include <linux/kallsyms.h>
 #include <linux/fault-inject.h>
 
 int setup_fault_attr(struct fault_attr *attr, char *str)
@@ -46,6 +49,82 @@ static int fail_task(struct fault_attr *
 	return !in_interrupt() && task->make_it_fail;
 }
 
+static int fail_any_address(struct fault_attr *attr)
+{
+	return (attr->address_start == 0 && attr->address_end == ULONG_MAX);
+}
+
+#ifdef CONFIG_STACK_UNWIND
+
+static asmlinkage int fail_stacktrace_callback(struct unwind_frame_info *info,
+						void *arg)
+{
+	int depth;
+	struct fault_attr *attr = arg;
+
+	for (depth = 0; depth < attr->stacktrace_depth
+			&& unwind(info) == 0 && UNW_PC(info); depth++) {
+		if (arch_unw_user_mode(info))
+			break;
+		if (attr->address_start <= UNW_PC(info) &&
+			       UNW_PC(info) < attr->address_end)
+			return 1;
+	}
+	return 0;
+}
+
+static int fail_stacktrace(struct fault_attr *attr)
+{
+	struct unwind_frame_info info;
+
+	return unwind_init_running(&info, fail_stacktrace_callback, attr);
+}
+
+#elif defined(CONFIG_STACKTRACE)
+
+#define MAX_STACK_TRACE_DEPTH 10
+
+static int fail_stacktrace(struct fault_attr *attr)
+{
+	struct stack_trace trace;
+	int depth = attr->stacktrace_depth;
+	unsigned long entries[MAX_STACK_TRACE_DEPTH];
+	int n;
+
+	if (depth == 0)
+		return 0;
+
+	trace.nr_entries = 0;
+	trace.entries = entries;
+	trace.max_entries = (depth < MAX_STACK_TRACE_DEPTH) ?
+				depth : MAX_STACK_TRACE_DEPTH;
+	trace.skip = 1;
+	trace.all_contexts = 0;
+
+	save_stack_trace(&trace, NULL);
+	for (n = 0; n < trace.nr_entries; n++)
+		if (attr->address_start <= entries[n] &&
+			       entries[n] < attr->address_end)
+			return 1;
+	return 0;
+}
+
+#else
+
+static inline int fail_stacktrace(struct fault_attr *attr)
+{
+	static int firsttime = 1;
+
+	if (firsttime) {
+		printk(KERN_WARNING
+		"This architecture does not implement save_stack_trace()\n");
+		firsttime = 0;
+	}
+	return 0;
+}
+
+#endif
+
 /*
  * This code is stolen from failmalloc-1.0
  * http://www.nongnu.org/failmalloc/
@@ -56,6 +135,9 @@ int should_fail(struct fault_attr *attr,
 	if (attr->task_filter && !fail_task(attr, current))
 		return 0;
 
+	if (!fail_any_address(attr) && !fail_stacktrace(attr))
+		return 0;
+
 	if (atomic_read(&attr->times) == 0)
 		return 0;
 
@@ -143,6 +225,15 @@ void cleanup_fault_attr_dentries(struct 
 	debugfs_remove(attr->dentries.task_filter_file);
 	attr->dentries.task_filter_file = NULL;
 
+	debugfs_remove(attr->dentries.stacktrace_depth_file);
+	attr->dentries.stacktrace_depth_file = NULL;
+
+	debugfs_remove(attr->dentries.address_start_file);
+	attr->dentries.address_start_file = NULL;
+
+	debugfs_remove(attr->dentries.address_end_file);
+	attr->dentries.address_end_file = NULL;
+
 	if (attr->dentries.dir)
 		WARN_ON(!simple_empty(attr->dentries.dir));
 
@@ -180,9 +271,23 @@ int init_fault_attr_dentries(struct faul
 	attr->dentries.task_filter_file = debugfs_create_bool("task-filter",
 						mode, dir, &attr->task_filter);
 
+	attr->dentries.stacktrace_depth_file =
+		debugfs_create_ul("stacktrace-depth", mode, dir,
+				  &attr->stacktrace_depth);
+
+	attr->dentries.address_start_file = debugfs_create_ul("address-start",
+					mode, dir, &attr->address_start);
+
+	attr->dentries.address_end_file =
+		debugfs_create_ul("address-end", mode, dir, &attr->address_end);
+
+
 	if (!attr->dentries.probability_file || !attr->dentries.interval_file
 	    || !attr->dentries.times_file || !attr->dentries.space_file
-	    || !attr->dentries.verbose_file || !attr->dentries.task_filter_file)
+	    || !attr->dentries.verbose_file || !attr->dentries.task_filter_file
+	    || !attr->dentries.stacktrace_depth_file
+	    || !attr->dentries.address_start_file
+	    || !attr->dentries.address_end_file)
 		goto fail;
 
 	return 0;
include/linux/fault-inject.h +7 -0
Index: 2.6-rc/include/linux/fault-inject.h
===================================================================
@@ -18,6 +18,9 @@ struct fault_attr {
 	atomic_t space;
 	unsigned long verbose;
 	u32 task_filter;
+	unsigned long stacktrace_depth;
+	unsigned long address_start;
+	unsigned long address_end;
 
 	unsigned long count;
 
@@ -32,6 +35,9 @@ struct fault_attr {
 		struct dentry *space_file;
 		struct dentry *verbose_file;
 		struct dentry *task_filter_file;
+		struct dentry *stacktrace_depth_file;
+		struct dentry *address_start_file;
+		struct dentry *address_end_file;
 	} dentries;
 
 #endif
@@ -40,6 +46,7 @@ struct fault_attr {
 #define FAULT_ATTR_INITIALIZER {				\
 		.interval = 1,					\
 		.times = ATOMIC_INIT(1),			\
+		.address_end = ULONG_MAX,			\
 	}
 
 #define DECLARE_FAULT_ATTR(name) struct fault_attr name = FAULT_ATTR_INITIALIZER	
lib/Kconfig.debug +2 -0
Index: 2.6-rc/lib/Kconfig.debug
===================================================================
@@ -415,6 +415,8 @@ config LKDTM
 
 config FAULT_INJECTION
 	bool
+	select STACKTRACE
+	select FRAME_POINTER
 
 config FAILSLAB
 	bool "Fault-injection capabilitiy for kmalloc"

--