Skip to contentSkip to conversation

dm delay:

PATCH In review Alasdair G Kergon 3 participants 5 messages May 8, 2007 – May 10, 2007

Subsystems (3)
lore
Share & tools
lore

Alasdair G Kergon

author maintainer
#0 lore raw

PATCH 23/26 dm delay:

From: Heinz Mauelshagen <[email protected]>

New device-mapper target that can delay I/O (for testing). Reads can be separated from writes, redirected to different underlying devices and delayed by differing amounts of time.

---

 Documentation/device-mapper/delay.txt |   26 ++
 drivers/md/Kconfig                    |    9 
 drivers/md/Makefile                   |    1 
 drivers/md/dm-delay.c                 |  383 ++++++++++++++++++++++++++++++++++
 4 files changed, 419 insertions(+)
Documentation/device-mapper/delay.txt new file +26 -0
Index: linux-2.6.21/Documentation/device-mapper/delay.txt
===================================================================
@@ -0,0 +1,26 @@
+dm-delay
+========
+
+Device-Mapper's "delay" target delays reads and/or writes
+and maps them to different devices.
+
+Parameters:
+    <device> <offset> <delay> [<write_device> <write_offset> <write_delay>]
+
+With separate write parameters, the first set is only used for reads.
+Delays are specified in milliseconds.
+
+Example scripts
+===============
+[[
+#!/bin/sh
+# Create device delaying rw operation for 500ms
+echo "0 `blockdev --getsize $1` delay $1 0 500" | dmsetup create delayed
+]]
+
+[[
+#!/bin/sh
+# Create device delaying only write operation for 500ms and
+# splitting reads and writes to different devices $1 $2
+echo "0 `blockdev --getsize $1` delay $1 0 0 $2 0 500" | dmsetup create delayed
+]]
drivers/md/Kconfig +9 -0
Index: linux-2.6.21/drivers/md/Kconfig
===================================================================
@@ -262,6 +262,15 @@ config DM_MULTIPATH_EMC
 	---help---
 	  Multipath support for EMC CX/AX series hardware.
 
+config DM_DELAY
+	tristate "I/O delaying target (EXPERIMENTAL)"
+	depends on BLK_DEV_DM && EXPERIMENTAL
+	---help---
+	A target that delays reads and/or writes and can send
+	them to different devices.  Useful for testing.
+
+	If unsure, say N.
+
 endmenu
 
 endif
drivers/md/Makefile +1 -0
Index: linux-2.6.21/drivers/md/Makefile
===================================================================
@@ -31,6 +31,7 @@ obj-$(CONFIG_MD_FAULTY)		+= faulty.o
 obj-$(CONFIG_BLK_DEV_MD)	+= md-mod.o
 obj-$(CONFIG_BLK_DEV_DM)	+= dm-mod.o
 obj-$(CONFIG_DM_CRYPT)		+= dm-crypt.o
+obj-$(CONFIG_DM_DELAY)		+= dm-delay.o
 obj-$(CONFIG_DM_MULTIPATH)	+= dm-multipath.o dm-round-robin.o
 obj-$(CONFIG_DM_MULTIPATH_EMC)	+= dm-emc.o
 obj-$(CONFIG_DM_SNAPSHOT)	+= dm-snapshot.o
drivers/md/dm-delay.c new file +383 -0

383 lines not on this page · raw message

#1 lore raw
Quoted 21 lines · On Tue, 8 May 2007 20:48:59 +0100 Alasdair G Kergon <[email protected]> wrote:

From: Heinz Mauelshagen <[email protected]>

New device-mapper target that can delay I/O (for testing). Reads can be separated from writes, redirected to different underlying devices and delayed by differing amounts of time.

..

drivers/md/dm-delay.c new file +13 -0
+/*
+ * Copyright (C) 2005-2007 Red Hat GmbH
+ *
+ * A target that delays reads and/or writes and can send
+ * them to different devices.
+ *
+ * This file is released under the GPL.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/blkdev.h>
+#include <linux/bio.h>

drivers/md/dm-delay.c line 23

Quoted 10 lines
drivers/md/dm-delay.c new file +10 -0
+#include <linux/slab.h>
+
+#include "dm.h"
+#include "dm-bio-list.h"
+
+#define DM_MSG_PREFIX "delay"
+
+struct delay_c {
+	struct timer_list delay_timer;
+	struct semaphore timer_lock;

Can we get this converted to a mutex asap, please?

It's only used in a single place and perhaps we don't need this lock at all?

Quoted 41 lines
drivers/md/dm-delay.c new file +41 -0
+	struct work_struct flush_expired_bios;
+	struct list_head delayed_bios;
+	atomic_t may_delay;
+	mempool_t *delayed_pool;
+
+	struct dm_dev *dev_read;
+	sector_t start_read;
+	unsigned read_delay;
+	unsigned reads;
+
+	struct dm_dev *dev_write;
+	sector_t start_write;
+	unsigned write_delay;
+	unsigned writes;
+};
+
+struct delay_info {
+	struct delay_c *context;
+	struct list_head list;
+	struct bio *bio;
+	unsigned long expires;
+};
+
+static DEFINE_MUTEX(delayed_bios_lock);
+
+static struct workqueue_struct *kdelayd_wq;
+static struct kmem_cache *delayed_cache;
+
+static void handle_delayed_timer(unsigned long data)
+{
+	struct delay_c *dc = (struct delay_c *)data;
+
+	queue_work(kdelayd_wq, &dc->flush_expired_bios);
+}
+
+static void queue_timeout(struct delay_c *dc, unsigned long expires)
+{
+	down(&dc->timer_lock);
+
+	if (!timer_pending(&dc->delay_timer) || expires < dc->delay_timer.expires)
+		mod_timer(&dc->delay_timer, expires);

drivers/md/dm-delay.c

Quoted 10 lines
drivers/md/dm-delay.c new file +7 -0
+
+	up(&dc->timer_lock);
+}
 
...
 
+
+	init_timer(&dc->delay_timer);
+	dc->delay_timer.function = handle_delayed_timer;
+	dc->delay_timer.data = (unsigned long)dc;

setup_timer() could be used here.

Quoted 16 lines
drivers/md/dm-delay.c new file +13 -0
+	INIT_WORK(&dc->flush_expired_bios, flush_expired_bios);
+	INIT_LIST_HEAD(&dc->delayed_bios);
+	init_MUTEX(&dc->timer_lock);
+	atomic_set(&dc->may_delay, 1);
+
+	ti->private = dc;
+	return 0;
+
+bad:
+	kfree(dc);
+	return -EINVAL;
+}
+
 
...
 

drivers/md/dm-delay.c line 256

Quoted 10 lines
drivers/md/dm-delay.c new file +10 -0
+
+static int __init dm_delay_init(void)
+{
+	int r = -ENOMEM;
+
+	kdelayd_wq = create_workqueue("kdelayd");
+	if (!kdelayd_wq) {
+		DMERR("Couldn't start kdelayd");
+		goto bad_queue;
+	}

Do we really really need one instance of kdelayd on each CPU?

I suspect a single-threaded workqueue would suffice here. We have a big global lock in this driver anyway....

Probably create_workqueue() should have defaulted to single-threaded on day one. Oh well.

drivers/md/dm-delay.c line 344

Quoted 4 lines
drivers/md/dm-delay.c new file +4 -0
+	delayed_cache = kmem_cache_create("dm-delay",
+					  sizeof(struct delay_info),
+					  __alignof__(struct delay_info),
+					  0, NULL, NULL);

We have an ugly^Wnice new KMEM_CACHE macro for this now.

What's that __alignof__ doing in there?

Quoted 21 lines
drivers/md/dm-delay.c new file +21 -0
+	if (!delayed_cache) {
+		DMERR("Couldn't create delayed bio cache.");
+		goto bad_memcache;
+	}
+
+	r = dm_register_target(&delay_target);
+	if (r < 0) {
+		DMERR("register failed %d", r);
+		goto bad_register;
+	}
+
+	return 0;
+
+bad_register:
+	kmem_cache_destroy(delayed_cache);
+bad_memcache:
+	destroy_workqueue(kdelayd_wq);
+bad_queue:
+	return r;
+}
+
#2 lore raw
Quoted 152 lines · On Tue, 8 May 2007 17:54:10 -0700 Andrew Morton <[email protected]>…
Quoted 21 lines · On Tue, 8 May 2007 20:48:59 +0100 Alasdair G Kergon <[email protected]> wrote:

From: Heinz Mauelshagen <[email protected]>

New device-mapper target that can delay I/O (for testing). Reads can be separated from writes, redirected to different underlying devices and delayed by differing amounts of time.

..

drivers/md/dm-delay.c new file +13 -0
+/*
+ * Copyright (C) 2005-2007 Red Hat GmbH
+ *
+ * A target that delays reads and/or writes and can send
+ * them to different devices.
+ *
+ * This file is released under the GPL.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/blkdev.h>
+#include <linux/bio.h>

drivers/md/dm-delay.c line 23

Quoted 10 lines
drivers/md/dm-delay.c new file +10 -0
+#include <linux/slab.h>
+
+#include "dm.h"
+#include "dm-bio-list.h"
+
+#define DM_MSG_PREFIX "delay"
+
+struct delay_c {
+	struct timer_list delay_timer;
+	struct semaphore timer_lock;

Can we get this converted to a mutex asap, please?

It's only used in a single place and perhaps we don't need this lock at all?

Quoted 41 lines
drivers/md/dm-delay.c new file +41 -0
+	struct work_struct flush_expired_bios;
+	struct list_head delayed_bios;
+	atomic_t may_delay;
+	mempool_t *delayed_pool;
+
+	struct dm_dev *dev_read;
+	sector_t start_read;
+	unsigned read_delay;
+	unsigned reads;
+
+	struct dm_dev *dev_write;
+	sector_t start_write;
+	unsigned write_delay;
+	unsigned writes;
+};
+
+struct delay_info {
+	struct delay_c *context;
+	struct list_head list;
+	struct bio *bio;
+	unsigned long expires;
+};
+
+static DEFINE_MUTEX(delayed_bios_lock);
+
+static struct workqueue_struct *kdelayd_wq;
+static struct kmem_cache *delayed_cache;
+
+static void handle_delayed_timer(unsigned long data)
+{
+	struct delay_c *dc = (struct delay_c *)data;
+
+	queue_work(kdelayd_wq, &dc->flush_expired_bios);
+}
+
+static void queue_timeout(struct delay_c *dc, unsigned long expires)
+{
+	down(&dc->timer_lock);
+
+	if (!timer_pending(&dc->delay_timer) || expires < dc->delay_timer.expires)
+		mod_timer(&dc->delay_timer, expires);

drivers/md/dm-delay.c

Quoted 10 lines
drivers/md/dm-delay.c new file +7 -0
+
+	up(&dc->timer_lock);
+}
 
...
 
+
+	init_timer(&dc->delay_timer);
+	dc->delay_timer.function = handle_delayed_timer;
+	dc->delay_timer.data = (unsigned long)dc;

setup_timer() could be used here.

Quoted 16 lines
drivers/md/dm-delay.c new file +13 -0
+	INIT_WORK(&dc->flush_expired_bios, flush_expired_bios);
+	INIT_LIST_HEAD(&dc->delayed_bios);
+	init_MUTEX(&dc->timer_lock);
+	atomic_set(&dc->may_delay, 1);
+
+	ti->private = dc;
+	return 0;
+
+bad:
+	kfree(dc);
+	return -EINVAL;
+}
+
 
...
 

drivers/md/dm-delay.c line 256

Quoted 10 lines
drivers/md/dm-delay.c new file +10 -0
+
+static int __init dm_delay_init(void)
+{
+	int r = -ENOMEM;
+
+	kdelayd_wq = create_workqueue("kdelayd");
+	if (!kdelayd_wq) {
+		DMERR("Couldn't start kdelayd");
+		goto bad_queue;
+	}

Do we really really need one instance of kdelayd on each CPU?

I suspect a single-threaded workqueue would suffice here. We have a big global lock in this driver anyway....

Probably create_workqueue() should have defaulted to single-threaded on day one. Oh well.

drivers/md/dm-delay.c line 344

Quoted 4 lines
drivers/md/dm-delay.c new file +4 -0
+	delayed_cache = kmem_cache_create("dm-delay",
+					  sizeof(struct delay_info),
+					  __alignof__(struct delay_info),
+					  0, NULL, NULL);

We have an ugly^Wnice new KMEM_CACHE macro for this now.

What's that __alignof__ doing in there?

Quoted 13 lines
drivers/md/dm-delay.c new file +13 -0
+	if (!delayed_cache) {
+		DMERR("Couldn't create delayed bio cache.");
+		goto bad_memcache;
+	}
+
+	r = dm_register_target(&delay_target);
+	if (r < 0) {
+		DMERR("register failed %d", r);
+		goto bad_register;
+	}
+
+	return 0;
+
Quoted 9 lines

drivers/md/dm-delay.c line 365

Quoted 8 lines
drivers/md/dm-delay.c new file +8 -0
+bad_register:
+	kmem_cache_destroy(delayed_cache);
+bad_memcache:
+	destroy_workqueue(kdelayd_wq);
+bad_queue:
+	return r;
+}
+

Guys, could I please get some response to this before I forget about it? The new per-cpu kernel thread problem is particularly important. We already fail to boot on 4096-CPU due to running out of pids.

Thanks.

#3 lore raw
Quoted 6 lines · Andrew Morton
Quoted 6 lines

drivers/md/dm-delay.c line 344

Quoted 4 lines
drivers/md/dm-delay.c new file +4 -0
+	delayed_cache = kmem_cache_create("dm-delay",
+					  sizeof(struct delay_info),
+					  __alignof__(struct delay_info),
+					  0, NULL, NULL);

We have an ugly^Wnice new KMEM_CACHE macro for this now.

How could I nicify the macro? Lower case?

#4 lore raw
Quoted 10 lines · On Thu, 10 May 2007 12:04:50 -0700 (PDT) Christoph Lameter <[email protected]>…
Quoted 6 lines · Andrew Morton
Quoted 6 lines

drivers/md/dm-delay.c line 344

Quoted 4 lines
drivers/md/dm-delay.c new file +4 -0
+	delayed_cache = kmem_cache_create("dm-delay",
+					  sizeof(struct delay_info),
+					  __alignof__(struct delay_info),
+					  0, NULL, NULL);

We have an ugly^Wnice new KMEM_CACHE macro for this now.

How could I nicify the macro? Lower case?

Lower case would be worse, IMO. Upper-case says "this is a macro". Lower-case macros are OK when they are macros pretending to be functions, but KMEM_CACHE is not such a thing.

I don't think it can be nicified, except by removing it. We generally prefer to open-code things rather than wrapping them up in macros.

But given the amount of variation amongst callers and the number of times this got screwed up, I agree that burying those details at a single site makes sense in this instance.