Skip to contentSkip to conversation

block layer (request based) multipath

RFC PATCH New Mike Christie 1 participant 1 message Oct 17, 2005

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 block layer (request based) multipath

The following patch implements a basic multipath driver at the block layer. It uses the dm-mutlipath code and really just uses requests instead of BIOs. I still have to finish some more trivial things like manual path onlining, manual failover support and deciding on a interface (currently using sysfs but I need to add some tests and locking if I am going to keep it - will probably switch to configfs later) but the basics are there and I wanted to get feedback before I proceed.

Why a request based multipath driver when we have MD multipath and DM multipath at the BIO layer? Good question :)

1. I have been trying to support some of the quirks in vendor's HW and to do this we need detailed error codes. If you look at the back of SCSI SPEC you will notice we have a boat load of errors codes, plus we can make up our own. And unfortunately to support special features they have used and abused these values :) Another thing the block layer needs is detailed error values, and I have tried to wedge the quirky vendor specific values into them but it has not gone so well. A generic block layer error code framework can describe a lot of errors but we cannot get the level of details needed for multipath without making the codes too SCSI and vendor specific.

By working at the request queue level we can use the BLOCK PC infrastructure (and later the generic packet one) to pass back all the error info we need for this vendor specific case. Note, for errors like transport or driver problems or generic device failures we would use the generic block layer codes when they are implemented.

2. Another quirk in storage boxes is the ability to manually fail over (sometimes this is the only way to fail over too). To do this today we are forming BLOCK PC SCSI requests from the DM multipath hw handlers and sending them from a driver that normally works on BIOs to the request queue.

If we move a layer down to the request level we can then have a common code path for sending the failover request with the Read/Write requests and we can have common code for the Read/Write and Failover vendor specific error code evaluation. In fact, this patch was a result of me adding better retry handling to dm-multipath for failover requests using the BLOCK PC infrastructure.

3. Some side benefits we get from working at the request level are:
A. Reuse the request_queue queueing capabilities for retries, congestion detection, and OOM handling (handling mempools, scheduling the next process fairly, being able to not over queue when blocked up etc).
B. IO Scheduler is in the right place (AS or deadline on top with noop underneath). C. Simplified path selectors since we no longer have to deal with BIOs merging.
D. SG_IO, Tape, CD or anyplace else that used to work with requests and could not be supported by dm-multipath can be supported - in thoery at least :)

The patch is made against 2.6.14-rc4. I have done some basic testing of this patch and failover works OK. The patch needs a lot of fixups. It is not ready for merging anywhere. And I am not asking for a formal review (will break up the patches for reviewers later) so do not waste your time. I am more concerned about getting feedback about the possible problems (I know some have said wait for 2.7, but if I am going to muck around with every block layer driver to go the error code wedging path this may be less invasive) and if the sense code passing is a gross layer violation or not. The hw_handler and path_selector code is from dm-multipath but was made to work with requests and sysfs. The major new code is in blk_mpath.c and blk_sysfs.c.

drivers/block/blk_mpath/blk_hw_handler.c +155 -0
diff -Naurp linux-2.6.14-rc4/drivers/block/blk_mpath/blk_hw_handler.c linux-2.6.14-rc4.work/drivers/block/blk_mpath/blk_hw_handler.c
@@ -0,0 +1,155 @@
+/*
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ * Copyright (C) 2005 Mike Christie All rights reserved.
+ *
+ * This file is released under the GPL.
+ *
+ * Multipath hardware handler registration.
+ */
+
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+
+#include "blk_hw_handler.h"
+
+struct hwh_internal {
+	struct hw_handler_type hwht;
+
+	struct list_head list;
+	long use;
+};
+
+#define hwht_to_hwhi(__hwht) container_of((__hwht), struct hwh_internal, hwht)
+
+static LIST_HEAD(_hw_handlers);
+static DECLARE_RWSEM(_hwh_lock);
+
+static struct hwh_internal *__find_hw_handler_type(const char *name)
+{
+	struct hwh_internal *hwhi;
+
+	list_for_each_entry(hwhi, &_hw_handlers, list) {
+		if (!strcmp(name, hwhi->hwht.name))
+			return hwhi;
+	}
+
+	return NULL;
+}
+
+static struct hwh_internal *get_hw_handler(const char *name)
+{
+	struct hwh_internal *hwhi;
+
+	down_read(&_hwh_lock);
+	hwhi = __find_hw_handler_type(name);
+	if (hwhi) {
+		if ((hwhi->use == 0) && !try_module_get(hwhi->hwht.module))
+			hwhi = NULL;
+		else
+			hwhi->use++;
+	}
+	up_read(&_hwh_lock);
+
+	return hwhi;
+}
+
+struct hw_handler_type *blk_get_hw_handler(const char *name)
+{
+	struct hwh_internal *hwhi;
+
+	if (!name)
+		return NULL;
+
+	hwhi = get_hw_handler(name);
+	if (!hwhi) {
+		request_module("blk_%s", name);
+		hwhi = get_hw_handler(name);
+	}
+
+	return hwhi ? &hwhi->hwht : NULL;
+}
+
+void blk_put_hw_handler(struct hw_handler_type *hwht)
+{
+	struct hwh_internal *hwhi;
+
+	if (!hwht)
+		return;
+
+	down_read(&_hwh_lock);
+	hwhi = __find_hw_handler_type(hwht->name);
+	if (!hwhi)
+		goto out;
+
+	if (--hwhi->use == 0)
+		module_put(hwhi->hwht.module);
+
+	if (hwhi->use < 0)
+		BUG();
+
+      out:
+	up_read(&_hwh_lock);
+}
+
+static struct hwh_internal *_alloc_hw_handler(struct hw_handler_type *hwht)
+{
+	struct hwh_internal *hwhi = kmalloc(sizeof(*hwhi), GFP_KERNEL);
+
+	if (hwhi) {
+		memset(hwhi, 0, sizeof(*hwhi));
+		hwhi->hwht = *hwht;
+	}
+
+	return hwhi;
+}
+
+int blk_register_hw_handler(struct hw_handler_type *hwht)
+{
+	int r = 0;
+	struct hwh_internal *hwhi = _alloc_hw_handler(hwht);
+
+	if (!hwhi)
+		return -ENOMEM;
+
+	down_write(&_hwh_lock);
+
+	if (__find_hw_handler_type(hwht->name)) {
+		kfree(hwhi);
+		r = -EEXIST;
+	} else
+		list_add(&hwhi->list, &_hw_handlers);
+
+	up_write(&_hwh_lock);
+
+	return r;
+}
+
+int blk_unregister_hw_handler(struct hw_handler_type *hwht)
+{
+	struct hwh_internal *hwhi;
+
+	down_write(&_hwh_lock);
+
+	hwhi = __find_hw_handler_type(hwht->name);
+	if (!hwhi) {
+		up_write(&_hwh_lock);
+		return -EINVAL;
+	}
+
+	if (hwhi->use) {
+		up_write(&_hwh_lock);
+		return -ETXTBSY;
+	}
+
+	list_del(&hwhi->list);
+
+	up_write(&_hwh_lock);
+
+	kfree(hwhi);
+
+	return 0;
+}
+
+EXPORT_SYMBOL_GPL(blk_register_hw_handler);
+EXPORT_SYMBOL_GPL(blk_unregister_hw_handler);
drivers/block/blk_mpath/blk_hw_handler.h +66 -0

66 lines not on this page · raw message

drivers/block/blk_mpath/blk_mpath.c +718 -0

718 lines not on this page · raw message

drivers/block/blk_mpath/blk_mpath.h +131 -0

131 lines not on this page · raw message

drivers/block/blk_mpath/blk_path_selector.c +158 -0

158 lines not on this page · raw message

drivers/block/blk_mpath/blk_path_selector.h +84 -0

84 lines not on this page · raw message

drivers/block/blk_mpath/blk_round_robin.c +182 -0

182 lines not on this page · raw message

drivers/block/blk_mpath/blk_sysfs.c +413 -0

413 lines not on this page · raw message

drivers/block/blk_mpath/Makefile +7 -0
diff -Naurp linux-2.6.14-rc4/drivers/block/blk_mpath/Makefile linux-2.6.14-rc4.work/drivers/block/blk_mpath/Makefile
@@ -0,0 +1,7 @@
+#
+# Makefile for block layer multipath
+#
+
+obj-$(CONFIG_BLK_MPATH) += blk_multipath.o blk_round_robin.o
+blk_multipath-objs := blk_mpath.o blk_sysfs.o blk_hw_handler.o \
+			blk_path_selector.o
drivers/block/Kconfig +6 -0
diff -Naurp linux-2.6.14-rc4/drivers/block/Kconfig linux-2.6.14-rc4.work/drivers/block/Kconfig
@@ -464,4 +464,10 @@ config ATA_OVER_ETH
 	This driver provides Support for ATA over Ethernet block
 	devices like the Coraid EtherDrive (R) Storage Blade.
 
+config BLK_MPATH
+	tristate "Block Layer Multipath"
+	depends on DM_MULTIPATH
+	help
+	  This driver provides the low level multipath handling
+
 endmenu
drivers/block/ll_rw_blk.c +8 -1
diff -Naurp linux-2.6.14-rc4/drivers/block/ll_rw_blk.c linux-2.6.14-rc4.work/drivers/block/ll_rw_blk.c
@@ -287,6 +287,7 @@ static inline void rq_init(request_queue
 	rq->nr_phys_segments = 0;
 	rq->sense = NULL;
 	rq->end_io = NULL;
+	rq->end_io_first = NULL;
 	rq->end_io_data = NULL;
 }
 
@@ -2298,6 +2299,7 @@ void blk_execute_rq_nowait(request_queue
 	rq->rq_disk = bd_disk;
 	rq->flags |= REQ_NOMERGE;
 	rq->end_io = done;
+
 	elv_add_request(q, rq, where, 1);
 	generic_unplug_device(q);
 }
@@ -2445,7 +2447,7 @@ void disk_round_stats(struct gendisk *di
 /*
  * queue lock must be held
  */
-static void __blk_put_request(request_queue_t *q, struct request *req)
+void __blk_put_request(request_queue_t *q, struct request *req)
 {
 	struct request_list *rl = req->rl;
 
@@ -2473,6 +2475,8 @@ static void __blk_put_request(request_qu
 	}
 }
 
+EXPORT_SYMBOL(__blk_put_request);
+
 void blk_put_request(struct request *req)
 {
 	/*
@@ -3110,6 +3114,9 @@ static int __end_that_request_first(stru
 	int total_bytes, bio_nbytes, error, next_idx = 0;
 	struct bio *bio;
 
+	if (req->end_io_first)
+		return req->end_io_first(req, uptodate, nr_bytes);
+
 	/*
 	 * extend uptodate bool to allow < 0 value to be direct io error
 	 */
drivers/block/Makefile +1 -0
diff -Naurp linux-2.6.14-rc4/drivers/block/Makefile linux-2.6.14-rc4.work/drivers/block/Makefile
@@ -45,3 +45,4 @@ obj-$(CONFIG_VIODASD)		+= viodasd.o
 obj-$(CONFIG_BLK_DEV_SX8)	+= sx8.o
 obj-$(CONFIG_BLK_DEV_UB)	+= ub.o
 
+obj-$(CONFIG_BLK_MPATH)		+= blk_mpath/
drivers/scsi/scsi_lib.c +2 -1
diff -Naurp linux-2.6.14-rc4/drivers/scsi/scsi_lib.c linux-2.6.14-rc4.work/drivers/scsi/scsi_lib.c
@@ -833,7 +833,8 @@ void scsi_io_completion(struct scsi_cmnd
 		if (sense_valid)
 			sense_deferred = scsi_sense_is_deferred(&sshdr);
 	}
-	if (blk_pc_request(req)) { /* SG_IO ioctl from block level */
+	/* SG_IO ioctl from block level or blk_mpath */
+	if (blk_pc_request(req) || blk_extended_err(req)) {
 		req->errors = result;
 		if (result) {
 			clear_errors = 0;
include/linux/blkdev.h +5 -0
diff -Naurp linux-2.6.14-rc4/include/linux/blkdev.h linux-2.6.14-rc4.work/include/linux/blkdev.h
@@ -103,6 +103,7 @@ void swap_io_context(struct io_context *
 
 struct request;
 typedef void (rq_end_io_fn)(struct request *);
+typedef int (rq_end_first_fn)(struct request *, int, int);
 
 struct request_list {
 	int count[2];
@@ -194,6 +195,7 @@ struct request {
 	 * completion callback. end_io_data should be folded in with waiting
 	 */
 	rq_end_io_fn *end_io;
+	rq_end_first_fn *end_io_first;
 	void *end_io_data;
 };
 
@@ -446,6 +448,7 @@ enum {
 #define blk_pc_request(rq)	((rq)->flags & REQ_BLOCK_PC)
 #define blk_noretry_request(rq)	((rq)->flags & REQ_FAILFAST)
 #define blk_rq_started(rq)	((rq)->flags & REQ_STARTED)
+#define blk_extended_err(rq)	(blk_noretry_request(rq) && rq->sense)
 
 #define blk_account_rq(rq)	(blk_rq_started(rq) && blk_fs_request(rq))
 
@@ -548,6 +551,7 @@ extern void blk_unregister_queue(struct 
 extern void register_disk(struct gendisk *dev);
 extern void generic_make_request(struct bio *bio);
 extern void blk_put_request(struct request *);
+extern void __blk_put_request(request_queue_t *, struct request *);
 extern void blk_end_sync_rq(struct request *rq);
 extern void blk_attempt_remerge(request_queue_t *, struct request *);
 extern struct request *blk_get_request(request_queue_t *, int, int);
@@ -569,6 +573,7 @@ extern int blk_rq_map_kern(request_queue
 extern int blk_rq_map_user_iov(request_queue_t *, struct request *, struct sg_iovec *, int);
 extern int blk_execute_rq(request_queue_t *, struct gendisk *,
 			  struct request *, int);
+
 static inline request_queue_t *bdev_get_queue(struct block_device *bdev)
 {
 	return bdev->bd_disk->queue;