Enhancement to md multipath in 2.4
PATCH 1/1 Enhancement to md multipath in 2.4
Morning all,
Jens Axboe did most of the work on this; I only stressed it a bit and fixed some bugs in it. As he is currently on vacation, I would still like to present it to you and solicit comments on it.
It compiles and passes my test script, so it can't be all wrong, I hope ;-) It certainly isn't worse than the current code.
I've also done a small patch to mdadm to allow access to the new functionality provided.
The enhancements include:
- Read & write balancing with front and tail merging of requests, otherwise picks least used path. (Done by Jens)
- Paths can be set to either active or spare; a spare path will be used in place of a failed active path but otherwise be disabled.
- A path can be manually "cleared" (marked non-faulty). This is explicitly only implemented for multipathing because it makes no sense for the other RAID levels where this is definetely the job of the recovery process.
Automatic reprobing of failed paths was deliberately not implemented; this can be done in user space, and the kernel shouldn't use live requests to do so.
- Some special cases in md.c for multipath were removed / fixed.
- md will now enable all paths it finds during autorun. This leads to "funny" messages ("Device changed to [07:04] from [00:00]" etc), but they can be safely ignored.
- Nested md devices are now also auto-detected; important for RAID1 on top of multipath for example, required for a true disaster resilient configuration. However, this isn't yet working perfectly and is subject to ongoing work ;-)
(If anyone has hints here, I would be grateful)
- Killed some code which made no sense for the multipath module; ie, code related to the md recovery.
- The downside: We needed to add 3 additional ioctl()s for this.
Patch attached.
Of course, this is still subject to the general comments about the block device error handling in 2.4.
Sincerely,
Lars Marowsky-Brée <[email protected]>
Signature · 2 lines
Immortality is an adequate definition of high availability for me. --- Gregory F. Pfister
drivers/md/md.c +116 -47
@@ -608,6 +608,7 @@md_list_add (&rdev->same_set, &mddev->disks); rdev->mddev = mddev; + rdev->desc_nr = mddev->nb_dev; mddev->nb_dev++;printk (KERN_INFO "md: bind<%s,%d>\n" ,partition_name (rdev->dev), mddev->nb_dev); }
@@ -782,7 +783,7 @@static void print_rdev (mdk_rdev_t *rdev) { -printk (KERN_INFO "md: rdev %s: O:%s, SZ:%08ld F:%d DN:%d " , +printk (KERN_INFO "md: rdev %s: O:%s, SZ:%08ld F:%d DN:%d\n" ,partition_name (rdev->dev),partition_name (rdev->old_dev), rdev->size, rdev->faulty, rdev->desc_nr);if (rdev->sb) {
@@ -984,7 +985,7 @@struct md_list_head *tmp; ITERATE_RDEV(mddev,rdev,tmp) { -if (rdev->faulty || rdev->alias_device) +if (rdev->faulty ||rdev_is_alias (rdev))continue ; sb = rdev->sb; *sb = *mddev->sb;
@@ -1036,11 +1037,11 @@printk (KERN_INFO "md: " );if (rdev->faulty)printk ("(skipping faulty " ); -if (rdev->alias_device) +if (rdev_is_alias (rdev))printk ("(skipping alias " );printk ("%s " ,partition_name (rdev->dev)); -if (!rdev->faulty && !rdev->alias_device) { +if (!rdev->faulty && !rdev_is_alias (rdev)) {printk ("[events: %08lx]" , (unsigned long )rdev->sb->events_lo); err +=write_disk_sb (rdev);
@@ -1221,7 +1222,7 @@ goto abort; sb = mddev->sb; freshest =NULL ; - + ITERATE_RDEV(mddev,rdev,tmp) {__u64 ev1, ev2;/*
@@ -1285,10 +1286,9 @@* Fix up changed device names ... but only if this disk has a * recent update time. Use faulty checksum ones too. */ -if (mddev->sb->level !=-4 ) ITERATE_RDEV(mddev,rdev,tmp) {__u64 ev1, ev2, ev3; -if (rdev->faulty || rdev->alias_device) { +if (rdev->faulty) {MD_BUG (); goto abort; }
@@ -1306,11 +1306,14 @@MD_BUG (); goto abort; } + desc = &sb->disks[rdev->desc_nr]; -if (rdev->old_dev !=MKDEV (desc->major, desc->minor)) { -MD_BUG (); - goto abort; +if ((mddev->sb->level !=-4 ) && (rdev->old_dev + !=MKDEV (desc->major, desc->minor))) { +MD_BUG (); + goto abort; } + desc->major =MAJOR (rdev->dev); desc->minor =MINOR (rdev->dev); desc = &rdev->sb->this_disk;
@@ -1386,12 +1389,11 @@* we cannot check rdev->number. * We can check the device though. */ -if ((sb->level ==-4 ) && (rdev->dev == -MKDEV (desc->major,desc->minor))) { +if ((sb->level ==-4 ) && + (rdev->dev == dev)) { found =1 ;break ; - } -if (rdev->desc_nr == desc->number) { + }else if (rdev->desc_nr == desc->number) { found =1 ;break ; }
@@ -1426,6 +1428,8 @@ rdev =find_rdev (mddev, dev);if (!rdev) { +if (mddev->sb->level ==-4 ) +continue ;MD_BUG (); goto abort; }
@@ -1436,15 +1440,7 @@* in the superblock: */ if (mddev->sb->level ==-4 ) { -if ((rdev->desc_nr !=-1 ) && (rdev->desc_nr != i)) { -MD_BUG (); - goto abort; - } rdev->desc_nr = i; -if (!first) - rdev->alias_device =1 ; -else - first =0 ; } }
@@ -1460,31 +1456,29 @@/* * Do a final reality check. */ -if (mddev->sb->level !=-4 ) { - ITERATE_RDEV(mddev,rdev,tmp) { -if (rdev->desc_nr ==-1 ) { + ITERATE_RDEV(mddev,rdev,tmp) { +if (rdev->desc_nr ==-1 ) { +MD_BUG (); + goto abort; + } +/* +* is the desc_nr unique? +*/ + ITERATE_RDEV(mddev,rdev2,tmp2) { +if ((rdev2 != rdev) && + (rdev2->desc_nr == rdev->desc_nr)) {MD_BUG (); goto abort; } -/* -* is the desc_nr unique? -*/ - ITERATE_RDEV(mddev,rdev2,tmp2) { -if ((rdev2 != rdev) && - (rdev2->desc_nr == rdev->desc_nr)) { -MD_BUG (); - goto abort; - } - } -/* -* is the device unique? -*/ - ITERATE_RDEV(mddev,rdev2,tmp2) { -if ((rdev2 != rdev) && - (rdev2->dev == rdev->dev)) { -MD_BUG (); - goto abort; - } + } +/* +* is the device unique? +*/ + ITERATE_RDEV(mddev,rdev2,tmp2) { +if ((rdev2 != rdev) && + (rdev2->dev == rdev->dev)) { +MD_BUG (); + goto abort; } } }
@@ -1909,6 +1903,28 @@*/ mddev->sb_dirty =0 ;do_md_stop (mddev,0 ); + }else { +/* Create an rdev for the freshly started md device +* and add to the end of the list */ +kdev_t dev =MKDEV (MD_MAJOR ,mddev->__minor); +if (md_import_device (dev,1 )) { +printk ("md: no nested md device found\n" ); + }else { + rdev =find_rdev_all (dev); +/* This should all be impossible because we _just_ +* imported the device! */ +if (!rdev) { +MD_BUG (); +return ; + } +if (rdev->faulty) { +MD_BUG (); +return ; + } +printk ("md: added md%d to the autodetection\n" , +mdidx (mddev)); +md_list_add (&rdev->pending, pending_raid_disks.prev); + } } }
@@ -2337,10 +2353,9 @@return -EINVAL ; } disk = &mddev->sb->disks[rdev->desc_nr]; -if (disk_active (disk)) { -MD_BUG (); +if (disk_active (disk)) goto busy; - } +if (disk_removed (disk)) {MD_BUG ();return -EINVAL ;
@@ -2368,6 +2383,50 @@return -EBUSY ; } +/* +* find the maching mdk_disk_t and pass to diskops +*/ +static int set_mp_params (mddev_t * mddev,kdev_t dev,int state) +{ +mdp_disk_t *disk; +mdk_rdev_t *rdev; + +if (!mddev->pers) +return -ENODEV ; +if (!mddev->pers->diskop) +return -ENXIO ; + + rdev =find_rdev (mddev, dev); +if (!rdev) +return -ENXIO ; + +if (rdev->desc_nr ==-1 ) { +MD_BUG (); +return -EINVAL ; + } + + disk = &mddev->sb->disks[rdev->desc_nr]; + +#if 0 +printk ("set_mp_params: state: %d dev: %s disk: %s (%d)\n" , + state, +partition_name (dev), +partition_name (MKDEV (disk->major,disk->minor)), + disk->number); +print_rdev (rdev); +#endif +switch (state) { +case SET_DISK_ACTIVE : +return mddev->pers->diskop (mddev, &disk,DISKOP_ACTIVATE_PATH ); +case SET_DISK_INACTIVE : +return mddev->pers->diskop (mddev, &disk,DISKOP_DISABLE_PATH ); +case SET_DISK_CLEAN : +return mddev->pers->diskop (mddev, &disk,DISKOP_CLEAN_PATH ); + } + +return -EINVAL ; +} +static int hot_add_disk (mddev_t * mddev,kdev_t dev) {int i, err, persistent;
@@ -2859,6 +2918,12 @@ goto done_unlock; } +case SET_DISK_CLEAN : +case SET_DISK_ACTIVE : +case SET_DISK_INACTIVE : + err =set_mp_params (mddev, (kdev_t ) arg, cmd); + goto done_unlock; +default : printk(KERN_WARNING "md: %s(pid %d) used obsolete MD ioctl, " "upgrade your software to use new ictls.\n",
@@ -3272,6 +3337,7 @@ } disk = &sb->disks[rdev->desc_nr];if (disk_faulty (disk)) { +print_desc (disk);MD_BUG ();continue ; }
@@ -3511,6 +3577,8 @@ sb = mddev->sb;if (!sb)continue ; +if (sb->level ==-4 ) +continue ;if (mddev->recovery_running)continue ;if (sb->active_disks == sb->raid_disks)
@@ -4037,6 +4105,7 @@MD_EXPORT_SYMBOL (md_wakeup_thread);MD_EXPORT_SYMBOL (md_print_devices);MD_EXPORT_SYMBOL (find_rdev_nr); +MD_EXPORT_SYMBOL (find_rdev);MD_EXPORT_SYMBOL (md_interrupt_thread);MD_EXPORT_SYMBOL (mddev_map);MD_EXPORT_SYMBOL (md_check_ordering);
drivers/md/multipath.c +409 -154
793 lines not on this page · raw message
include/linux/raid/md_k.h +8 -1
@@ -173,12 +173,16 @@mdp_super_t *sb;unsigned long sb_offset; -int alias_device;/* device alias to the same disk */ int faulty;/* if faulty do not issue IO requests */ int desc_nr;/* descriptor index in the superblock */ }; +static inline int rdev_is_alias (mdk_rdev_t * rdev) +{ +return ((rdev->sb->level ==-4 ) && (rdev->desc_nr !=0 )); +} +/* * disk operations in a working array: */
@@ -187,6 +191,9 @@#define DISKOP_SPARE_ACTIVE 2#define DISKOP_HOT_REMOVE_DISK 3#define DISKOP_HOT_ADD_DISK 4 +#define DISKOP_ACTIVATE_PATH 5 +#define DISKOP_DISABLE_PATH 6 +#define DISKOP_CLEAN_PATH 7typedef struct mdk_personality_s mdk_personality_t ;
include/linux/raid/md_u.h +5 -0
@@ -44,6 +44,11 @@#define STOP_ARRAY_RO _IO (MD_MAJOR, 0x33)#define RESTART_ARRAY_RW _IO (MD_MAJOR, 0x34) +/* multi-path extensions */ +#define SET_DISK_CLEAN _IO (MD_MAJOR, 0x35) +#define SET_DISK_ACTIVE _IO (MD_MAJOR, 0x36) +#define SET_DISK_INACTIVE _IO (MD_MAJOR, 0x37) + typedef struct mdu_version_s { int major; int minor;
include/linux/raid/multipath.h +6 -0
@@ -15,6 +15,11 @@int spare;int used_slot; + +atomic_t nr_pending_buffers; +unsigned long last_start_sector; +unsigned long last_end_sector; +int last_rw; };struct multipath_private_data {
@@ -63,6 +68,7 @@struct buffer_head *master_bh;struct buffer_head bh_req;struct multipath_bh *next_mp;/* next for retry or in free list */ +struct multipath_info *multipath; };/* bits for multipath_bh.state */ #define MPBH_Uptodate 1