dm snapshot: allow live exception store handover between tables
PATCH 1/1 dm snapshot: allow live exception store handover between tables
Permit in-use snapshot exception data to be 'handed over' from one snapshot instance to another. This is a pre-requisite for patches that allow the changes made in a snapshot device to be merged back into its origin device and also allows device resizing.
The basic call sequence is:
dmsetup load new_snapshot (referencing the existing in-use cow device)
- the ctr code detects that the cow is already in use and links the two snapshot target instances together
dmsetup suspend original_snapshot
dmsetup resume new_snapshot
- the new_snapshot becomes live, and if anything now tries to access the original one it will receive EIO
dmsetup remove original_snapshot
(There can only be two snapshot targets referencing the same cow device simultaneously.)
--- drivers/md/dm-snap.c | 233 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 225 insertions(+), 8 deletions(-)
drivers/md/dm-snap.c +225 -8
@@ -75,6 +75,26 @@ struct dm_snapshot {/* Whether or not owning mapped_device is suspended */ int suspended; +/* +* 'is_handover_destination' denotes another snapshot with the same +* cow block device (as identified with find_snapshot_using_cow) +* will hand over its exception store to this snapshot. +* +* 'is_handover_destination' is set in snapshot_ctr if an existing +* snapshot has the same cow device. The handover is performed, +* and 'is_handover_destination' is cleared, when either of the +* following occurs: +* - src (old) snapshot, that is handing over, is destructed +* - dest (new) snapshot, that is accepting the handover, is resumed +*/ +int is_handover_destination; + +/* +* reference to the other snapshot that will participate in the +* exception store handover; src references dest, dest references src +*/ +struct dm_snapshot *handover_snap; +mempool_t *pending_pool;atomic_t pending_exceptions_count;
@@ -528,6 +548,31 @@ static int dm_add_exception(void *contexreturn 0 ; } +static struct dm_snapshot *find_snapshot_using_cow (struct dm_snapshot *snap) +{ +struct dm_snapshot *s, *handover_snap =NULL ; +struct origin *o; + +down_read (&_origins_lock); + + o =__lookup_origin (snap->origin->bdev); +if (!o) + goto out; + +list_for_each_entry (s, &o->snapshots, list) { +if (s == snap || !bdev_equal (s->cow->bdev, snap->cow->bdev)) +continue ; + + handover_snap = s; +break ; + } + +out: +up_read (&_origins_lock); + +return handover_snap; +} +#define min_not_zero (l, r) (((l) == 0) ? (r) : (((r) == 0) ? (l) : min(l, r)))/*
@@ -599,11 +644,64 @@ static int init_hash_tables(struct dm_sn }/* +* Reserve snap_src for handover to snap_dest. +*/ +static int link_snapshots_for_handover (struct dm_snapshot *snap_src, +struct dm_snapshot *snap_dest) +{ +int r = -EINVAL ; + +down_write (&snap_src->lock); + +/* Another handover already set? */ +if (snap_src->handover_snap) + goto out; + + snap_src->handover_snap = snap_dest; + + snap_dest->handover_snap = snap_src; + snap_dest->is_handover_destination =1 ; + + r =0 ; + +out: +up_write (&snap_src->lock); +return r; +} + +/* +* Unreserve snap_src for handover to snap_dest. +*/ +static int unlink_snapshots_for_handover (struct dm_snapshot *snap_src, +struct dm_snapshot *snap_dest) +{ +int r = -EINVAL ; + +down_write (&snap_src->lock); + +/* make sure these snapshots are already linked */ +if ((snap_src->handover_snap != snap_dest) || + (snap_dest->handover_snap != snap_src)) + goto out; + + snap_src->handover_snap =NULL ; + + snap_dest->handover_snap =NULL ; + snap_dest->is_handover_destination =0 ; + + r =0 ; + +out: +up_write (&snap_src->lock); +return r; +} + +/* * Construct a snapshot mapping: <origin_dev> <COW-dev> <p/n> <chunk-size> */ static int snapshot_ctr (struct dm_target *ti,unsigned int argc,char **argv) { -struct dm_snapshot *s; +struct dm_snapshot *s, *handover_snap;int i;int r = -EINVAL ;char *origin_path, *cow_path;
@@ -659,6 +757,8 @@ static int snapshot_ctr(struct dm_target s->active =0 ; s->suspended =0 ;atomic_set (&s->pending_exceptions_count,0 ); + s->is_handover_destination =0 ; + s->handover_snap =NULL ;init_rwsem (&s->lock);spin_lock_init (&s->pe_lock);
@@ -694,6 +794,27 @@ static int snapshot_ctr(struct dm_targetspin_lock_init (&s->tracked_chunk_lock); +/* Does snapshot need exceptions handing over to it? */ + handover_snap =find_snapshot_using_cow (s); +if (handover_snap) { + r =link_snapshots_for_handover (handover_snap, s); +if (r) { + ti->error ="Unable to handover snapshot to " +"two devices at once." ; + goto bad_load_and_register; + } + } + +bio_list_init (&s->queued_bios); +INIT_WORK (&s->queued_bios_work, flush_queued_bios); + + ti->private = s; + ti->num_flush_requests =1 ; + +if (handover_snap) +/* register_snapshot is deferred until after handover_exceptions */ +return 0 ; +/* Metadata must only be loaded into one table at once */ r = s->store->type->read_metadata (s->store, dm_add_exception, (void *)s);
@@ -705,13 +826,11 @@ static int snapshot_ctr(struct dm_targetDMWARN ("Snapshot is marked invalid." ); } -bio_list_init (&s->queued_bios); -INIT_WORK (&s->queued_bios_work, flush_queued_bios); -if (!s->store->chunk_size) { ti->error ="Chunk size not set" ; goto bad_load_and_register; } + ti->split_io = s->store->chunk_size;/* Add snapshot to the list of snapshots for this origin */ /* Exceptions aren't triggered till snapshot_resume() is called */
@@ -721,10 +840,6 @@ static int snapshot_ctr(struct dm_target goto bad_load_and_register; } - ti->private = s; - ti->split_io = s->store->chunk_size; - ti->num_flush_requests =1 ; -return 0 ; bad_load_and_register:
@@ -765,15 +880,90 @@ static void __free_exceptions(struct dm_dm_exception_table_exit (&s->complete, exception_cache); } +static void handover_exceptions (struct dm_snapshot *old, +struct dm_snapshot *new) +{ +union { +struct dm_exception_table table_swap; +struct dm_exception_store *store_swap; + } u; + +BUG_ON ((old->handover_snap != new) || + (new->handover_snap != old)); +BUG_ON ((old->is_handover_destination !=0 ) || + (new->is_handover_destination !=1 )); +BUG_ON (!old->suspended); + +/* make sure old snapshot is still valid */ +if (!old->valid) { + new->valid =0 ; +DMERR ("Unable to handover exceptions " +"from an invalid snapshot." ); +return ; + } + +/* swap exceptions tables and stores */ + u.table_swap = new->complete; + new->complete = old->complete; + old->complete = u.table_swap; + u.store_swap = new->store; + new->store = old->store; + old->store = u.store_swap; + + new->store->snap = new; + old->store->snap = old; + +/* reset split_io to store's chunk_size */ +if (new->ti->split_io != new->store->chunk_size) + new->ti->split_io = new->store->chunk_size; + +/* Mark old snapshot invalid and inactive */ + old->valid =0 ; + old->active =0 ; + +/* Reset handover_snap references */ + old->handover_snap =NULL ; + new->handover_snap =NULL ; + + new->is_handover_destination =0 ; +} +static void snapshot_dtr (struct dm_target *ti) {#ifdef CONFIG_DM_DEBUG int i;#endif struct dm_snapshot *s = ti->private; +struct dm_snapshot *new_snap =NULL ;flush_workqueue (ksnapd); +/* This snapshot may need to handover its exception store */ +down_write (&s->lock); +if (s->handover_snap) { +if (!s->is_handover_destination) { +/* Handover exceptions to another snapshot */ + new_snap = s->handover_snap; + +down_write_nested (&new_snap->lock, +SINGLE_DEPTH_NESTING ); +handover_exceptions (s, new_snap); +up_write (&new_snap->lock); + }else { +/* allow table_clear to cancel handover */ +unlink_snapshots_for_handover (s->handover_snap, s); + } + } +/* An incomplete exception handover is not allowed */ +BUG_ON (s->handover_snap); +up_write (&s->lock); + +if (new_snap &®ister_snapshot (new_snap)) { +DMERR ("Unable to register snapshot " +"after exception handover." ); + new_snap->valid =0 ; + } +/* Prevent further origin writes from using this snapshot. */ /* After this returns there can be no new kcopyd jobs. */ unregister_snapshot (s);
@@ -1189,10 +1379,37 @@ static void snapshot_presuspend(struct dstatic void snapshot_resume (struct dm_target *ti) {struct dm_snapshot *s = ti->private; +struct dm_snapshot *old_snap, *new_snap =NULL ;down_write (&s->lock); +if (s->handover_snap) { +if (!s->is_handover_destination) { +DMERR ("Unable to handover exceptions to " +"another snapshot on resume." ); + goto out; + } +/* Get exception store from another snapshot */ + old_snap = s->handover_snap; + new_snap = s; + +down_write_nested (&old_snap->lock, +SINGLE_DEPTH_NESTING ); +handover_exceptions (old_snap, new_snap); +up_write (&old_snap->lock); + } +/* An incomplete exception handover is not allowed */ +BUG_ON (s->handover_snap); + +if (new_snap &®ister_snapshot (new_snap)) { +DMERR ("Unable to register snapshot " +"after exception handover." ); + new_snap->valid =0 ; + goto out; + } + s->active =1 ; s->suspended =0 ; +out:up_write (&s->lock); }
Quoted 16 lines · Mike Snitzer
Permit in-use snapshot exception data to be 'handed over' from one snapshot instance to another. This is a pre-requisite for patches that allow the changes made in a snapshot device to be merged back into its origin device and also allows device resizing.
The basic call sequence is:
dmsetup load
new_snapshot(referencing the existing in-use cow device)
- the ctr code detects that the cow is already in use and links the two snapshot target instances together
dmsetup suspendoriginal_snapshot
dmsetup resumenew_snapshot
- thenew_snapshotbecomes live, and if anything now tries to access the original one it will receive EIO
dmsetup removeoriginal_snapshot
Quoted 5 lines
(There can only be two snapshot targets referencing the same cow device simultaneously.)
Signed-off-by Mike Snitzer <[email protected]>Cc Mikulas Patocka <[email protected]>
Mikulas,
I removed your 'Signed-off-by' because handover has changed considerably from your initial version. Please review the new patch and reply with the appropriate tag of your chosing.
Alasdair,
Here is the incremental patch that illustrates the relevant changes that I layered ontop of the current patch you have in your tree.
. changed snapshot_ctr() to avoid read_metadata entirely for
snapshot-merge (unnecessary if handover will occur)
- adjusted a few other things in snapshot_ctr to accomodate early return in the case of handover
. new snapshot's ti->split_io is now reset in handover_exceptions() to
match the chunk_size of the store that was just handed over
. register_snapshot() for snapshot-merge snapshot is now done after
handover_exceptions rather than in snapshot_ctr()
- not done in handover_exceptions() to avoid lock order issues between
s->lock and _origins_lock
. remove support for old->resume handover
- now that snapshot_ctr no longer does read_metadata() this is no longer needed to support 'lvchange --refresh'
. allow table_clear to cancel handover (in snapshot_dtr)
--- drivers/md/dm-snap.c | 115 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 88 insertions(+), 27 deletions(-)
drivers/md/dm-snap.c +88 -27
@@ -644,7 +644,7 @@ static int init_hash_tables(struct dm_sn }/* -* Reserve snap_source for handover to snap_dest. +* Reserve snap_src for handover to snap_dest. */ static int link_snapshots_for_handover (struct dm_snapshot *snap_src,struct dm_snapshot *snap_dest)
@@ -670,6 +670,33 @@ out: }/* +* Unreserve snap_src for handover to snap_dest. +*/ +static int unlink_snapshots_for_handover (struct dm_snapshot *snap_src, +struct dm_snapshot *snap_dest) +{ +int r = -EINVAL ; + +down_write (&snap_src->lock); + +/* make sure these snapshots are already linked */ +if ((snap_src->handover_snap != snap_dest) || + (snap_dest->handover_snap != snap_src)) + goto out; + + snap_src->handover_snap =NULL ; + + snap_dest->handover_snap =NULL ; + snap_dest->is_handover_destination =0 ; + + r =0 ; + +out: +up_write (&snap_src->lock); +return r; +} + +/* * Construct a snapshot mapping: <origin_dev> <COW-dev> <p/n> <chunk-size> */ static int snapshot_ctr (struct dm_target *ti,unsigned int argc,char **argv)
@@ -778,6 +805,16 @@ static int snapshot_ctr(struct dm_target } } +bio_list_init (&s->queued_bios); +INIT_WORK (&s->queued_bios_work, flush_queued_bios); + + ti->private = s; + ti->num_flush_requests =1 ; + +if (handover_snap) +/* register_snapshot is deferred until after handover_exceptions */ +return 0 ; +/* Metadata must only be loaded into one table at once */ r = s->store->type->read_metadata (s->store, dm_add_exception, (void *)s);
@@ -789,13 +826,11 @@ static int snapshot_ctr(struct dm_targetDMWARN ("Snapshot is marked invalid." ); } -bio_list_init (&s->queued_bios); -INIT_WORK (&s->queued_bios_work, flush_queued_bios); -if (!s->store->chunk_size) { ti->error ="Chunk size not set" ; goto bad_load_and_register; } + ti->split_io = s->store->chunk_size;/* Add snapshot to the list of snapshots for this origin */ /* Exceptions aren't triggered till snapshot_resume() is called */
@@ -805,10 +840,6 @@ static int snapshot_ctr(struct dm_target goto bad_load_and_register; } - ti->private = s; - ti->split_io = s->store->chunk_size; - ti->num_flush_requests =1 ; -return 0 ; bad_load_and_register:
@@ -863,6 +894,15 @@ static void handover_exceptions(struct d (new->is_handover_destination != 1));BUG_ON (!old->suspended); +/* make sure old snapshot is still valid */ +if (!old->valid) { + new->valid =0 ; +DMERR ("Unable to handover exceptions " +"from an invalid snapshot." ); +return ; + } + +/* swap exceptions tables and stores */ u.table_swap = new->complete; new->complete = old->complete; old->complete = u.table_swap;
@@ -873,6 +913,10 @@ static void handover_exceptions(struct d new->store->snap = new; old->store->snap = old; +/* reset split_io to store's chunk_size */ +if (new->ti->split_io != new->store->chunk_size) + new->ti->split_io = new->store->chunk_size; +/* Mark old snapshot invalid and inactive */ old->valid =0 ; old->active =0 ;
@@ -890,23 +934,36 @@ static void snapshot_dtr(struct dm_targeint i;#endif struct dm_snapshot *s = ti->private; -struct dm_snapshot *new_snap; +struct dm_snapshot *new_snap =NULL ;flush_workqueue (ksnapd);/* This snapshot may need to handover its exception store */ down_write (&s->lock);if (s->handover_snap) { - new_snap = s->handover_snap; +if (!s->is_handover_destination) { +/* Handover exceptions to another snapshot */ + new_snap = s->handover_snap; -down_write_nested (&new_snap->lock,SINGLE_DEPTH_NESTING ); -handover_exceptions (s, new_snap); -up_write (&new_snap->lock); +down_write_nested (&new_snap->lock, +SINGLE_DEPTH_NESTING ); +handover_exceptions (s, new_snap); +up_write (&new_snap->lock); + }else { +/* allow table_clear to cancel handover */ +unlink_snapshots_for_handover (s->handover_snap, s); + } }/* An incomplete exception handover is not allowed */ BUG_ON (s->handover_snap);up_write (&s->lock); +if (new_snap &®ister_snapshot (new_snap)) { +DMERR ("Unable to register snapshot " +"after exception handover." ); + new_snap->valid =0 ; + } +/* Prevent further origin writes from using this snapshot. */ /* After this returns there can be no new kcopyd jobs. */ unregister_snapshot (s);
@@ -1322,33 +1379,37 @@ static void snapshot_presuspend(struct dstatic void snapshot_resume (struct dm_target *ti) {struct dm_snapshot *s = ti->private; -struct dm_snapshot *old_snap, *new_snap, *lock_snap; +struct dm_snapshot *old_snap, *new_snap =NULL ;down_write (&s->lock);if (s->handover_snap) { -/* -* Initially assumes this snapshot will get -* exception store from another snapshot -*/ +if (!s->is_handover_destination) { +DMERR ("Unable to handover exceptions to " +"another snapshot on resume." ); + goto out; + } +/* Get exception store from another snapshot */ old_snap = s->handover_snap; new_snap = s; - lock_snap = old_snap; -if (!s->is_handover_destination) { -/* Handover exceptions to another snapshot */ - old_snap = s; - new_snap = s->handover_snap; - lock_snap = new_snap; - } -down_write_nested (&lock_snap->lock, +down_write_nested (&old_snap->lock,SINGLE_DEPTH_NESTING );handover_exceptions (old_snap, new_snap); -up_write (&lock_snap->lock); +up_write (&old_snap->lock); }/* An incomplete exception handover is not allowed */ BUG_ON (s->handover_snap); + +if (new_snap &®ister_snapshot (new_snap)) { +DMERR ("Unable to register snapshot " +"after exception handover." ); + new_snap->valid =0 ; + goto out; + } + s->active =1 ; s->suspended =0 ; +out:up_write (&s->lock); }
Quoted 13 lines · Mike Snitzer
Alasdair,
Here is the incremental patch that illustrates the relevant changes that I layered ontop of the current patch you have in your tree.
. changed
snapshot_ctr()to avoidread_metadataentirely for
snapshot-merge (unnecessary if handover will occur)
- adjusted a few other things insnapshot_ctrto accomodate early return in the case of handover. new snapshot's
ti->split_iois now reset inhandover_exceptions()to
match thechunk_sizeof the store that was just handed over
Quoted 8 lines
.
register_snapshot()for snapshot-merge snapshot is now done after
handover_exceptionsrather than insnapshot_ctr()
- not done inhandover_exceptions()to avoid lock order issues betweens->lock and _origins_lock. remove support for
old->resumehandover
- now thatsnapshot_ctrno longer doesread_metadata()this is no longer needed to support 'lvchange --refresh'
OK, actually in practice 'lvchange --refresh' is:
old->suspendnew->ctrold->resume
- device-mapper: snapshots: Unable to handover exceptions to another snapshot on resume.new->resume
- snapshot_resume: handing over exceptions
So it looks like we really do need to allow snapshot_resume to trigger handover if the old is resumed before the new. (snapshot-merge is always activated last by lvm because it acts as the origin).
I added this before, and mistakenly thought it wasn't now:
http://patchwork.kernel.org/patch/57787/
Without it the old snapshot would be active with the exceptions that get handed over to the snapshot-merge. Quite bad.
Here is the incremental patch to add it back:
drivers/md/dm-snap.c +16 -10
@@ -82,9 +82,10 @@ struct dm_snapshot {* * 'is_handover_destination' is set in snapshot_ctr if an existing * snapshot has the same cow device. The handover is performed, -* and 'is_handover_destination' is cleared, when either of the +* and 'is_handover_destination' is cleared, when one of the * following occurs: * - src (old) snapshot, that is handing over, is destructed +* - src (old) snapshot, that is handing over, is resumed * - dest (new) snapshot, that is accepting the handover, is resumed */ int is_handover_destination;
@@ -1379,23 +1380,28 @@ static void snapshot_presuspend(struct dm_target *ti)static void snapshot_resume (struct dm_target *ti) {struct dm_snapshot *s = ti->private; -struct dm_snapshot *old_snap, *new_snap =NULL ; +struct dm_snapshot *lock_snap, *old_snap, *new_snap =NULL ;down_write (&s->lock);if (s->handover_snap) { -if (!s->is_handover_destination) { -DMERR ("Unable to handover exceptions to " -"another snapshot on resume." ); - goto out; - } -/* Get exception store from another snapshot */ +/* +* Initially assumes this snapshot will get +* exception store from another snapshot +*/ old_snap = s->handover_snap; new_snap = s; + lock_snap = old_snap; -down_write_nested (&old_snap->lock, +if (!s->is_handover_destination) { +/* Handover exceptions to another snapshot */ + old_snap = s; + new_snap = s->handover_snap; + lock_snap = new_snap; + } +down_write_nested (&lock_snap->lock,SINGLE_DEPTH_NESTING );handover_exceptions (old_snap, new_snap); -up_write (&old_snap->lock); +up_write (&lock_snap->lock); }/* An incomplete exception handover is not allowed */ BUG_ON (s->handover_snap);
I'll post v2 of the overall handover patch; it will include this patch.
Mike
Alasdair G Kergon
maintainerQuoted 3 lines · Mike Snitzer
OK, actually in practice 'lvchange --refresh' is:
old->suspendnew->ctr
Those two need to be swapped.
(I think it's a pre-existing userspace bug that's simple to fix.)
Quoted 4 lines
old->resume
- device-mapper: snapshots: Unable to handover exceptions to another snapshot on resume.new->resume
-snapshot_resume: handing over exceptions
Handover to new device and make it live first, then remove old device. It doesn't make sense to me to remove the old device before the new one has successfully taken over from it.
Alasdair
Quoted 7 lines · Alasdair G Kergon
Quoted 3 lines · Mike Snitzer
OK, actually in practice 'lvchange --refresh' is:
old->suspendnew->ctrThose two need to be swapped.
(I think it's a pre-existing userspace bug that's simple to fix.)
Why do they _need_ to be swapped?
Quoted 8 lines
Quoted 4 lines
old->resume
- device-mapper: snapshots: Unable to handover exceptions to another snapshot on resume.new->resume
-snapshot_resume: handing over exceptionsHandover to new device and make it live first, then remove old device. It doesn't make sense to me to remove the old device before the new one has successfully taken over from it.
OK, I'm not sure how to make that happen within lvm2's deptree. I'll have a look. So you're saying refresh should result in:
new->ctrold->suspendnew->resumeold->resume
Mike
Alasdair G Kergon
maintainerQuoted 7 lines · Mike Snitzer
Quoted 5 lines · Alasdair G Kergon
Quoted 3 lines · Mike Snitzer
OK, actually in practice 'lvchange --refresh' is:
old->suspendnew->ctrThose two need to be swapped.
Why do they _need_ to be swapped?
old->suspend stops I/O to the device.new->ctr allocates memory and in a bad case might lead to waiting for I/O to the device we've just suspended.
Alasdair
Quoted 6 lines · Alasdair G Kergon
Quoted 5 lines · Mike Snitzer
Quoted 4 lines · Alasdair G Kergon
Quoted 3 lines · Mike Snitzer
OK, actually in practice 'lvchange --refresh' is:
old->suspendnew->ctr
Quoted 6 lines
Quoted 2 lines
Quoted 1 line
Those two need to be swapped.
Why do they _need_ to be swapped?
old->suspendstops I/O to the device.new->ctrallocates memory and in a bad case might lead to waiting for I/O to the device we've just suspended.
Ouch, yes good point.
So I'll put the following on my lvm2 TODO:
- establish a REFRESH flag (set on lvchange --refresh)
- if REFRESH is set then _lv_suspend() will treat the live metadata as if it is precommitted metadata
Mike
--
lvm-devel mailing list
[email protected]
https://www.redhat.com/mailman/listinfo/lvm-devel
Mikulas Patocka
maintainerQuoted 9 lines
drivers/md/dm-snap.c +9
-0+/* +* 'is_handover_destination' denotes another snapshot with the same +* cow block device (as identified with find_snapshot_using_cow) +* will hand over its exception store to this snapshot. +* +* 'is_handover_destination' is set in snapshot_ctr if an existing +* snapshot has the same cow device. The handover is performed, +* and 'is_handover_destination' is cleared, when either of the +* following occurs:
drivers/md/dm-snap.c struct dm_snapshot { · line 96
Quoted 10 lines
drivers/md/dm-snap.c +10
-0+* - src (old) snapshot, that is handing over, is destructed +* - dest (new) snapshot, that is accepting the handover, is resumed +*/ +int is_handover_destination; + +/* +* reference to the other snapshot that will participate in the +* exception store handover; src references dest, dest references src +*/ +struct dm_snapshot *handover_snap;
Hi
Please drop snapshot-linking through this "handover_snap" field, it will create real or possible problems with locking or with dangling pointers (i.e. are both properly locked? Which to lock first? If you destroy a snapshot, you must check if something links to it, etc. --- these problems could be solved, but if they can be avoided by dropping linking, it's better to avoid them).
When you are about to do a handover (Alasdair is arranging generic dm core do handover only un resume), just search for a possible handover candidate with find_snapshot_using_cow.
Mikulas
Quoted 22 lines · Mikulas Patocka
Quoted 9 lines
drivers/md/dm-snap.c +9
-0+/* +* 'is_handover_destination' denotes another snapshot with the same +* cow block device (as identified with find_snapshot_using_cow) +* will hand over its exception store to this snapshot. +* +* 'is_handover_destination' is set in snapshot_ctr if an existing +* snapshot has the same cow device. The handover is performed, +* and 'is_handover_destination' is cleared, when either of the +* following occurs: drivers/md/dm-snap.c struct dm_snapshot { · line 96
Quoted 10 lines
drivers/md/dm-snap.c +10
-0+* - src (old) snapshot, that is handing over, is destructed +* - dest (new) snapshot, that is accepting the handover, is resumed +*/ +int is_handover_destination; + +/* +* reference to the other snapshot that will participate in the +* exception store handover; src references dest, dest references src +*/ +struct dm_snapshot *handover_snap;Hi
Quoted 6 lines
Please drop snapshot-linking through this "
handover_snap" field, it will create real or possible problems with locking or with dangling pointers (i.e. are both properly locked? Which to lock first? If you destroy a snapshot, you must check if something links to it, etc. --- these problems could be solved, but if they can be avoided by dropping linking, it's better to avoid them).
I agree, lock_snapshots_for_handover() is too complicated. But by dropping these 'handover_snap' associations we'll lose the ability to have tight constraints on what handover will take place on resume of new snapshot (ideally a source of handover will only have one destination).
The new snapshot (destination of exception handover) is not registered with the origin until after resume's handover_exceptions (via register_snapshot). And therefore it can't be found via find_snapshot_using_cow(). But that is a good thing as we'll know that if find_snapshot_using_cow does return a snapshot then it is the source of the handover.
If we keep regitration of the new snapshot late, as I believe we need to, then we do not have a way to check for a conflicting handover (in snapshot_ctr). We'd then have a situation where N new snapshots _could_ be racing to get the exceptions handed over from a single old snapshot.
So I think we'll need the snapshot to have a 'is_handover_source' flag; it would allow us to check of the 'handover_snap' in snapshot_ctr() already 'is_handover_source' and if so fail the new snapshot_ctr(). I'll look to add this after I have coded the handover to not use any state. Looking will be dead simple (just need to take the old snapshot's lock).
Quoted 3 lines
When you are about to do a handover (Alasdair is arranging generic dm core do handover only un resume), just search for a possible handover candidate with
find_snapshot_using_cow.
As I think you're aware: Alasdair's core DM changes have nothing to do with actually constraining handover to be on resume. His changes make handover on resume safe; as we can guarantee that handover on resume actually worked (and if not rollback to old map). A very welcomed advance.
Thanks for your review,
Mike
Mikulas Patocka
maintainerQuoted 14 lines
Quoted 8 lines
Hi
Please drop snapshot-linking through this "
handover_snap" field, it will create real or possible problems with locking or with dangling pointers (i.e. are both properly locked? Which to lock first? If you destroy a snapshot, you must check if something links to it, etc. --- these problems could be solved, but if they can be avoided by dropping linking, it's better to avoid them).I agree,
lock_snapshots_for_handover()is too complicated. But by dropping these 'handover_snap' associations we'll lose the ability to have tight constraints on what handover will take place on resume of new snapshot (ideally a source of handover will only have one destination).
Quoted 6 lines
The new snapshot (destination of exception handover) is not registered with the origin until after resume's
handover_exceptions(viaregister_snapshot). And therefore it can't be found viafind_snapshot_using_cow(). But that is a good thing as we'll know that iffind_snapshot_using_cowdoes return a snapshot then it is the source of the handover.
It is registered in snapshot_ctr --- that's how it was originally written. So it should be possible to find it.
You moved the registration away? So move it back.
I'm somehow concerned, how this simple handover code (that I wrote 1.5 years ago, on a train way from Berlin to Prague) is getting more and more complicated, just for handling some "can't happen" cases.
First of all, you don't have to handle user's misbehavior --- if the user writes dd if=/dev/zero of=/dev/hda, he loses his data --- there is no need to protect the user from doing it. Similarly, if the user loads bad dm table, he may lose his data too and there is little that can be done.
You can add protection from double target load as a bonus --- but do it only as long as it doesn't complicate the code. If this protection from double load would complicate the code too much, then don't do it.
Last night, when talking with Alasdair, we found out that the "active" variable could be used as a token --- there could be at most one active snapshot and on handover, you would set "active=0" on the old snapshot and "active=1" on the new snapshot, always enforcing that there is just one snapshot active.
But generally:
1. make handover work the simplest way without any protection (like I wrote it).
2. add the protection from double load only if it doesn't complicate the code too much (for example, checking that there is just one "active" snapshot should be relatively easy).
Mikulas
Quoted 26 lines
If we keep regitration of the new snapshot late, as I believe we need to, then we do not have a way to check for a conflicting handover (in
snapshot_ctr). We'd then have a situation where N new snapshots _could_ be racing to get the exceptions handed over from a single old snapshot.So I think we'll need the snapshot to have a '
is_handover_source' flag; it would allow us to check of the 'handover_snap' insnapshot_ctr()already 'is_handover_source' and if so fail the newsnapshot_ctr(). I'll look to add this after I have coded the handover to not use any state. Looking will be dead simple (just need to take the old snapshot's lock).Quoted 3 lines
When you are about to do a handover (Alasdair is arranging generic dm core do handover only un resume), just search for a possible handover candidate with
find_snapshot_using_cow.As I think you're aware: Alasdair's core DM changes have nothing to do with actually constraining handover to be on resume. His changes make handover on resume safe; as we can guarantee that handover on resume actually worked (and if not rollback to old map). A very welcomed advance.
Thanks for your review,
Mike
Quoted 21 lines · Mikulas Patocka
Quoted 14 lines
Quoted 8 lines
Hi
Please drop snapshot-linking through this "
handover_snap" field, it will create real or possible problems with locking or with dangling pointers (i.e. are both properly locked? Which to lock first? If you destroy a snapshot, you must check if something links to it, etc. --- these problems could be solved, but if they can be avoided by dropping linking, it's better to avoid them).I agree,
lock_snapshots_for_handover()is too complicated. But by dropping these 'handover_snap' associations we'll lose the ability to have tight constraints on what handover will take place on resume of new snapshot (ideally a source of handover will only have one destination).Quoted 6 lines
The new snapshot (destination of exception handover) is not registered with the origin until after resume's
handover_exceptions(viaregister_snapshot). And therefore it can't be found viafind_snapshot_using_cow(). But that is a good thing as we'll know that iffind_snapshot_using_cowdoes return a snapshot then it is the source of the handover.
Quoted 4 lines
It is registered in
snapshot_ctr--- that's how it was originally written. So it should be possible to find it.You moved the registration away? So move it back.
Why? That would be counter-productive.
register_snapshot() is skipped if handover will be performed. The reason is register_snapshot() looks at the store's chunk_size. Problem is we don't have the store yet. Your handover relied on rereading the snapshot header. I no longer read_metadata() at all if the store will be handed over.
Quoted 3 lines
I'm somehow concerned, how this simple handover code (that I wrote 1.5 years ago, on a train way from Berlin to Prague) is getting more and more complicated, just for handling some "can't happen" cases.
Believe me, I'd rather not be lingering over the finer points of exception handover; but once I hit the fact that snapshot_ctr() was imposing that the COW device must not be suspended it opened a can or worms (lvchange --refresh exposed this, in general it is fair to assume the cow device isn't suspended in snapshot_ctr :).
The handover-based snapshot_ctr() needing to access the COW device begged the followong questions:
why are we reading the snapshot header via read_metadata()? Why are we doing any IO to the COW if it will just be handed over?
Answer: we don't have a real need, so don't. Not only that, but Alasdair asserted read_metadata() must not be called if another snapshot is already actively using the COW. What if something else is changing it at the same time that the new snapshot_ctr() re-reads it?
Anyway, best to avoid all of that (read_metadata and register_snapshot) if we're doing exception handover.
Quoted 8 lines
First of all, you don't have to handle user's misbehavior --- if the user writes dd if=/dev/zero of=/dev/hda, he loses his data --- there is no need to protect the user from doing it. Similarly, if the user loads bad dm table, he may lose his data too and there is little that can be done.
You can add protection from double target load as a bonus --- but do it only as long as it doesn't complicate the code. If this protection from double load would complicate the code too much, then don't do it.
I just posted the latest version of the handover patch (v7); it no longer has 'handover_snap' pointers. As such the locking is simplified.
It also handles protecting against double target loads.
Quoted 5 lines
Last night, when talking with Alasdair, we found out that the "active" variable could be used as a token --- there could be at most one active snapshot and on handover, you would set "active=0" on the old snapshot and "active=1" on the new snapshot, always enforcing that there is just one snapshot active.
Right, find_snapshot_using_cow() in v7 of the patch does make use of active to know to overlook a snapshot on the origin's 'snapshots' list that are not active (snapshot may be left on the list after handover, snapshot_dtr() hasn't come through yet, but it is both invalid and inactive). v7 handover_exceptions() sets: snap_src->active = 0;
Quoted 6 lines
But generally:
1. make handover work the simplest way without any protection (like I wrote it).
2. add the protection from double load only if it doesn't complicate the code too much (for example, checking that there is just one "active" snapshot should be relatively easy).
I think v7 is fairly robust; I've tested it pretty extensively both standalone and with the rest of the snapshot-merge patches. As always the full snapshot-merge DM patchset is available here:
http://people.redhat.com/msnitzer/patches/snapshot-merge/kernel/2.6.32/
Mikulas Patocka
maintainerQuoted 5 lines
Quoted 4 lines
It is registered in
snapshot_ctr--- that's how it was originally written. So it should be possible to find it.You moved the registration away? So move it back.
Quoted 7 lines
Why? That would be counter-productive.
register_snapshot()is skipped if handover will be performed. The reason isregister_snapshot()looks at the store'schunk_size. Problem is we don't have the store yet. Your handover relied on rereading the snapshot header. I no longerread_metadata()at all if the store will be handed over.
If you don't want to read the header and get the chunk size from there, you can get it from the existing active snapshot. And then register the new to-be-activated snapshot with active == 0 (so that it will be skipped for exception reallocation).
Quoted 9 lines
Quoted 3 lines
I'm somehow concerned, how this simple handover code (that I wrote 1.5 years ago, on a train way from Berlin to Prague) is getting more and more complicated, just for handling some "can't happen" cases.
Believe me, I'd rather not be lingering over the finer points of exception handover; but once I hit the fact that
snapshot_ctr()was imposing that the COW device must not be suspended it opened a can or worms (lvchange --refresh exposed this, in general it is fair to assume the cow device isn't suspended insnapshot_ctr:).
No one should be calling any target constructor if there are any suspended devices. But there may be bugs in lvm2 that do that ... and you may need to work around them ...
The reason: the constructor can allocate memory and that could attempt to flush pages to the suspended devices and deadlock.
Quoted 4 lines
The handover-based
snapshot_ctr()needing to access the COW device begged the followong questions:
why are we reading the snapshot header viaread_metadata()? Why are we doing any IO to the COW if it will just be handed over?
Originaly it read the whole snapshot store :) And I optimized away the worst thing (reading exceptions) because it takes a lot of memory.
Quoted 4 lines
Answer: we don't have a real need, so don't. Not only that, but Alasdair asserted
read_metadata()must not be called if another snapshot is already actively using the COW. What if something else is changing it at the same time that the newsnapshot_ctr()re-reads it?
It won't cause trouble. It will just read incomplete list of exceptions (that will be thrown away anyway). But reading exceptions is a memory waste, my patch didn't do it too.
Mikulas