block/replication.c: Properly attach children
PATCH 1/1 block/replication.c: Properly attach children
The replication driver needs access to the children block-nodes of it's child so it can issue bdrv_make_empty to manage the replication. However, it does this by directly copying the BdrvChilds, which is wrong.
Fix this by properly attaching the block-nodes with bdrv_attach_child().
Also, remove a workaround introduced in commit 6ecbc6c52672db5c13805735ca02784879ce8285 "replication: Avoid blk_make_empty() on read-only child".
---
-v2: Test for BDRV_CHILD_PRIMARY in replication_child_perm, since
bs->file might not be set yet. (Vladimir)
block/replication.c | 94 +++++++++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 33 deletions(-)
block/replication.c +61 -33
@@ -166,7 +166,12 @@ static void replication_child_perm(BlockDriverState *bs, BdrvChild *c, uint64_t perm, uint64_t shared, uint64_t *nperm, uint64_t *nshared) { - *nperm =BLK_PERM_CONSISTENT_READ ; + if (role & BDRV_CHILD_PRIMARY) { + *nperm =BLK_PERM_CONSISTENT_READ ; + } else { + *nperm =0 ; + } +if ((bs->open_flags & (BDRV_O_INACTIVE |BDRV_O_RDWR )) ==BDRV_O_RDWR ) { *nperm |=BLK_PERM_WRITE ; }
@@ -340,17 +345,7 @@ static void secondary_do_checkpoint(BDRVReplicationState *s, Error **errp)return ; } -BlockBackend *blk =blk_new (qemu_get_current_aio_context (), -BLK_PERM_WRITE ,BLK_PERM_ALL ); -blk_insert_bs (blk, s->hidden_disk->bs, &local_err); -if (local_err) { -error_propagate (errp, local_err); -blk_unref (blk); -return ; - } - - ret =blk_make_empty (blk, errp); -blk_unref (blk); + ret =bdrv_make_empty (s->hidden_disk, errp);if (ret <0 ) {return ; }
@@ -365,27 +360,35 @@ static void reopen_backing_file(BlockDriverState *bs, bool writable, Error **errp) {BDRVReplicationState *s = bs->opaque; +BdrvChild *hidden_disk, *secondary_disk;BlockReopenQueue *reopen_queue =NULL ; +/* +* s->hidden_disk and s->secondary_disk may not be set yet, as they will +* only be set after the children are writable. +*/ + hidden_disk = bs->file->bs->backing; + secondary_disk = hidden_disk->bs->backing; +if (writable) { - s->orig_hidden_read_only =bdrv_is_read_only (s->hidden_disk->bs); - s->orig_secondary_read_only =bdrv_is_read_only (s->secondary_disk->bs); + s->orig_hidden_read_only =bdrv_is_read_only (hidden_disk->bs); + s->orig_secondary_read_only =bdrv_is_read_only (secondary_disk->bs); } -bdrv_subtree_drained_begin (s->hidden_disk->bs); -bdrv_subtree_drained_begin (s->secondary_disk->bs); +bdrv_subtree_drained_begin (hidden_disk->bs); +bdrv_subtree_drained_begin (secondary_disk->bs);if (s->orig_hidden_read_only) {QDict *opts =qdict_new ();qdict_put_bool (opts,BDRV_OPT_READ_ONLY , !writable); - reopen_queue =bdrv_reopen_queue (reopen_queue, s->hidden_disk->bs, + reopen_queue =bdrv_reopen_queue (reopen_queue, hidden_disk->bs, opts, true); }if (s->orig_secondary_read_only) {QDict *opts =qdict_new ();qdict_put_bool (opts,BDRV_OPT_READ_ONLY , !writable); - reopen_queue =bdrv_reopen_queue (reopen_queue, s->secondary_disk->bs, + reopen_queue =bdrv_reopen_queue (reopen_queue, secondary_disk->bs, opts, true); }
@@ -393,8 +396,8 @@ static void reopen_backing_file(BlockDriverState *bs, bool writable,bdrv_reopen_multiple (reopen_queue, errp); } -bdrv_subtree_drained_end (s->hidden_disk->bs); -bdrv_subtree_drained_end (s->secondary_disk->bs); +bdrv_subtree_drained_end (hidden_disk->bs); +bdrv_subtree_drained_end (secondary_disk->bs); }static void backup_job_cleanup (BlockDriverState *bs)
@@ -451,6 +454,7 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode,BlockDriverState *bs = rs->opaque;BDRVReplicationState *s;BlockDriverState *top_bs; +BdrvChild *active_disk, *hidden_disk, *secondary_disk;int64_t active_length, hidden_length, disk_length;AioContext *aio_context;Error *local_err =NULL ;
@@ -488,32 +492,32 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode,case REPLICATION_MODE_PRIMARY :break ;case REPLICATION_MODE_SECONDARY : - s->active_disk = bs->file; -if (!s->active_disk || !s->active_disk->bs || - !s->active_disk->bs->backing) { + active_disk = bs->file; +if (!active_disk || !active_disk->bs || + !active_disk->bs->backing) {error_setg (errp,"Active disk doesn't have backing file" );aio_context_release (aio_context);return ; } - s->hidden_disk = s->active_disk->bs->backing; -if (!s->hidden_disk->bs || !s->hidden_disk->bs->backing) { + hidden_disk = active_disk->bs->backing; +if (!hidden_disk->bs || !hidden_disk->bs->backing) {error_setg (errp,"Hidden disk doesn't have backing file" );aio_context_release (aio_context);return ; } - s->secondary_disk = s->hidden_disk->bs->backing; -if (!s->secondary_disk->bs || !bdrv_has_blk (s->secondary_disk->bs)) { + secondary_disk = hidden_disk->bs->backing; +if (!secondary_disk->bs || !bdrv_has_blk (secondary_disk->bs)) {error_setg (errp,"The secondary disk doesn't have block backend" );aio_context_release (aio_context);return ; }/* verify the length */ - active_length =bdrv_getlength (s->active_disk->bs); - hidden_length =bdrv_getlength (s->hidden_disk->bs); - disk_length =bdrv_getlength (s->secondary_disk->bs); + active_length =bdrv_getlength (active_disk->bs); + hidden_length =bdrv_getlength (hidden_disk->bs); + disk_length =bdrv_getlength (secondary_disk->bs);if (active_length <0 || hidden_length <0 || disk_length <0 || active_length != hidden_length || hidden_length != disk_length) { error_setg(errp, "Active disk, hidden disk, secondary disk's length"
@@ -523,10 +527,10 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, }/* Must be true, or the bdrv_getlength() calls would have failed */ -assert (s->active_disk->bs->drv && s->hidden_disk->bs->drv); +assert (active_disk->bs->drv && hidden_disk->bs->drv); -if (!s->active_disk->bs->drv->bdrv_make_empty || - !s->hidden_disk->bs->drv->bdrv_make_empty) { +if (!active_disk->bs->drv->bdrv_make_empty || + !hidden_disk->bs->drv->bdrv_make_empty) {error_setg (errp,"Active disk or hidden disk doesn't support make_empty" );aio_context_release (aio_context);
@@ -541,6 +545,28 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode,return ; } + s->active_disk = active_disk; + +bdrv_ref (hidden_disk->bs); + s->hidden_disk =bdrv_attach_child (bs, hidden_disk->bs,"hidden disk" , + &child_of_bds,BDRV_CHILD_DATA , + &local_err); +if (local_err) { +error_propagate (errp, local_err); +aio_context_release (aio_context); +return ; + } + +bdrv_ref (secondary_disk->bs); + s->secondary_disk =bdrv_attach_child (bs, secondary_disk->bs, +"secondary disk" , &child_of_bds, +BDRV_CHILD_DATA , &local_err); +if (local_err) { +error_propagate (errp, local_err); +aio_context_release (aio_context); +return ; + } +/* start backup job now */ error_setg (&s->blocker,"Block device is in use by internal backup job" );
@@ -646,7 +672,9 @@ static void replication_done(void *opaque, int ret) s->stage =BLOCK_REPLICATION_DONE ; s->active_disk =NULL ; +bdrv_unref_child (bs, s->secondary_disk); s->secondary_disk =NULL ; +bdrv_unref_child (bs, s->hidden_disk); s->hidden_disk =NULL ; s->error =0 ; }else {
Signature · 1 line
2.20.1
Quoted 26 lines · Lukas Straub
The replication driver needs access to the children block-nodes of it's child so it can issue
bdrv_make_emptyto manage the replication. However, it does this by directly copying the BdrvChilds, which is wrong.Fix this by properly attaching the block-nodes with
bdrv_attach_child().Also, remove a workaround introduced in commit
6ecbc6c52672db5c13805735ca02784879ce8285"replication: Avoidblk_make_empty()on read-only child".Signed-off-by Lukas Straub <[email protected]>---
(quoted diff) +0
-1-v2: Test for BDRV_CHILD_PRIMARY in replication_child_perm, since bs->file might not be set yet. (Vladimir) block/replication.c | 94 +++++++++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 33 deletions(-)block/replication.c +0
-0
block/replication.c static void replication_child_perm(BlockDriverState *bs, BdrvChild *c, · line 173
Quoted 9 lines
block/replication.c +5
-1@@ -166,7 +166,12 @@ static void replication_child_perm(BlockDriverState *bs, BdrvChild *c, uint64_t perm, uint64_t shared, uint64_t *nperm, uint64_t *nshared) { - *nperm =BLK_PERM_CONSISTENT_READ ; + if (role & BDRV_CHILD_PRIMARY) { + *nperm =BLK_PERM_CONSISTENT_READ ; + } else { + *nperm =0 ; + }
Why you drop READ access for other children? You don't mention it in commit-msg..
Upd: ok now I see that we are not going to read from hidden_disk child, and that's the only "other" child that worth to mention. Still, we should be sure that hidden_disk child gets WRITE permission in case we are going to call bdrv_make_empty on it.
Quoted 10 lines
block/replication.c +1
-2+if ((bs->open_flags & (BDRV_O_INACTIVE |BDRV_O_RDWR )) ==BDRV_O_RDWR ) { *nperm |=BLK_PERM_WRITE ; }@@ -340,17 +345,7 @@ static void secondary_do_checkpoint(BDRVReplicationState *s, Error **errp)return ; } -BlockBackend *blk =blk_new (qemu_get_current_aio_context (), -BLK_PERM_WRITE ,BLK_PERM_ALL );
block/replication.c static void secondary_do_checkpoint(BDRVReplicationState *s, Error **errp) · line 348
Quoted 10 lines
block/replication.c +1
-9-blk_insert_bs (blk, s->hidden_disk->bs, &local_err); -if (local_err) { -error_propagate (errp, local_err); -blk_unref (blk); -return ; - } - - ret =blk_make_empty (blk, errp); -blk_unref (blk); + ret =bdrv_make_empty (s->hidden_disk, errp);
So, here you rely on BLK_PERM_WRITE being set in replication_child_perm().. Probably that's OK, however logic is changed. Shouldn't we always require write permission in replication_child_perm() for hidden_disk ?
Quoted 18 lines
block/replication.c +8
-0if (ret <0 ) {return ; }@@ -365,27 +360,35 @@ static void reopen_backing_file(BlockDriverState *bs, bool writable, Error **errp) {BDRVReplicationState *s = bs->opaque; +BdrvChild *hidden_disk, *secondary_disk;BlockReopenQueue *reopen_queue =NULL ; +/* +* s->hidden_disk and s->secondary_disk may not be set yet, as they will +* only be set after the children are writable. +*/ + hidden_disk = bs->file->bs->backing; + secondary_disk = hidden_disk->bs->backing; +if (writable) {
block/replication.c static void reopen_backing_file(BlockDriverState *bs, bool writable, · line 379
Quoted 10 lines
block/replication.c +4
-4- s->orig_hidden_read_only =bdrv_is_read_only (s->hidden_disk->bs); - s->orig_secondary_read_only =bdrv_is_read_only (s->secondary_disk->bs); + s->orig_hidden_read_only =bdrv_is_read_only (hidden_disk->bs); + s->orig_secondary_read_only =bdrv_is_read_only (secondary_disk->bs); } -bdrv_subtree_drained_begin (s->hidden_disk->bs); -bdrv_subtree_drained_begin (s->secondary_disk->bs); +bdrv_subtree_drained_begin (hidden_disk->bs); +bdrv_subtree_drained_begin (secondary_disk->bs);
That kind of staff may be done as a separate preparation patch, with no-logic-change refactoring, this makes the main logic-change patch simpler.
Quoted 98 lines
block/replication.c +19
-17if (s->orig_hidden_read_only) {QDict *opts =qdict_new ();qdict_put_bool (opts,BDRV_OPT_READ_ONLY , !writable); - reopen_queue =bdrv_reopen_queue (reopen_queue, s->hidden_disk->bs, + reopen_queue =bdrv_reopen_queue (reopen_queue, hidden_disk->bs, opts, true); }if (s->orig_secondary_read_only) {QDict *opts =qdict_new ();qdict_put_bool (opts,BDRV_OPT_READ_ONLY , !writable); - reopen_queue =bdrv_reopen_queue (reopen_queue, s->secondary_disk->bs, + reopen_queue =bdrv_reopen_queue (reopen_queue, secondary_disk->bs, opts, true); }, probably it could be a separate patch if it is needed.@@ -393,8 +396,8 @@ static void reopen_backing_file(BlockDriverState *bs, bool writable,bdrv_reopen_multiple (reopen_queue, errp); } -bdrv_subtree_drained_end (s->hidden_disk->bs); -bdrv_subtree_drained_end (s->secondary_disk->bs); +bdrv_subtree_drained_end (hidden_disk->bs); +bdrv_subtree_drained_end (secondary_disk->bs); }static void backup_job_cleanup (BlockDriverState *bs)@@ -451,6 +454,7 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode,BlockDriverState *bs = rs->opaque;BDRVReplicationState *s;BlockDriverState *top_bs; +BdrvChild *active_disk, *hidden_disk, *secondary_disk;int64_t active_length, hidden_length, disk_length;AioContext *aio_context;Error *local_err =NULL ;@@ -488,32 +492,32 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode,case REPLICATION_MODE_PRIMARY :break ;case REPLICATION_MODE_SECONDARY : - s->active_disk = bs->file; -if (!s->active_disk || !s->active_disk->bs || - !s->active_disk->bs->backing) { + active_disk = bs->file; +if (!active_disk || !active_disk->bs || + !active_disk->bs->backing) {error_setg (errp,"Active disk doesn't have backing file" );aio_context_release (aio_context);return ; } - s->hidden_disk = s->active_disk->bs->backing; -if (!s->hidden_disk->bs || !s->hidden_disk->bs->backing) { + hidden_disk = active_disk->bs->backing; +if (!hidden_disk->bs || !hidden_disk->bs->backing) {error_setg (errp,"Hidden disk doesn't have backing file" );aio_context_release (aio_context);return ; } - s->secondary_disk = s->hidden_disk->bs->backing; -if (!s->secondary_disk->bs || !bdrv_has_blk (s->secondary_disk->bs)) { + secondary_disk = hidden_disk->bs->backing; +if (!secondary_disk->bs || !bdrv_has_blk (secondary_disk->bs)) {error_setg (errp,"The secondary disk doesn't have block backend" );aio_context_release (aio_context);return ; } , probably it could be a separate patch if it is needed./* verify the length */ - active_length =bdrv_getlength (s->active_disk->bs); - hidden_length =bdrv_getlength (s->hidden_disk->bs); - disk_length =bdrv_getlength (s->secondary_disk->bs); + active_length =bdrv_getlength (active_disk->bs); + hidden_length =bdrv_getlength (hidden_disk->bs); + disk_length =bdrv_getlength (secondary_disk->bs);if (active_length <0 || hidden_length <0 || disk_length <0 || active_length != hidden_length || hidden_length != disk_length) { error_setg(errp, "Active disk, hidden disk, secondary disk's length"@@ -523,10 +527,10 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, }/* Must be true, or the bdrv_getlength() calls would have failed */ -assert (s->active_disk->bs->drv && s->hidden_disk->bs->drv); +assert (active_disk->bs->drv && hidden_disk->bs->drv); -if (!s->active_disk->bs->drv->bdrv_make_empty || - !s->hidden_disk->bs->drv->bdrv_make_empty) { +if (!active_disk->bs->drv->bdrv_make_empty || + !hidden_disk->bs->drv->bdrv_make_empty) {error_setg (errp,"Active disk or hidden disk doesn't support make_empty" );aio_context_release (aio_context);@@ -541,6 +545,28 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode,return ; } + s->active_disk = active_disk;
block/replication.c static void replication_start(ReplicationState *rs, ReplicationMode mode, · line 558
Quoted 10 lines
block/replication.c +10
-0+ +bdrv_ref (hidden_disk->bs); + s->hidden_disk =bdrv_attach_child (bs, hidden_disk->bs,"hidden disk" , + &child_of_bds,BDRV_CHILD_DATA , + &local_err); +if (local_err) { +error_propagate (errp, local_err); +aio_context_release (aio_context); +return ; + }
Ok, the point of creating hidden_disk is to call bdrv_make_empty on it.
block/replication.c static void replication_start(ReplicationState *rs, ReplicationMode mode, · line 568
Quoted 10 lines
block/replication.c +10
-0+ +bdrv_ref (secondary_disk->bs); + s->secondary_disk =bdrv_attach_child (bs, secondary_disk->bs, +"secondary disk" , &child_of_bds, +BDRV_CHILD_DATA , &local_err); +if (local_err) { +error_propagate (errp, local_err); +aio_context_release (aio_context); +return ; + }
But s->secondary_disk child is actually unused.. No reason to create it.
Quoted 6 lines
block/replication.c +1
-0+/* start backup job now */ error_setg (&s->blocker,"Block device is in use by internal backup job" );
block/replication.c static void replication_done(void *opaque, int ret) · line 680
Quoted 9 lines
block/replication.c +2
-0@@ -646,7 +672,9 @@ static void replication_done(void *opaque, int ret) s->stage =BLOCK_REPLICATION_DONE ; s->active_disk =NULL ; +bdrv_unref_child (bs, s->secondary_disk); s->secondary_disk =NULL ; +bdrv_unref_child (bs, s->hidden_disk); s->hidden_disk =NULL ; s->error =0 ; }else {
For me it looks like the good way to update is:
1. drop s->active_disk. it seems to be just a copy of bs->file, better to use bs->file directly, like other drivers do.
2. reduce usage of s->hidden_disk and s->secondary_disk, like you do in this patch, using local variables instead. Also probably just drop s->secondary_disk..
3. introduce a child, to be used with bdrv_make_empty(s->hidden_disk)
And these are 3 separate patches. 1 and 2 may be merged, or instead 2 may be divided into two (to refactor secondary_disk and hidden_disk separately)..
Still, I'm not a maintainer of replication, neither I have good understanding of how it works, so don't take my advises to heart :)
Signature · 2 lines
Best regards, Vladimir
Hi,
Thanks for your review. More below.
Btw: There is a overview of the replication design in docs/block-replication.txt
Quoted 39 lines · On Wed, 7 Jul 2021 16:01:31 +0300 Vladimir Sementsov-Ogievskiy…
Quoted 26 lines · Lukas Straub
The replication driver needs access to the children block-nodes of it's child so it can issue
bdrv_make_emptyto manage the replication. However, it does this by directly copying the BdrvChilds, which is wrong.Fix this by properly attaching the block-nodes with
bdrv_attach_child().Also, remove a workaround introduced in commit
6ecbc6c52672db5c13805735ca02784879ce8285"replication: Avoidblk_make_empty()on read-only child".Signed-off-by Lukas Straub <[email protected]>---
(quoted diff) +0
-1-v2: Test for BDRV_CHILD_PRIMARY in replication_child_perm, since bs->file might not be set yet. (Vladimir) block/replication.c | 94 +++++++++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 33 deletions(-)block/replication.c +0
-0block/replication.c static void replication_child_perm(BlockDriverState *bs, BdrvChild *c, · line 173
Quoted 9 lines
block/replication.c +5
-1@@ -166,7 +166,12 @@ static void replication_child_perm(BlockDriverState *bs, BdrvChild *c, uint64_t perm, uint64_t shared, uint64_t *nperm, uint64_t *nshared) { - *nperm =BLK_PERM_CONSISTENT_READ ; + if (role & BDRV_CHILD_PRIMARY) { + *nperm =BLK_PERM_CONSISTENT_READ ; + } else { + *nperm =0 ; + }Why you drop READ access for other children? You don't mention it in commit-msg..
Quoted 2 lines
Upd: ok now I see that we are not going to read from
hidden_diskchild, and that's the only "other" child that worth to mention. Still, we should be sure thathidden_diskchild gets WRITE permission in case we are going to callbdrv_make_emptyon it.
The code below that in replication_child_perm() should make sure of that or am i misunderstanding it?
Or do you mean that it should always request WRITE regardless of bs->open_flags & BDRV_O_INACTIVE?
Quoted 47 lines
Quoted 10 lines
block/replication.c +1
-2+if ((bs->open_flags & (BDRV_O_INACTIVE |BDRV_O_RDWR )) ==BDRV_O_RDWR ) { *nperm |=BLK_PERM_WRITE ; }@@ -340,17 +345,7 @@ static void secondary_do_checkpoint(BDRVReplicationState *s, Error **errp)return ; } -BlockBackend *blk =blk_new (qemu_get_current_aio_context (), -BLK_PERM_WRITE ,BLK_PERM_ALL );block/replication.c static void secondary_do_checkpoint(BDRVReplicationState *s, Error **errp) · line 348
Quoted 10 lines
block/replication.c +1
-9-blk_insert_bs (blk, s->hidden_disk->bs, &local_err); -if (local_err) { -error_propagate (errp, local_err); -blk_unref (blk); -return ; - } - - ret =blk_make_empty (blk, errp); -blk_unref (blk); + ret =bdrv_make_empty (s->hidden_disk, errp);So, here you rely on
BLK_PERM_WRITEbeing set inreplication_child_perm().. Probably that's OK, however logic is changed. Shouldn't we always require write permission inreplication_child_perm()forhidden_disk?Quoted 24 lines
block/replication.c +10
-2if (ret <0 ) {return ; }@@ -365,27 +360,35 @@ static void reopen_backing_file(BlockDriverState *bs, bool writable, Error **errp) {BDRVReplicationState *s = bs->opaque; +BdrvChild *hidden_disk, *secondary_disk;BlockReopenQueue *reopen_queue =NULL ; +/* +* s->hidden_disk and s->secondary_disk may not be set yet, as they will +* only be set after the children are writable. +*/ + hidden_disk = bs->file->bs->backing; + secondary_disk = hidden_disk->bs->backing; +if (writable) { - s->orig_hidden_read_only =bdrv_is_read_only (s->hidden_disk->bs); - s->orig_secondary_read_only =bdrv_is_read_only (s->secondary_disk->bs); + s->orig_hidden_read_only =bdrv_is_read_only (hidden_disk->bs); + s->orig_secondary_read_only =bdrv_is_read_only (secondary_disk->bs); }
Quoted 6 lines
block/replication.c static void reopen_backing_file(BlockDriverState *bs, bool writable, · line 379
Quoted 4 lines
block/replication.c +2
-2-bdrv_subtree_drained_begin (s->hidden_disk->bs); -bdrv_subtree_drained_begin (s->secondary_disk->bs); +bdrv_subtree_drained_begin (hidden_disk->bs); +bdrv_subtree_drained_begin (secondary_disk->bs);That kind of staff may be done as a separate preparation patch, with no-logic-change refactoring, this makes the main logic-change patch simpler.
Yes, makes sense. Will do in the next version.
Quoted 60 lines
Quoted 60 lines
block/replication.c +10
-9if (s->orig_hidden_read_only) {QDict *opts =qdict_new ();qdict_put_bool (opts,BDRV_OPT_READ_ONLY , !writable); - reopen_queue =bdrv_reopen_queue (reopen_queue, s->hidden_disk->bs, + reopen_queue =bdrv_reopen_queue (reopen_queue, hidden_disk->bs, opts, true); }if (s->orig_secondary_read_only) {QDict *opts =qdict_new ();qdict_put_bool (opts,BDRV_OPT_READ_ONLY , !writable); - reopen_queue =bdrv_reopen_queue (reopen_queue, s->secondary_disk->bs, + reopen_queue =bdrv_reopen_queue (reopen_queue, secondary_disk->bs, opts, true); }, probably it could be a separate patch if it is needed.@@ -393,8 +396,8 @@ static void reopen_backing_file(BlockDriverState *bs, bool writable,bdrv_reopen_multiple (reopen_queue, errp); } -bdrv_subtree_drained_end (s->hidden_disk->bs); -bdrv_subtree_drained_end (s->secondary_disk->bs); +bdrv_subtree_drained_end (hidden_disk->bs); +bdrv_subtree_drained_end (secondary_disk->bs); }static void backup_job_cleanup (BlockDriverState *bs)@@ -451,6 +454,7 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode,BlockDriverState *bs = rs->opaque;BDRVReplicationState *s;BlockDriverState *top_bs; +BdrvChild *active_disk, *hidden_disk, *secondary_disk;int64_t active_length, hidden_length, disk_length;AioContext *aio_context;Error *local_err =NULL ;@@ -488,32 +492,32 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode,case REPLICATION_MODE_PRIMARY :break ;case REPLICATION_MODE_SECONDARY : - s->active_disk = bs->file; -if (!s->active_disk || !s->active_disk->bs || - !s->active_disk->bs->backing) { + active_disk = bs->file; +if (!active_disk || !active_disk->bs || + !active_disk->bs->backing) {error_setg (errp,"Active disk doesn't have backing file" );aio_context_release (aio_context);return ; } - s->hidden_disk = s->active_disk->bs->backing; -if (!s->hidden_disk->bs || !s->hidden_disk->bs->backing) { + hidden_disk = active_disk->bs->backing; +if (!hidden_disk->bs || !hidden_disk->bs->backing) {error_setg (errp,"Hidden disk doesn't have backing file" );aio_context_release (aio_context);return ; }
Quoted 9 lines
block/replication.c static void replication_start(ReplicationState *rs, ReplicationMode mode,
Quoted 9 lines
block/replication.c +2
-2- s->secondary_disk = s->hidden_disk->bs->backing; -if (!s->secondary_disk->bs || !bdrv_has_blk (s->secondary_disk->bs)) { + secondary_disk = hidden_disk->bs->backing; +if (!secondary_disk->bs || !bdrv_has_blk (secondary_disk->bs)) {error_setg (errp,"The secondary disk doesn't have block backend" );aio_context_release (aio_context);return ; } , probably it could be a separate patch if it is needed.
Ok.
Quoted 53 lines
Quoted 29 lines
block/replication.c +7
-6/* verify the length */ - active_length =bdrv_getlength (s->active_disk->bs); - hidden_length =bdrv_getlength (s->hidden_disk->bs); - disk_length =bdrv_getlength (s->secondary_disk->bs); + active_length =bdrv_getlength (active_disk->bs); + hidden_length =bdrv_getlength (hidden_disk->bs); + disk_length =bdrv_getlength (secondary_disk->bs);if (active_length <0 || hidden_length <0 || disk_length <0 || active_length != hidden_length || hidden_length != disk_length) { error_setg(errp, "Active disk, hidden disk, secondary disk's length"@@ -523,10 +527,10 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, }/* Must be true, or the bdrv_getlength() calls would have failed */ -assert (s->active_disk->bs->drv && s->hidden_disk->bs->drv); +assert (active_disk->bs->drv && hidden_disk->bs->drv); -if (!s->active_disk->bs->drv->bdrv_make_empty || - !s->hidden_disk->bs->drv->bdrv_make_empty) { +if (!active_disk->bs->drv->bdrv_make_empty || + !hidden_disk->bs->drv->bdrv_make_empty) {error_setg (errp,"Active disk or hidden disk doesn't support make_empty" );aio_context_release (aio_context);@@ -541,6 +545,28 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode,return ; } + s->active_disk = active_disk;block/replication.c static void replication_start(ReplicationState *rs, ReplicationMode mode, · line 558
Quoted 10 lines
block/replication.c +10
-0+ +bdrv_ref (hidden_disk->bs); + s->hidden_disk =bdrv_attach_child (bs, hidden_disk->bs,"hidden disk" , + &child_of_bds,BDRV_CHILD_DATA , + &local_err); +if (local_err) { +error_propagate (errp, local_err); +aio_context_release (aio_context); +return ; + }Ok, the point of creating
hidden_diskis to callbdrv_make_emptyon it.block/replication.c static void replication_start(ReplicationState *rs, ReplicationMode mode, · line 568
Quoted 10 lines
block/replication.c +10
-0+ +bdrv_ref (secondary_disk->bs); + s->secondary_disk =bdrv_attach_child (bs, secondary_disk->bs, +"secondary disk" , &child_of_bds, +BDRV_CHILD_DATA , &local_err); +if (local_err) { +error_propagate (errp, local_err); +aio_context_release (aio_context); +return ; + }
Quoted 1 line
But
s->secondary_diskchild is actually unused.. No reason to create it.
It is used, look closely at replication_co_writev().
If the commit job (run during failover in replication_stop()) fails, replication enters "failover failed" state. In that state it writes directly to the secondary disk if possible (i.e. if the sector to write is not already allocated on the active or hidden disk).
It does this so the active and hidden disks don't grow larger, since in the COLO use-case they usually are put on a ramdisk with limited size.
Quoted 18 lines
Quoted 6 lines
block/replication.c +1
-0+/* start backup job now */ error_setg (&s->blocker,"Block device is in use by internal backup job" );block/replication.c static void replication_done(void *opaque, int ret) · line 680
Quoted 9 lines
block/replication.c +2
-0@@ -646,7 +672,9 @@ static void replication_done(void *opaque, int ret) s->stage =BLOCK_REPLICATION_DONE ; s->active_disk =NULL ; +bdrv_unref_child (bs, s->secondary_disk); s->secondary_disk =NULL ; +bdrv_unref_child (bs, s->hidden_disk); s->hidden_disk =NULL ; s->error =0 ; }else {For me it looks like the good way to update is:
Quoted 5 lines
1. drop
s->active_disk. it seems to be just a copy ofbs->file, better to usebs->filedirectly, like other drivers do.
2. reduce usage ofs->hidden_diskands->secondary_disk, like you do in this patch, using local variables instead. Also probably just drops->secondary_disk..
3. introduce a child, to be used withbdrv_make_empty(s->hidden_disk)And these are 3 separate patches. 1 and 2 may be merged, or instead 2 may be divided into two (to refactor
secondary_diskandhidden_diskseparately)..
Sound good, will do.
Quoted 2 lines
Still, I'm not a maintainer of replication, neither I have good understanding of how it works, so don't take my advises to heart :)
Quoted 48 lines · Lukas Straub
Hi,
Thanks for your review. More below.Btw: There is a overview of the replication design in
docs/block-replication.txtQuoted 39 lines · On Wed, 7 Jul 2021 16:01:31 +0300 Vladimir Sementsov-Ogievskiy…
Quoted 26 lines · Lukas Straub
The replication driver needs access to the children block-nodes of it's child so it can issue
bdrv_make_emptyto manage the replication. However, it does this by directly copying the BdrvChilds, which is wrong.Fix this by properly attaching the block-nodes with
bdrv_attach_child().Also, remove a workaround introduced in commit
6ecbc6c52672db5c13805735ca02784879ce8285"replication: Avoidblk_make_empty()on read-only child".Signed-off-by Lukas Straub <[email protected]>---
(quoted diff) +0
-1-v2: Test for BDRV_CHILD_PRIMARY in replication_child_perm, since bs->file might not be set yet. (Vladimir) block/replication.c | 94 +++++++++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 33 deletions(-)block/replication.c +0
-0block/replication.c static void replication_child_perm(BlockDriverState *bs, BdrvChild *c, · line 173
Quoted 9 lines
block/replication.c +5
-1@@ -166,7 +166,12 @@ static void replication_child_perm(BlockDriverState *bs, BdrvChild *c, uint64_t perm, uint64_t shared, uint64_t *nperm, uint64_t *nshared) { - *nperm =BLK_PERM_CONSISTENT_READ ; + if (role & BDRV_CHILD_PRIMARY) { + *nperm =BLK_PERM_CONSISTENT_READ ; + } else { + *nperm =0 ; + }Why you drop READ access for other children? You don't mention it in commit-msg..
Quoted 8 lines
Quoted 2 lines
Upd: ok now I see that we are not going to read from
hidden_diskchild, and that's the only "other" child that worth to mention. Still, we should be sure thathidden_diskchild gets WRITE permission in case we are going to callbdrv_make_emptyon it.The code below that in
replication_child_perm()should make sure of that or am i misunderstanding it?Or do you mean that it should always request WRITE regardless of
bs->open_flags&BDRV_O_INACTIVE?
Probably that.
I mean the following:
Do you know the circumstances when secondary_do_checkpoint() is called? With new code, could it be called when we don't have WRITE permission on hidden_disk? If it could, it's a bug.
And this patch should answer this question, because pre-patch we create blk with explicitly set BLK_PERM_WRITE. After-patch we rely on existing code in replication_child_perm().
Quoted 182 lines
Quoted 48 lines
Quoted 11 lines
block/replication.c +1
-2+if ((bs->open_flags & (BDRV_O_INACTIVE |BDRV_O_RDWR )) ==BDRV_O_RDWR ) { *nperm |=BLK_PERM_WRITE ; }@@ -340,17 +345,7 @@ static void secondary_do_checkpoint(BDRVReplicationState *s, Error **errp)return ; } -BlockBackend *blk =blk_new (qemu_get_current_aio_context (), -BLK_PERM_WRITE ,BLK_PERM_ALL );block/replication.c static void secondary_do_checkpoint(BDRVReplicationState *s, Error **errp) · line 348
Quoted 10 lines
block/replication.c +1
-9-blk_insert_bs (blk, s->hidden_disk->bs, &local_err); -if (local_err) { -error_propagate (errp, local_err); -blk_unref (blk); -return ; - } - - ret =blk_make_empty (blk, errp); -blk_unref (blk); + ret =bdrv_make_empty (s->hidden_disk, errp);So, here you rely on
BLK_PERM_WRITEbeing set inreplication_child_perm().. Probably that's OK, however logic is changed. Shouldn't we always require write permission inreplication_child_perm()forhidden_disk?Quoted 24 lines
block/replication.c +10
-2if (ret <0 ) {return ; }@@ -365,27 +360,35 @@ static void reopen_backing_file(BlockDriverState *bs, bool writable, Error **errp) {BDRVReplicationState *s = bs->opaque; +BdrvChild *hidden_disk, *secondary_disk;BlockReopenQueue *reopen_queue =NULL ; +/* +* s->hidden_disk and s->secondary_disk may not be set yet, as they will +* only be set after the children are writable. +*/ + hidden_disk = bs->file->bs->backing; + secondary_disk = hidden_disk->bs->backing; +if (writable) { - s->orig_hidden_read_only =bdrv_is_read_only (s->hidden_disk->bs); - s->orig_secondary_read_only =bdrv_is_read_only (s->secondary_disk->bs); + s->orig_hidden_read_only =bdrv_is_read_only (hidden_disk->bs); + s->orig_secondary_read_only =bdrv_is_read_only (secondary_disk->bs); }Quoted 6 lines
block/replication.c static void reopen_backing_file(BlockDriverState *bs, bool writable, · line 379
Quoted 4 lines
block/replication.c +2
-2-bdrv_subtree_drained_begin (s->hidden_disk->bs); -bdrv_subtree_drained_begin (s->secondary_disk->bs); +bdrv_subtree_drained_begin (hidden_disk->bs); +bdrv_subtree_drained_begin (secondary_disk->bs);That kind of staff may be done as a separate preparation patch, with no-logic-change refactoring, this makes the main logic-change patch simpler.
Yes, makes sense. Will do in the next version.
Quoted 60 lines
Quoted 60 lines
block/replication.c +10
-9if (s->orig_hidden_read_only) {QDict *opts =qdict_new ();qdict_put_bool (opts,BDRV_OPT_READ_ONLY , !writable); - reopen_queue =bdrv_reopen_queue (reopen_queue, s->hidden_disk->bs, + reopen_queue =bdrv_reopen_queue (reopen_queue, hidden_disk->bs, opts, true); }if (s->orig_secondary_read_only) {QDict *opts =qdict_new ();qdict_put_bool (opts,BDRV_OPT_READ_ONLY , !writable); - reopen_queue =bdrv_reopen_queue (reopen_queue, s->secondary_disk->bs, + reopen_queue =bdrv_reopen_queue (reopen_queue, secondary_disk->bs, opts, true); }, probably it could be a separate patch if it is needed.@@ -393,8 +396,8 @@ static void reopen_backing_file(BlockDriverState *bs, bool writable,bdrv_reopen_multiple (reopen_queue, errp); } -bdrv_subtree_drained_end (s->hidden_disk->bs); -bdrv_subtree_drained_end (s->secondary_disk->bs); +bdrv_subtree_drained_end (hidden_disk->bs); +bdrv_subtree_drained_end (secondary_disk->bs); }static void backup_job_cleanup (BlockDriverState *bs)@@ -451,6 +454,7 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode,BlockDriverState *bs = rs->opaque;BDRVReplicationState *s;BlockDriverState *top_bs; +BdrvChild *active_disk, *hidden_disk, *secondary_disk;int64_t active_length, hidden_length, disk_length;AioContext *aio_context;Error *local_err =NULL ;@@ -488,32 +492,32 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode,case REPLICATION_MODE_PRIMARY :break ;case REPLICATION_MODE_SECONDARY : - s->active_disk = bs->file; -if (!s->active_disk || !s->active_disk->bs || - !s->active_disk->bs->backing) { + active_disk = bs->file; +if (!active_disk || !active_disk->bs || + !active_disk->bs->backing) {error_setg (errp,"Active disk doesn't have backing file" );aio_context_release (aio_context);return ; } - s->hidden_disk = s->active_disk->bs->backing; -if (!s->hidden_disk->bs || !s->hidden_disk->bs->backing) { + hidden_disk = active_disk->bs->backing; +if (!hidden_disk->bs || !hidden_disk->bs->backing) {error_setg (errp,"Hidden disk doesn't have backing file" );aio_context_release (aio_context);return ; }Quoted 9 lines
block/replication.c static void replication_start(ReplicationState *rs, ReplicationMode mode,
Quoted 9 lines
block/replication.c +2
-2- s->secondary_disk = s->hidden_disk->bs->backing; -if (!s->secondary_disk->bs || !bdrv_has_blk (s->secondary_disk->bs)) { + secondary_disk = hidden_disk->bs->backing; +if (!secondary_disk->bs || !bdrv_has_blk (secondary_disk->bs)) {error_setg (errp,"The secondary disk doesn't have block backend" );aio_context_release (aio_context);return ; } , probably it could be a separate patch if it is needed.Ok.
Quoted 53 lines
Quoted 29 lines
block/replication.c +7
-6/* verify the length */ - active_length =bdrv_getlength (s->active_disk->bs); - hidden_length =bdrv_getlength (s->hidden_disk->bs); - disk_length =bdrv_getlength (s->secondary_disk->bs); + active_length =bdrv_getlength (active_disk->bs); + hidden_length =bdrv_getlength (hidden_disk->bs); + disk_length =bdrv_getlength (secondary_disk->bs);if (active_length <0 || hidden_length <0 || disk_length <0 || active_length != hidden_length || hidden_length != disk_length) { error_setg(errp, "Active disk, hidden disk, secondary disk's length"@@ -523,10 +527,10 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode, }/* Must be true, or the bdrv_getlength() calls would have failed */ -assert (s->active_disk->bs->drv && s->hidden_disk->bs->drv); +assert (active_disk->bs->drv && hidden_disk->bs->drv); -if (!s->active_disk->bs->drv->bdrv_make_empty || - !s->hidden_disk->bs->drv->bdrv_make_empty) { +if (!active_disk->bs->drv->bdrv_make_empty || + !hidden_disk->bs->drv->bdrv_make_empty) {error_setg (errp,"Active disk or hidden disk doesn't support make_empty" );aio_context_release (aio_context);@@ -541,6 +545,28 @@ static void replication_start(ReplicationState *rs, ReplicationMode mode,return ; } + s->active_disk = active_disk;block/replication.c static void replication_start(ReplicationState *rs, ReplicationMode mode, · line 558
Quoted 10 lines
block/replication.c +10
-0+ +bdrv_ref (hidden_disk->bs); + s->hidden_disk =bdrv_attach_child (bs, hidden_disk->bs,"hidden disk" , + &child_of_bds,BDRV_CHILD_DATA , + &local_err); +if (local_err) { +error_propagate (errp, local_err); +aio_context_release (aio_context); +return ; + }Ok, the point of creating
hidden_diskis to callbdrv_make_emptyon it.Quoted 10 lines
block/replication.c +10
-0+ +bdrv_ref (secondary_disk->bs); + s->secondary_disk =bdrv_attach_child (bs, secondary_disk->bs, +"secondary disk" , &child_of_bds, +BDRV_CHILD_DATA , &local_err); +if (local_err) { +error_propagate (errp, local_err); +aio_context_release (aio_context); +return ; + }
Quoted 3 lines
Quoted 1 line
But
s->secondary_diskchild is actually unused.. No reason to create it.It is used, look closely at
replication_co_writev().
Ah right, sorry. "base = s->secondary_disk" and next works with base. I see that now.
Quoted 40 lines
If the commit job (run during failover in
replication_stop()) fails, replication enters "failover failed" state. In that state it writes directly to the secondary disk if possible (i.e. if the sector to write is not already allocated on the active or hidden disk).It does this so the active and hidden disks don't grow larger, since in the COLO use-case they usually are put on a ramdisk with limited size.
Quoted 18 lines
Quoted 6 lines
block/replication.c +1
-0+/* start backup job now */ error_setg (&s->blocker,"Block device is in use by internal backup job" );block/replication.c static void replication_done(void *opaque, int ret) · line 680
Quoted 9 lines
block/replication.c +2
-0@@ -646,7 +672,9 @@ static void replication_done(void *opaque, int ret) s->stage =BLOCK_REPLICATION_DONE ; s->active_disk =NULL ; +bdrv_unref_child (bs, s->secondary_disk); s->secondary_disk =NULL ; +bdrv_unref_child (bs, s->hidden_disk); s->hidden_disk =NULL ; s->error =0 ; }else {For me it looks like the good way to update is:
Quoted 5 lines
1. drop
s->active_disk. it seems to be just a copy ofbs->file, better to usebs->filedirectly, like other drivers do.
2. reduce usage ofs->hidden_diskands->secondary_disk, like you do in this patch, using local variables instead. Also probably just drops->secondary_disk..
3. introduce a child, to be used withbdrv_make_empty(s->hidden_disk)And these are 3 separate patches. 1 and 2 may be merged, or instead 2 may be divided into two (to refactor
secondary_diskandhidden_diskseparately)..Sound good, will do.
Quoted 2 lines
Still, I'm not a maintainer of replication, neither I have good understanding of how it works, so don't take my advises to heart :)
Signature · 2 lines
Best regards, Vladimir