fix migration with bitmaps and mirror
v4 cover fix migration with bitmaps and mirror
v4: (Max's patch marking filters as filters already in master) 03: careful select child of filter, avoid crash
04: add Eric's r-b
05-06: tweak error message, keep Andrey's r-b, add Eric's r-b
Vladimir Sementsov-Ogievskiy (6):
migration/block-dirty-bitmap: refactor init_dirty_bitmap_migration block/dirty-bitmap: add bdrv_has_named_bitmaps helper migration/block-dirty-bitmap: fix bitmaps pre-blockdev migration
during mirror job
iotests: 194: test also migration of dirty bitmap migration/block-dirty-bitmap: add_bitmaps_to_list: check disk name
once
migration/block-dirty-bitmap: forbid migration by generated node-name
include/block/dirty-bitmap.h | 1 + block/dirty-bitmap.c | 13 ++++ migration/block-dirty-bitmap.c | 130 ++++++++++++++++++++++++--------- tests/qemu-iotests/194 | 14 +++- tests/qemu-iotests/194.out | 6 ++ 5 files changed, 127 insertions(+), 37 deletions(-)
Signature · 1 line
2.21.0
PATCH v4 1/6 migration/block-dirty-bitmap: refactor init_dirty_bitmap_migration
Split out handling one bs, it is needed for the following commit, which will handle BlockBackends separately.
--- migration/block-dirty-bitmap.c | 89 +++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 40 deletions(-)
migration/block-dirty-bitmap.c +49 -40
@@ -268,57 +268,66 @@ static void dirty_bitmap_mig_cleanup(void) }/* Called with iothread lock taken. */ -static int init_dirty_bitmap_migration (void ) +static int add_bitmaps_to_list (BlockDriverState *bs,const char *bs_name) { -BlockDriverState *bs;BdrvDirtyBitmap *bitmap;DirtyBitmapMigBitmapState *dbms;Error *local_err =NULL ; - dirty_bitmap_mig_state.bulk_completed = false; - dirty_bitmap_mig_state.prev_bs =NULL ; - dirty_bitmap_mig_state.prev_bitmap =NULL ; - dirty_bitmap_mig_state.no_bitmaps = false; + FOR_EACH_DIRTY_BITMAP(bs, bitmap) { +if (!bdrv_dirty_bitmap_name (bitmap)) { +continue ; + } -for (bs =bdrv_next_all_states (NULL ); bs; bs =bdrv_next_all_states (bs)) { -const char *name =bdrv_get_device_or_node_name (bs); +if (!bs_name ||strcmp (bs_name,"" ) ==0 ) { +error_report ("Found bitmap '%s' in unnamed node %p. It can't " +"be migrated" ,bdrv_dirty_bitmap_name (bitmap), bs); +return -1 ; + } - FOR_EACH_DIRTY_BITMAP(bs, bitmap) { -if (!bdrv_dirty_bitmap_name (bitmap)) { -continue ; - } +if (bdrv_dirty_bitmap_check (bitmap,BDRV_BITMAP_DEFAULT , &local_err)) { +error_report_err (local_err); +return -1 ; + } -if (!name ||strcmp (name,"" ) ==0 ) { -error_report ("Found bitmap '%s' in unnamed node %p. It can't " -"be migrated" ,bdrv_dirty_bitmap_name (bitmap), bs); - goto fail; - } +bdrv_ref (bs); +bdrv_dirty_bitmap_set_busy (bitmap, true); + + dbms =g_new0 (DirtyBitmapMigBitmapState,1 ); + dbms->bs = bs; + dbms->node_name = bs_name; + dbms->bitmap = bitmap; + dbms->total_sectors =bdrv_nb_sectors (bs); + dbms->sectors_per_chunk =CHUNK_SIZE *8 * +bdrv_dirty_bitmap_granularity (bitmap) >>BDRV_SECTOR_BITS ; +if (bdrv_dirty_bitmap_enabled (bitmap)) { + dbms->flags |=DIRTY_BITMAP_MIG_START_FLAG_ENABLED ; + } +if (bdrv_dirty_bitmap_get_persistence (bitmap)) { + dbms->flags |=DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT ; + } -if (bdrv_dirty_bitmap_check (bitmap,BDRV_BITMAP_DEFAULT , - &local_err)) { -error_report_err (local_err); - goto fail; - } +QSIMPLEQ_INSERT_TAIL (&dirty_bitmap_mig_state.dbms_list, + dbms, entry); + } -bdrv_ref (bs); -bdrv_dirty_bitmap_set_busy (bitmap, true); - - dbms =g_new0 (DirtyBitmapMigBitmapState,1 ); - dbms->bs = bs; - dbms->node_name = name; - dbms->bitmap = bitmap; - dbms->total_sectors =bdrv_nb_sectors (bs); - dbms->sectors_per_chunk =CHUNK_SIZE *8 * -bdrv_dirty_bitmap_granularity (bitmap) >>BDRV_SECTOR_BITS ; -if (bdrv_dirty_bitmap_enabled (bitmap)) { - dbms->flags |=DIRTY_BITMAP_MIG_START_FLAG_ENABLED ; - } -if (bdrv_dirty_bitmap_get_persistence (bitmap)) { - dbms->flags |=DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT ; - } +return 0 ; +} + +/* Called with iothread lock taken. */ +static int init_dirty_bitmap_migration (void ) +{ +BlockDriverState *bs; +DirtyBitmapMigBitmapState *dbms; + + dirty_bitmap_mig_state.bulk_completed = false; + dirty_bitmap_mig_state.prev_bs =NULL ; + dirty_bitmap_mig_state.prev_bitmap =NULL ; + dirty_bitmap_mig_state.no_bitmaps = false; -QSIMPLEQ_INSERT_TAIL (&dirty_bitmap_mig_state.dbms_list, - dbms, entry); +for (bs =bdrv_next_all_states (NULL ); bs; bs =bdrv_next_all_states (bs)) { +if (add_bitmaps_to_list (bs,bdrv_get_device_or_node_name (bs))) { + goto fail; } }
Signature · 1 line
2.21.0
PATCH v4 2/6 block/dirty-bitmap: add bdrv_has_named_bitmaps helper
In reply to Vladimir Sementsov-OgievskiyTo be used for bitmap migration in further commit.
--- include/block/dirty-bitmap.h | 1 + block/dirty-bitmap.c | 13 +++++++++++++ 2 files changed, 14 insertions(+)
include/block/dirty-bitmap.h +1 -0
@@ -95,6 +95,7 @@ int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap);void bdrv_dirty_bitmap_truncate (BlockDriverState *bs,int64_t bytes);bool bdrv_dirty_bitmap_readonly (const BdrvDirtyBitmap *bitmap);bool bdrv_has_readonly_bitmaps (BlockDriverState *bs); +bool bdrv_has_named_bitmaps (BlockDriverState *bs);bool bdrv_dirty_bitmap_get_autoload (const BdrvDirtyBitmap *bitmap);bool bdrv_dirty_bitmap_get_persistence (BdrvDirtyBitmap *bitmap);bool bdrv_dirty_bitmap_inconsistent (const BdrvDirtyBitmap *bitmap);
block/dirty-bitmap.c +13 -0
@@ -818,6 +818,19 @@ bool bdrv_has_readonly_bitmaps(BlockDriverState *bs)return false; } +bool bdrv_has_named_bitmaps (BlockDriverState *bs) +{ +BdrvDirtyBitmap *bm; + + QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) { +if (bdrv_dirty_bitmap_name (bm)) { +return true; + } + } + +return false; +} +/* Called with BQL taken. */ void bdrv_dirty_bitmap_set_persistence (BdrvDirtyBitmap *bitmap,bool persistent) {
Signature · 1 line
2.21.0
PATCH v4 3/6 migration/block-dirty-bitmap: fix bitmaps pre-blockdev migration during mirror job
In reply to Vladimir Sementsov-OgievskiyImportant thing for bitmap migration is to select destination block node to obtain the migrated bitmap.
Prepatch, on source we use bdrv_get_device_or_node_name() to identify the node, and on target we do bdrv_lookup_bs. bdrv_get_device_or_node_name() returns blk name only for direct children of blk. So, bitmaps of direct children of blks are migrated by blk name and others - by node name.
Old libvirt is unprepared to bitmap migration by node-name, node-names are mostly auto-generated. So actually only migration by blk name works for it.
Newer libvirt will use new interface (which will be added soon) to specify node-mapping for bitmaps migration explicitly. Still, let's improve the current behavior a bit.
Now, consider classic libvirt migrations assisted by mirror block job:
mirror block job inserts filter, so our source is not a direct child of blk, and bitmaps are migrated by node-names. And this just doesn't work with auto-generated node names.
Let's fix it by using blk-name even if some implicit filters are inserted.
Note2: we, of course, can't skip filters and use blk name to migrate bitmaps in filtered node by blk name for this blk if these filters have named bitmaps which should be migrated.
--- migration/block-dirty-bitmap.c | 45 +++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-)
migration/block-dirty-bitmap.c +44 -1
@@ -319,14 +319,54 @@ static int init_dirty_bitmap_migration(void) {BlockDriverState *bs;DirtyBitmapMigBitmapState *dbms; +GHashTable *handled_by_blk =g_hash_table_new (NULL ,NULL ); +BlockBackend *blk; dirty_bitmap_mig_state.bulk_completed = false; dirty_bitmap_mig_state.prev_bs =NULL ; dirty_bitmap_mig_state.prev_bitmap =NULL ; dirty_bitmap_mig_state.no_bitmaps = false; +/* +* Use blockdevice name for direct (or filtered) children of named block +* backends. +*/ +for (blk =blk_next (NULL ); blk; blk =blk_next (blk)) { +const char *name =blk_name (blk); + +if (!name ||strcmp (name,"" ) ==0 ) { +continue ; + } + + bs =blk_bs (blk); + +/* Skip filters without bitmaos */ +while (bs && bs->drv && bs->drv->is_filter && + !bdrv_has_named_bitmaps (bs)) + { +if (bs->backing) { + bs = bs->backing->bs; + }else if (bs->file) { + bs = bs->file->bs; + }else { + bs =NULL ; + } + } + +if (bs && bs->drv && !bs->drv->is_filter) { +if (add_bitmaps_to_list (bs, name)) { + goto fail; + } +g_hash_table_add (handled_by_blk, bs); + } + } +for (bs =bdrv_next_all_states (NULL ); bs; bs =bdrv_next_all_states (bs)) { -if (add_bitmaps_to_list (bs,bdrv_get_device_or_node_name (bs))) { +if (g_hash_table_contains (handled_by_blk, bs)) { +continue ; + } + +if (add_bitmaps_to_list (bs,bdrv_get_node_name (bs))) { goto fail; } }
@@ -340,9 +380,12 @@ static int init_dirty_bitmap_migration(void) dirty_bitmap_mig_state.no_bitmaps = true; } +g_hash_table_destroy (handled_by_blk); +return 0 ; fail: +g_hash_table_destroy (handled_by_blk);dirty_bitmap_mig_cleanup ();return -1 ;
Signature · 1 line
2.21.0
Re: [PATCH v4 3/6] migration/block-dirty-bitmap: fix bitmaps pre-blockdev migration during mirror job
Quoted 29 lines · Vladimir Sementsov-Ogievskiy
Important thing for bitmap migration is to select destination block node to obtain the migrated bitmap.
Prepatch, on source we use
bdrv_get_device_or_node_name()to identify the node, and on target we dobdrv_lookup_bs.bdrv_get_device_or_node_name()returns blk name only for direct children of blk. So, bitmaps of direct children of blks are migrated by blk name and others - by node name.Old libvirt is unprepared to bitmap migration by node-name, node-names are mostly auto-generated. So actually only migration by blk name works for it.
Newer libvirt will use new interface (which will be added soon) to specify node-mapping for bitmaps migration explicitly. Still, let's improve the current behavior a bit.
Now, consider classic libvirt migrations assisted by mirror block job:
mirror block job inserts filter, so our source is not a direct child of blk, and bitmaps are migrated by node-names. And this just doesn't work with auto-generated node names.Let's fix it by using blk-name even if some implicit filters are inserted.
Note2: we, of course, can't skip filters and use blk name to migrate bitmaps in filtered node by blk name for this blk if these filters have named bitmaps which should be migrated.
Quoted 5 lines
Signed-off-by Vladimir Sementsov-Ogievskiy <[email protected]>--- migration/block-dirty-bitmap.c | 45 +++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-)
Thanks; I'm queuing this series through my bitmaps tree, and will send a pull request probably on Monday.
Review
Quoted 2 lines
(quoted diff) +2
-0+ + /* Skip filters without bitmaos */
bitmaps - I'm touching that up
Signature · 3 lines
Eric Blake, Principal Software Engineer Red Hat, Inc. +1-919-301-3226 Virtualization: qemu.org | libvirt.org
PATCH v4 4/6 iotests: 194: test also migration of dirty bitmap
In reply to Vladimir Sementsov-OgievskiyTest that dirty bitmap migration works when we deal with mirror.
--- tests/qemu-iotests/194 | 14 ++++++++++---- tests/qemu-iotests/194.out | 6 ++++++ 2 files changed, 16 insertions(+), 4 deletions(-)
tests/qemu-iotests/194 +10 -4
@@ -42,6 +42,8 @@ with iotests.FilePath('source.img') as source_img_path, \ .add_incoming('unix:{0}'.format(migration_sock_path)) .launch()) + source_vm.qmp_log('block-dirty-bitmap-add', node='drive0', name='bitmap0') + iotests.log('Launching NBD server on destination...') iotests.log(dest_vm.qmp('nbd-server-start', addr={'type': 'unix', 'data': {'path': nbd_sock_path}})) iotests.log(dest_vm.qmp('nbd-server-add', device='drive0', writable=True))
@@ -61,12 +63,14 @@ with iotests.FilePath('source.img') as source_img_path, \ filters=[iotests.filter_qmp_event]) iotests.log('Starting migration...') - source_vm.qmp('migrate-set-capabilities', - capabilities=[{'capability': 'events', 'state': True}]) - dest_vm.qmp('migrate-set-capabilities', - capabilities=[{'capability': 'events', 'state': True}]) + capabilities = [{'capability': 'events', 'state': True}, + {'capability': 'dirty-bitmaps', 'state': True}] + source_vm.qmp('migrate-set-capabilities', capabilities=capabilities) + dest_vm.qmp('migrate-set-capabilities', capabilities=capabilities) iotests.log(source_vm.qmp('migrate', uri='unix:{0}'.format(migration_sock_path))) + source_vm.qmp_log('migrate-start-postcopy') + while True: event1 = source_vm.event_wait('MIGRATION') iotests.log(event1, filters=[iotests.filter_qmp_event])
@@ -82,3 +86,5 @@ with iotests.FilePath('source.img') as source_img_path, \ iotests.log('Stopping the NBD server on destination...') iotests.log(dest_vm.qmp('nbd-server-stop')) break + + iotests.log(source_vm.qmp('query-block')['return'][0]['dirty-bitmaps'])
tests/qemu-iotests/194.out +6 -0
@@ -1,4 +1,6 @@ Launching VMs... +{"execute": "block-dirty-bitmap-add", "arguments": {"name": "bitmap0", "node": "drive0"}} +{"return": {}} Launching NBD server on destination... {"return": {}} {"return": {}}
@@ -8,11 +10,15 @@ Waiting for `drive-mirror` to complete... {"data": {"device": "mirror-job0", "len": 1073741824, "offset": 1073741824, "speed": 0, "type": "mirror"}, "event": "BLOCK_JOB_READY", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} Starting migration... {"return": {}} +{"execute": "migrate-start-postcopy", "arguments": {}} +{"return": {}} {"data": {"status": "setup"}, "event": "MIGRATION", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} {"data": {"status": "active"}, "event": "MIGRATION", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} +{"data": {"status": "postcopy-active"}, "event": "MIGRATION", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} {"data": {"status": "completed"}, "event": "MIGRATION", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} Gracefully ending the `drive-mirror` job on source... {"return": {}} {"data": {"device": "mirror-job0", "len": 1073741824, "offset": 1073741824, "speed": 0, "type": "mirror"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} Stopping the NBD server on destination... {"return": {}} +[{"busy": false, "count": 0, "granularity": 65536, "name": "bitmap0", "persistent": false, "recording": true, "status": "active"}]
Signature · 1 line
2.21.0
Re: [PATCH v4 4/6] iotests: 194: test also migration of dirty bitmap
Quoted 8 lines · Vladimir Sementsov-Ogievskiy
Test that dirty bitmap migration works when we deal with mirror.
Signed-off-by Vladimir Sementsov-Ogievskiy <[email protected]>Reviewed-by Eric Blake <[email protected]>--- tests/qemu-iotests/194 | 14 ++++++++++---- tests/qemu-iotests/194.out | 6 ++++++ 2 files changed, 16 insertions(+), 4 deletions(-)
Hi!
This test broke the iotest in the gitlab CI:
https://gitlab.com/huth/qemu/-/jobs/578520599#L3780
it works again when I revert this commit.
Could the test be reworked so that it works in CI pipelines, too? Otherwise, I think it's best if we disable it in the .gitlab-ci.yml file...
Thomas
Re: [PATCH v4 4/6] iotests: 194: test also migration of dirty bitmap
Quoted 14 lines · Thomas Huth
Quoted 8 lines · Vladimir Sementsov-Ogievskiy
Test that dirty bitmap migration works when we deal with mirror.
Signed-off-by Vladimir Sementsov-Ogievskiy <[email protected]>Reviewed-by Eric Blake <[email protected]>--- tests/qemu-iotests/194 | 14 ++++++++++---- tests/qemu-iotests/194.out | 6 ++++++ 2 files changed, 16 insertions(+), 4 deletions(-)Hi!
This test broke the iotest in the gitlab CI:
Quoted 9 lines
https://gitlab.com/huth/qemu/-/jobs/578520599#L3780
it works again when I revert this commit.
Could the test be reworked so that it works in CI pipelines, too? Otherwise, I think it's best if we disable it in the .gitlab-ci.yml file...
Thomas
Hi!
from the link you provided:
...
../configure --cc=clang --enable-werror --disable-tcg --audio-drv-list=""
...
194 fail [06:31:15] [06:31:16] output mismatch (see 194.out.bad)
builds/huth/qemu/build/tests/qemu-iotests/194.out.bad renamed from builds/huth/qemu/tests/qemu-iotests/194.out +1 -0
@@ -22,3 +22,4 @@ Stopping the NBD server on destination... {"return": {}} [{"busy": false, "count": 0, "granularity": 65536, "name": "bitmap0", "persistent": false, "recording": true, "status": "active"}] +WARNING:qemu.machine:qemu received signal 6: /builds/huth/qemu/build/tests/qemu-iotests/../../x86_64-softmmu/qemu-system-x86_64 -display none -vga none -chardev socket,id=mon,path=/tmp/tmp.e9BvOjVl1W/qemudest-15756-monitor.sock -mon chardev=mon,mode=control -qtest unix:path=/tmp/tmp.e9BvOjVl1W/qemudest-15756-qtest.sock -accel qtest -nodefaults -display none -accel qtest -drive if=virtio,id=drive0,file=/builds/huth/qemu/build/tests/qemu-iotests/scratch/15756-dest.img,format=raw,cache=writeback,aio=threads -incoming unix:/tmp/tmp.e9BvOjVl1W/15756-migration.sock
- Qemu aborted. Not good. Definitely is better to fix it than just exclude the test.. I can't reproduce. Could you provide backtrace from coredump?
Signature · 2 lines
Best regards, Vladimir
Re: [PATCH v4 4/6] iotests: 194: test also migration of dirty bitmap
Quoted 22 lines · Vladimir Sementsov-Ogievskiy
Quoted 21 lines · Thomas Huth
Quoted 8 lines · Vladimir Sementsov-Ogievskiy
Test that dirty bitmap migration works when we deal with mirror.
Signed-off-by Vladimir Sementsov-Ogievskiy <[email protected]>Reviewed-by Eric Blake <[email protected]>---
tests/qemu-iotests/194 | 14 ++++++++++----
tests/qemu-iotests/194.out | 6 ++++++2 files changed, 16 insertions(+), 4 deletions(-)Hi!
This test broke the iotest in the gitlab CI:
https://gitlab.com/huth/qemu/-/jobs/578520599#L3780
it works again when I revert this commit.
Could the test be reworked so that it works in CI pipelines, too? Otherwise, I think it's best if we disable it in the .gitlab-ci.yml file...
[...]
Quoted 3 lines
- Qemu aborted. Not good. Definitely is better to fix it than just exclude the test.. I can't reproduce. Could you provide backtrace from coredump?
It aborted in block/dirty-bitmap.c, line 295, that's the "assert(!bdrv_dirty_bitmap_busy(bitmap));" if I got it right.
Full backtrace here:
https://gitlab.com/huth/qemu/-/jobs/580553686#L3638
HTH,
Thomas
Re: [PATCH v4 4/6] iotests: 194: test also migration of dirty bitmap
Quoted 28 lines · Thomas Huth
Quoted 22 lines · Vladimir Sementsov-Ogievskiy
Quoted 21 lines · Thomas Huth
Quoted 8 lines · Vladimir Sementsov-Ogievskiy
Test that dirty bitmap migration works when we deal with mirror.
Signed-off-by Vladimir Sementsov-Ogievskiy <[email protected]>Reviewed-by Eric Blake <[email protected]>--- tests/qemu-iotests/194 | 14 ++++++++++---- tests/qemu-iotests/194.out | 6 ++++++ 2 files changed, 16 insertions(+), 4 deletions(-)Hi!
This test broke the iotest in the gitlab CI:
https://gitlab.com/huth/qemu/-/jobs/578520599#L3780
it works again when I revert this commit.
Could the test be reworked so that it works in CI pipelines, too? Otherwise, I think it's best if we disable it in the .gitlab-ci.yml file...
[...]
Quoted 3 lines
- Qemu aborted. Not good. Definitely is better to fix it than just exclude the test.. I can't reproduce. Could you provide backtrace from coredump?
Quoted 7 lines
It aborted in
block/dirty-bitmap.c, line 295, that's the "assert(!bdrv_dirty_bitmap_busy(bitmap));" if I got it right.Full backtrace here:
Aha, missed it, thanks.
Hm. in 194 iotest we wait for migration finish on source, but not on target. I can assume, that in your setup target shutdown occurs earlier than migration finish, so we have busy bitmap on shutdown. It's known bug (at least for me :), patch is in list:
[PATCH v2 00/22] Fix error handling during bitmap postcopy
[..]
[PATCH v2 10/22] migration/block-dirty-bitmap: cancel migration on shutdown (https://lists.gnu.org/archive/html/qemu-devel/2020-02/msg04288.html)
If target is turned of prior to postcopy finished, target crashes
because busy bitmaps are found at shutdown.
Canceling incoming migration helps, as it removes all unfinished (and
therefore busy) bitmaps.
[..]
Still, of course iotest 194 should be fixed too, to wait for migration finish on target too, I'll send a patch.
Signature · 2 lines
Best regards, Vladimir
Re: [PATCH v4 4/6] iotests: 194: test also migration of dirty bitmap
Quoted 32 lines · Vladimir Sementsov-Ogievskiy
Quoted 31 lines · Thomas Huth
Quoted 22 lines · Vladimir Sementsov-Ogievskiy
Quoted 21 lines · Thomas Huth
Quoted 8 lines · Vladimir Sementsov-Ogievskiy
Test that dirty bitmap migration works when we deal with mirror.
Signed-off-by Vladimir Sementsov-Ogievskiy <[email protected]>Reviewed-by Eric Blake <[email protected]>---
tests/qemu-iotests/194 | 14 ++++++++++----
tests/qemu-iotests/194.out | 6 ++++++2 files changed, 16 insertions(+), 4 deletions(-)Hi!
This test broke the iotest in the gitlab CI:
https://gitlab.com/huth/qemu/-/jobs/578520599#L3780
it works again when I revert this commit.
Could the test be reworked so that it works in CI pipelines, too? Otherwise, I think it's best if we disable it in the .gitlab-ci.yml file...
[...]
Quoted 3 lines
- Qemu aborted. Not good. Definitely is better to fix it than just exclude the test.. I can't reproduce. Could you provide backtrace from coredump?
It aborted in
block/dirty-bitmap.c, line 295, that's the "assert(!bdrv_dirty_bitmap_busy(bitmap));" if I got it right.
Quoted 6 lines
Quoted 4 lines
Full backtrace here:
Aha, missed it, thanks.
No, you didn't miss it. That was a new run where I added some more debugging magic to the gitlab-ci.yml file:
build-tcg-disabled:
[...]
- make -j"$JOBS"
- sysctl -w kernel.core_pattern=/tmp/core
- ulimit -c
- cd tests/qemu-iotests/
- ./check -raw 194 || find /tmp -name "*core*"
- gdb -ex bt ../../x86_64-softmmu/qemu-system-x86_64 /tmp/core*
Quoted 3 lines
Hm. in 194 iotest we wait for migration finish on source, but not on target. I can assume, that in your setup target shutdown occurs earlier than migration finish, so we have busy bitmap on shutdown.
Sounds plausible. Especially if you consider that these gitlab CI containers only run with one virtual CPU, so there is likely more delay here.
Quoted 13 lines
It's known bug (at least for me :), patch is in list:
[PATCH v2 00/22] Fix error handling during bitmap postcopy [..]
[PATCH v2 10/22] migration/block-dirty-bitmap: cancel migration on shutdown
(https://lists.gnu.org/archive/html/qemu-devel/2020-02/msg04288.html)
If target is turned of prior to postcopy finished, target crashes because busy bitmaps are found at shutdown.
Canceling incoming migration helps, as it removes all unfinished (and therefore busy) bitmaps.
[..]
Quoted 2 lines
Still, of course iotest 194 should be fixed too, to wait for migration finish on target too, I'll send a patch.
Thanks, I'll give it a try!
Thomas
PATCH v4 5/6 migration/block-dirty-bitmap: add_bitmaps_to_list: check disk name once
In reply to Vladimir Sementsov-Ogievskiy--- migration/block-dirty-bitmap.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-)
migration/block-dirty-bitmap.c +11 -6
@@ -274,17 +274,22 @@ static int add_bitmaps_to_list(BlockDriverState *bs, const char *bs_name)DirtyBitmapMigBitmapState *dbms;Error *local_err =NULL ; + bitmap =bdrv_dirty_bitmap_first (bs); +if (!bitmap) { +return 0 ; + } + +if (!bs_name ||strcmp (bs_name,"" ) ==0 ) { +error_report ("Bitmap '%s' in unnamed node can't be migrated" , +bdrv_dirty_bitmap_name (bitmap)); +return -1 ; + } + FOR_EACH_DIRTY_BITMAP(bs, bitmap) {if (!bdrv_dirty_bitmap_name (bitmap)) {continue ; } -if (!bs_name ||strcmp (bs_name,"" ) ==0 ) { -error_report ("Found bitmap '%s' in unnamed node %p. It can't " -"be migrated" ,bdrv_dirty_bitmap_name (bitmap), bs); -return -1 ; - } -if (bdrv_dirty_bitmap_check (bitmap,BDRV_BITMAP_DEFAULT , &local_err)) {error_report_err (local_err);return -1 ;
Signature · 1 line
2.21.0
PATCH v4 6/6 migration/block-dirty-bitmap: forbid migration by generated node-name
In reply to Vladimir Sementsov-OgievskiyIt actually never worked with libvirt, as auto-generated names are different on source and destination.
It's unsafe and useless to migrate by auto-generated node-names, so let's forbid it.
--- migration/block-dirty-bitmap.c | 7 +++++++ 1 file changed, 7 insertions(+)
migration/block-dirty-bitmap.c +7 -0
@@ -285,6 +285,13 @@ static int add_bitmaps_to_list(BlockDriverState *bs, const char *bs_name)return -1 ; } +if (bs_name[0 ] =='#' ) { +error_report ("Bitmap '%s' in a node with auto-generated " +"name '%s' can't be migrated" , +bdrv_dirty_bitmap_name (bitmap), bs_name); +return -1 ; + } + FOR_EACH_DIRTY_BITMAP(bs, bitmap) {if (!bdrv_dirty_bitmap_name (bitmap)) {continue ;
Signature · 1 line
2.21.0
Quoted 5 lines · Vladimir Sementsov-Ogievskiy
v4: (Max's patch marking filters as filters already in master) 03: careful select child of filter, avoid crash
04: add Eric's r-b
05-06: tweak error message, keep Andrey's r-b, add Eric's r-b
Quoted 9 lines
Vladimir Sementsov-Ogievskiy (6):
migration/block-dirty-bitmap: refactorinit_dirty_bitmap_migrationblock/dirty-bitmap: addbdrv_has_named_bitmapshelper migration/block-dirty-bitmap: fix bitmaps pre-blockdev migration
during mirror job
iotests: 194: test also migration of dirty bitmap migration/block-dirty-bitmap:add_bitmaps_to_list: check disk name
once
migration/block-dirty-bitmap: forbid migration by generated node-name
3 and 5 have rather long subject lines, as shown by the wrapping inserted by git (3 even exceeds 80 columns on its own, even before git adds prefixing). Should I try to touch this up in my staging queue? For example:
migration: fix non-blockdev bitmap migration with mirror
doesn't lose too much information, but is definitely shorter for 3.
Signature · 3 lines
Eric Blake, Principal Software Engineer Red Hat, Inc. +1-919-301-3226 Virtualization: qemu.org | libvirt.org
Quoted 16 lines · Eric Blake
Quoted 5 lines · Vladimir Sementsov-Ogievskiy
v4: (Max's patch marking filters as filters already in master) 03: careful select child of filter, avoid crash
04: add Eric's r-b
05-06: tweak error message, keep Andrey's r-b, add Eric's r-bQuoted 9 lines
Vladimir Sementsov-Ogievskiy (6):
migration/block-dirty-bitmap: refactorinit_dirty_bitmap_migrationblock/dirty-bitmap: addbdrv_has_named_bitmapshelper migration/block-dirty-bitmap: fix bitmaps pre-blockdev migration during mirror job
iotests: 194: test also migration of dirty bitmap migration/block-dirty-bitmap:add_bitmaps_to_list: check disk name once
migration/block-dirty-bitmap: forbid migration by generated node-name
Quoted 6 lines
3 and 5 have rather long subject lines, as shown by the wrapping inserted by git (3 even exceeds 80 columns on its own, even before git adds prefixing). Should I try to touch this up in my staging queue? For example:
migration: fix non-blockdev bitmap migration with mirror
doesn't lose too much information, but is definitely shorter for 3.
No objections, of course
Signature · 2 lines
Best regards, Vladimir