fix migration with bitmaps and mirror
cover fix migration with bitmaps and mirror
Hi all!
It's a continuation for
"bitmap migration bug with -drive while block mirror runs" <[email protected]>
https://lists.gnu.org/archive/html/qemu-devel/2019-09/msg07241.html
The problem is that bitmaps migrated to node with same node-name or blk-parent name. And currently only the latter actually work in libvirt. And with mirror-top filter it doesn't work, because bdrv_get_device_or_node_name don't go through filters.
Fix this by handling filtered children of block backends in separate.
Max Reitz (1):
block: Mark commit and mirror as filter drivers
Vladimir Sementsov-Ogievskiy (4):
migretion/block-dirty-bitmap: refactor init_dirty_bitmap_migration block/dirty-bitmap: add bdrv_has_named_bitmaps helper migration/block-dirty-bitmap: fix bitmaps migration during mirror job
iotests: 194: test also migration of dirty bitmap
include/block/block_int.h | 8 ++- include/block/dirty-bitmap.h | 1 + block/commit.c | 2 + block/dirty-bitmap.c | 13 ++++ block/mirror.c | 2 + migration/block-dirty-bitmap.c | 118 +++++++++++++++++++++++---------- tests/qemu-iotests/194 | 14 ++-- tests/qemu-iotests/194.out | 6 ++ 8 files changed, 121 insertions(+), 43 deletions(-)
Signature · 1 line
2.21.0
PATCH 1/5 block: Mark commit and mirror as filter drivers
From: Max Reitz <[email protected]>
The commit and mirror block nodes are filters, so they should be marked as such.
--- include/block/block_int.h | 8 +++++--- block/commit.c | 2 ++ block/mirror.c | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-)
include/block/block_int.h +5 -3
@@ -89,9 +89,11 @@ struct BlockDriver {int instance_size;/* set to true if the BlockDriver is a block filter. Block filters pass -* certain callbacks that refer to data (see block.c) to their bs->file if -* the driver doesn't implement them. Drivers that do not wish to forward -* must implement them and return -ENOTSUP. +* certain callbacks that refer to data (see block.c) to their bs->file +* or bs->backing (whichever one exists) if the driver doesn't implement +* them. Drivers that do not wish to forward must implement them and return +* -ENOTSUP. +* Note that filters are not allowed to modify data. */ bool is_filter;/* for snapshots block filter like Quorum can implement the
block/commit.c +2 -0
@@ -253,6 +253,8 @@ static BlockDriver bdrv_commit_top = { .bdrv_co_block_status = bdrv_co_block_status_from_backing, .bdrv_refresh_filename = bdrv_commit_top_refresh_filename, .bdrv_child_perm = bdrv_commit_top_child_perm, + + .is_filter = true, }; void commit_start(const char *job_id, BlockDriverState *bs,
block/mirror.c +2 -0
@@ -1487,6 +1487,8 @@ static BlockDriver bdrv_mirror_top = { .bdrv_refresh_filename = bdrv_mirror_top_refresh_filename, .bdrv_child_perm = bdrv_mirror_top_child_perm, .bdrv_refresh_limits = bdrv_mirror_top_refresh_limits, + + .is_filter = true, }; static BlockJob *mirror_start_job(
Signature · 1 line
2.21.0
PATCH 2/5 migretion/block-dirty-bitmap: refactor init_dirty_bitmap_migration
In reply to Vladimir Sementsov-OgievskiySplit out handling one bs, it is needed for the following commit, which will handle BlockBackends in separate.
--- migration/block-dirty-bitmap.c | 93 +++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 42 deletions(-)
migration/block-dirty-bitmap.c +51 -42
@@ -268,59 +268,68 @@ static void dirty_bitmap_mig_cleanup(void) }/* Called with iothread lock taken. */ -static int init_dirty_bitmap_migration (void ) +static int add_bitmaps_to_list (BlockDriverState *bs,const char *bs_name) { -BlockDriverState *bs;BdrvDirtyBitmap *bitmap;DirtyBitmapMigBitmapState *dbms;Error *local_err =NULL ; - dirty_bitmap_mig_state.bulk_completed = false; - dirty_bitmap_mig_state.prev_bs =NULL ; - dirty_bitmap_mig_state.prev_bitmap =NULL ; - dirty_bitmap_mig_state.no_bitmaps = false; +for (bitmap =bdrv_dirty_bitmap_next (bs,NULL ); bitmap; + bitmap =bdrv_dirty_bitmap_next (bs, bitmap)) + { +if (!bdrv_dirty_bitmap_name (bitmap)) { +continue ; + } -for (bs =bdrv_next_all_states (NULL ); bs; bs =bdrv_next_all_states (bs)) { -const char *name =bdrv_get_device_or_node_name (bs); +if (!bs_name ||strcmp (bs_name,"" ) ==0 ) { +error_report ("Found bitmap '%s' in unnamed node %p. It can't " +"be migrated" ,bdrv_dirty_bitmap_name (bitmap), bs); +return -1 ; + } -for (bitmap =bdrv_dirty_bitmap_next (bs,NULL ); bitmap; - bitmap =bdrv_dirty_bitmap_next (bs, bitmap)) - { -if (!bdrv_dirty_bitmap_name (bitmap)) { -continue ; - } +if (bdrv_dirty_bitmap_check (bitmap,BDRV_BITMAP_DEFAULT , &local_err)) { +error_report_err (local_err); +return -1 ; + } -if (!name ||strcmp (name,"" ) ==0 ) { -error_report ("Found bitmap '%s' in unnamed node %p. It can't " -"be migrated" ,bdrv_dirty_bitmap_name (bitmap), bs); - goto fail; - } +bdrv_ref (bs); +bdrv_dirty_bitmap_set_busy (bitmap, true); + + dbms =g_new0 (DirtyBitmapMigBitmapState,1 ); + dbms->bs = bs; + dbms->node_name = bs_name; + dbms->bitmap = bitmap; + dbms->total_sectors =bdrv_nb_sectors (bs); + dbms->sectors_per_chunk =CHUNK_SIZE *8 * +bdrv_dirty_bitmap_granularity (bitmap) >>BDRV_SECTOR_BITS ; +if (bdrv_dirty_bitmap_enabled (bitmap)) { + dbms->flags |=DIRTY_BITMAP_MIG_START_FLAG_ENABLED ; + } +if (bdrv_dirty_bitmap_get_persistence (bitmap)) { + dbms->flags |=DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT ; + } -if (bdrv_dirty_bitmap_check (bitmap,BDRV_BITMAP_DEFAULT , - &local_err)) { -error_report_err (local_err); - goto fail; - } +QSIMPLEQ_INSERT_TAIL (&dirty_bitmap_mig_state.dbms_list, + dbms, entry); + } -bdrv_ref (bs); -bdrv_dirty_bitmap_set_busy (bitmap, true); - - dbms =g_new0 (DirtyBitmapMigBitmapState,1 ); - dbms->bs = bs; - dbms->node_name = name; - dbms->bitmap = bitmap; - dbms->total_sectors =bdrv_nb_sectors (bs); - dbms->sectors_per_chunk =CHUNK_SIZE *8 * -bdrv_dirty_bitmap_granularity (bitmap) >>BDRV_SECTOR_BITS ; -if (bdrv_dirty_bitmap_enabled (bitmap)) { - dbms->flags |=DIRTY_BITMAP_MIG_START_FLAG_ENABLED ; - } -if (bdrv_dirty_bitmap_get_persistence (bitmap)) { - dbms->flags |=DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT ; - } +return 0 ; +} + +/* Called with iothread lock taken. */ +static int init_dirty_bitmap_migration (void ) +{ +BlockDriverState *bs; +DirtyBitmapMigBitmapState *dbms; + + dirty_bitmap_mig_state.bulk_completed = false; + dirty_bitmap_mig_state.prev_bs =NULL ; + dirty_bitmap_mig_state.prev_bitmap =NULL ; + dirty_bitmap_mig_state.no_bitmaps = false; -QSIMPLEQ_INSERT_TAIL (&dirty_bitmap_mig_state.dbms_list, - dbms, entry); +for (bs =bdrv_next_all_states (NULL ); bs; bs =bdrv_next_all_states (bs)) { +if (add_bitmaps_to_list (bs,bdrv_get_device_or_node_name (bs))) { + goto fail; } }
Signature · 1 line
2.21.0
PATCH 3/5 block/dirty-bitmap: add bdrv_has_named_bitmaps helper
In reply to Vladimir Sementsov-OgievskiyTo be used for bitmap migration in further commit.
--- include/block/dirty-bitmap.h | 1 + block/dirty-bitmap.c | 13 +++++++++++++ 2 files changed, 14 insertions(+)
include/block/dirty-bitmap.h +1 -0
@@ -101,6 +101,7 @@ int64_t bdrv_get_meta_dirty_count(BdrvDirtyBitmap *bitmap);void bdrv_dirty_bitmap_truncate (BlockDriverState *bs,int64_t bytes);bool bdrv_dirty_bitmap_readonly (const BdrvDirtyBitmap *bitmap);bool bdrv_has_readonly_bitmaps (BlockDriverState *bs); +bool bdrv_has_named_bitmaps (BlockDriverState *bs);bool bdrv_dirty_bitmap_get_autoload (const BdrvDirtyBitmap *bitmap);bool bdrv_dirty_bitmap_get_persistence (BdrvDirtyBitmap *bitmap);bool bdrv_dirty_bitmap_inconsistent (const BdrvDirtyBitmap *bitmap);
block/dirty-bitmap.c +13 -0
@@ -742,6 +742,19 @@ bool bdrv_has_readonly_bitmaps(BlockDriverState *bs)return false; } +bool bdrv_has_named_bitmaps (BlockDriverState *bs) +{ +BdrvDirtyBitmap *bm; + + QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) { +if (bdrv_dirty_bitmap_name (bm)) { +return true; + } + } + +return false; +} +/* Called with BQL taken. */ void bdrv_dirty_bitmap_set_persistence (BdrvDirtyBitmap *bitmap,bool persistent) {
Signature · 1 line
2.21.0
PATCH 4/5 migration/block-dirty-bitmap: fix bitmaps migration during mirror job
In reply to Vladimir Sementsov-OgievskiyImportant thing for bitmap migration is to select destination block node to obtain the migrated bitmap.
Prepatch, on source we use bdrv_get_device_or_node_name() to identify the node, and on target we do bdrv_lookup_bs. bdrv_get_device_or_node_name() returns blk name only for direct children of blk. So, bitmaps of direct children of blks are migrated by blk name and others - by node name.
Libvirt currently is unprepared to bitmap migration by node-name, node-names are mostly auto-generated. So actually only migration by blk name works.
Now, consider classic libvirt migrations assisted by mirror block job:
mirror block job inserts filter, so our source is not a direct child of blk, and bitmaps are migrated by node-names. And this just don't work.
Let's fix it by allowing use blk-name even if some implicit filters are inserted.
Note, that we possibly want to allow explicit filters skipping too, but this is another story.
Note2: we, of course, can't skip filters and use blk name to migrate bitmaps in filtered node by blk name for this blk if these filters have named bitmaps which should be migrated.
--- migration/block-dirty-bitmap.c | 39 +++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-)
migration/block-dirty-bitmap.c +38 -1
@@ -321,14 +321,48 @@ static int init_dirty_bitmap_migration(void) {BlockDriverState *bs;DirtyBitmapMigBitmapState *dbms; +GHashTable *handled_by_blk =g_hash_table_new (NULL ,NULL ); +BlockBackend *blk; dirty_bitmap_mig_state.bulk_completed = false; dirty_bitmap_mig_state.prev_bs =NULL ; dirty_bitmap_mig_state.prev_bitmap =NULL ; dirty_bitmap_mig_state.no_bitmaps = false; +/* +* Use blockdevice name for direct (or filtered) children of named block +* backends. +*/ +for (blk =blk_next (NULL ); blk; blk =blk_next (blk)) { +const char *name =blk_name (blk); + +if (!name ||strcmp (name,"" ) ==0 ) { +continue ; + } + + bs =blk_bs (blk); + +/* Skip filters without bitmaos */ +while (bs && bs->drv && bs->drv->is_filter && + !bdrv_has_named_bitmaps (bs)) + { + bs = bs->backing->bs ?: bs->file->bs; + } + +if (bs && bs->drv && !bs->drv->is_filter) { +if (add_bitmaps_to_list (bs, name)) { + goto fail; + } +g_hash_table_add (handled_by_blk, bs); + } + } +for (bs =bdrv_next_all_states (NULL ); bs; bs =bdrv_next_all_states (bs)) { -if (add_bitmaps_to_list (bs,bdrv_get_device_or_node_name (bs))) { +if (g_hash_table_contains (handled_by_blk, bs)) { +continue ; + } + +if (add_bitmaps_to_list (bs,bdrv_get_node_name (bs))) { goto fail; } }
@@ -342,9 +376,12 @@ static int init_dirty_bitmap_migration(void) dirty_bitmap_mig_state.no_bitmaps = true; } +g_hash_table_destroy (handled_by_blk); +return 0 ; fail: +g_hash_table_destroy (handled_by_blk);dirty_bitmap_mig_cleanup ();return -1 ;
Signature · 1 line
2.21.0
PATCH 5/5 iotests: 194: test also migration of dirty bitmap
In reply to Vladimir Sementsov-OgievskiyTest that dirty bitmap migration works when we deal with mirror.
--- tests/qemu-iotests/194 | 14 ++++++++++---- tests/qemu-iotests/194.out | 6 ++++++ 2 files changed, 16 insertions(+), 4 deletions(-)
tests/qemu-iotests/194 +10 -4
@@ -42,6 +42,8 @@ with iotests.FilePath('source.img') as source_img_path, \ .add_incoming('unix:{0}'.format(migration_sock_path)) .launch()) + source_vm.qmp_log('block-dirty-bitmap-add', node='drive0', name='bitmap0') + iotests.log('Launching NBD server on destination...') iotests.log(dest_vm.qmp('nbd-server-start', addr={'type': 'unix', 'data': {'path': nbd_sock_path}})) iotests.log(dest_vm.qmp('nbd-server-add', device='drive0', writable=True))
@@ -61,12 +63,14 @@ with iotests.FilePath('source.img') as source_img_path, \ filters=[iotests.filter_qmp_event]) iotests.log('Starting migration...') - source_vm.qmp('migrate-set-capabilities', - capabilities=[{'capability': 'events', 'state': True}]) - dest_vm.qmp('migrate-set-capabilities', - capabilities=[{'capability': 'events', 'state': True}]) + capabilities = [{'capability': 'events', 'state': True}, + {'capability': 'dirty-bitmaps', 'state': True}] + source_vm.qmp('migrate-set-capabilities', capabilities=capabilities) + dest_vm.qmp('migrate-set-capabilities', capabilities=capabilities) iotests.log(source_vm.qmp('migrate', uri='unix:{0}'.format(migration_sock_path))) + source_vm.qmp_log('migrate-start-postcopy') + while True: event1 = source_vm.event_wait('MIGRATION') iotests.log(event1, filters=[iotests.filter_qmp_event])
@@ -82,3 +86,5 @@ with iotests.FilePath('source.img') as source_img_path, \ iotests.log('Stopping the NBD server on destination...') iotests.log(dest_vm.qmp('nbd-server-stop')) break + + iotests.log(source_vm.qmp('query-block')['return'][0]['dirty-bitmaps'])
tests/qemu-iotests/194.out +6 -0
@@ -1,4 +1,6 @@ Launching VMs... +{"execute": "block-dirty-bitmap-add", "arguments": {"name": "bitmap0", "node": "drive0"}} +{"return": {}} Launching NBD server on destination... {"return": {}} {"return": {}}
@@ -8,11 +10,15 @@ Waiting for `drive-mirror` to complete... {"data": {"device": "mirror-job0", "len": 1073741824, "offset": 1073741824, "speed": 0, "type": "mirror"}, "event": "BLOCK_JOB_READY", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} Starting migration... {"return": {}} +{"execute": "migrate-start-postcopy", "arguments": {}} +{"return": {}} {"data": {"status": "setup"}, "event": "MIGRATION", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} {"data": {"status": "active"}, "event": "MIGRATION", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} +{"data": {"status": "postcopy-active"}, "event": "MIGRATION", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} {"data": {"status": "completed"}, "event": "MIGRATION", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} Gracefully ending the `drive-mirror` job on source... {"return": {}} {"data": {"device": "mirror-job0", "len": 1073741824, "offset": 1073741824, "speed": 0, "type": "mirror"}, "event": "BLOCK_JOB_COMPLETED", "timestamp": {"microseconds": "USECS", "seconds": "SECS"}} Stopping the NBD server on destination... {"return": {}} +[{"busy": false, "count": 0, "granularity": 65536, "name": "bitmap0", "persistent": false, "recording": true, "status": "active"}]
Signature · 1 line
2.21.0
ping
Quoted 33 lines · Vladimir Sementsov-Ogievskiy
Hi all!
It's a continuation for
"bitmap migration bug with -drive while block mirror runs" <[email protected]>
https://lists.gnu.org/archive/html/qemu-devel/2019-09/msg07241.htmlThe problem is that bitmaps migrated to node with same node-name or blk-parent name. And currently only the latter actually work in libvirt. And with mirror-top filter it doesn't work, because
bdrv_get_device_or_node_namedon't go through filters.Fix this by handling filtered children of block backends in separate.
Max Reitz (1):
block: Mark commit and mirror as filter driversVladimir Sementsov-Ogievskiy (4):
migretion/block-dirty-bitmap: refactorinit_dirty_bitmap_migrationblock/dirty-bitmap: addbdrv_has_named_bitmapshelper migration/block-dirty-bitmap: fix bitmaps migration during mirror job
iotests: 194: test also migration of dirty bitmapinclude/block/block_int.h | 8 ++- include/block/dirty-bitmap.h | 1 + block/commit.c | 2 + block/dirty-bitmap.c | 13 ++++ block/mirror.c | 2 + migration/block-dirty-bitmap.c | 118 +++++++++++++++++++++++---------- tests/qemu-iotests/194 | 14 ++-- tests/qemu-iotests/194.out | 6 ++ 8 files changed, 121 insertions(+), 43 deletions(-)
Signature · 2 lines
Best regards, Vladimir