Skip to contentSkip to conversation

Fix crashes on early shutdown during bitmaps postcopy

PATCH In review Vladimir Sementsov-Ogievskiy 2 participants 14 messages Jan 22, 2020 – Feb 11, 2020

lore
Share & tools
lore
#0 lore raw

cover Fix crashes on early shutdown during bitmaps postcopy

Hi all!

Patches 5 and 6 fixes two crashes, triggered by new test case in patch 7.

Vladimir Sementsov-Ogievskiy (7):
migration/block-dirty-bitmap: refactor incoming state to be one struct migration/block-dirty-bitmap: rename finish_lock to just lock migration/block-dirty-bitmap: simplify dirty_bitmap_load_complete migration/block-dirty-bitmap: keep bitmap state for all bitmaps migration/block-dirty-bitmap: cancel migration on shutdown
migration: handle to_src_file on target only for ram postcopy qemu-iotests/199: add early shutdown case to bitmaps postcopy

 migration/migration.h          |   1 +
 migration/block-dirty-bitmap.c | 171 +++++++++++++++++++++------------
 migration/migration.c          |   7 ++
 migration/savevm.c             |  19 ++--
 tests/qemu-iotests/199         |  12 ++-
 tests/qemu-iotests/199.out     |   4 +-
 6 files changed, 142 insertions(+), 72 deletions(-)
Signature · 1 line
2.21.0
#1 lore raw

PATCH 1/7 migration/block-dirty-bitmap: refactor incoming state to be one struct

Move enabled_bitmaps and finish_lock, which are part of incoming state to DirtyBitmapLoadState, and make static global variable to store state instead of static local one.

---
 migration/block-dirty-bitmap.c | 77 +++++++++++++++++++---------------
 1 file changed, 43 insertions(+), 34 deletions(-)
migration/block-dirty-bitmap.c +43 -34
index 7eafface61..281d20f41d 100644
@@ -125,6 +125,13 @@ typedef struct DirtyBitmapMigState {
     BlockDriverState *prev_bs;
     BdrvDirtyBitmap *prev_bitmap;
 } DirtyBitmapMigState;
+static DirtyBitmapMigState dirty_bitmap_mig_state;
+
+typedef struct DirtyBitmapLoadBitmapState {
+    BlockDriverState *bs;
+    BdrvDirtyBitmap *bitmap;
+    bool migrated;
+} DirtyBitmapLoadBitmapState;
 
 typedef struct DirtyBitmapLoadState {
     uint32_t flags;
@@ -132,21 +139,15 @@ typedef struct DirtyBitmapLoadState {
     char bitmap_name[256];
     BlockDriverState *bs;
     BdrvDirtyBitmap *bitmap;
-} DirtyBitmapLoadState;
 
-static DirtyBitmapMigState dirty_bitmap_mig_state;
-
-typedef struct DirtyBitmapLoadBitmapState {
-    BlockDriverState *bs;
-    BdrvDirtyBitmap *bitmap;
-    bool migrated;
-} DirtyBitmapLoadBitmapState;
-static GSList *enabled_bitmaps;
-QemuMutex finish_lock;
+    GSList *enabled_bitmaps;
+    QemuMutex finish_lock;
+} DirtyBitmapLoadState;
+static DirtyBitmapLoadState dbm_load_state;
 
 void init_dirty_bitmap_incoming_migration(void)
 {
-    qemu_mutex_init(&finish_lock);
+    qemu_mutex_init(&dbm_load_state.finish_lock);
 }
 
 static uint32_t qemu_get_bitmap_flags(QEMUFile *f)
@@ -439,8 +440,9 @@ static void dirty_bitmap_save_pending(QEMUFile *f, void *opaque,
 }
 
 /* First occurrence of this bitmap. It should be created if doesn't exist */
-static int dirty_bitmap_load_start(QEMUFile *f, DirtyBitmapLoadState *s)
+static int dirty_bitmap_load_start(QEMUFile *f)
 {
+    DirtyBitmapLoadState *s = &dbm_load_state;
     Error *local_err = NULL;
     uint32_t granularity = qemu_get_be32(f);
     uint8_t flags = qemu_get_byte(f);
@@ -482,7 +484,8 @@ static int dirty_bitmap_load_start(QEMUFile *f, DirtyBitmapLoadState *s)
         b->bs = s->bs;
         b->bitmap = s->bitmap;
         b->migrated = false;
-        enabled_bitmaps = g_slist_prepend(enabled_bitmaps, b);
+        dbm_load_state.enabled_bitmaps =
+            g_slist_prepend(dbm_load_state.enabled_bitmaps, b);
     }
 
     return 0;
@@ -492,9 +495,11 @@ void dirty_bitmap_mig_before_vm_start(void)
 {
     GSList *item;
 
-    qemu_mutex_lock(&finish_lock);
+    qemu_mutex_lock(&dbm_load_state.finish_lock);
 
-    for (item = enabled_bitmaps; item; item = g_slist_next(item)) {
+    for (item = dbm_load_state.enabled_bitmaps; item;
+         item = g_slist_next(item))
+    {
         DirtyBitmapLoadBitmapState *b = item->data;
 
         if (b->migrated) {
@@ -506,21 +511,24 @@ void dirty_bitmap_mig_before_vm_start(void)
         g_free(b);
     }
 
-    g_slist_free(enabled_bitmaps);
-    enabled_bitmaps = NULL;
+    g_slist_free(dbm_load_state.enabled_bitmaps);
+    dbm_load_state.enabled_bitmaps = NULL;
 
-    qemu_mutex_unlock(&finish_lock);
+    qemu_mutex_unlock(&dbm_load_state.finish_lock);
 }
 
-static void dirty_bitmap_load_complete(QEMUFile *f, DirtyBitmapLoadState *s)
+static void dirty_bitmap_load_complete(QEMUFile *f)
 {
+    DirtyBitmapLoadState *s = &dbm_load_state;
     GSList *item;
     trace_dirty_bitmap_load_complete();
     bdrv_dirty_bitmap_deserialize_finish(s->bitmap);
 
-    qemu_mutex_lock(&finish_lock);
+    qemu_mutex_lock(&dbm_load_state.finish_lock);
 
-    for (item = enabled_bitmaps; item; item = g_slist_next(item)) {
+    for (item = dbm_load_state.enabled_bitmaps; item;
+         item = g_slist_next(item))
+    {
         DirtyBitmapLoadBitmapState *b = item->data;
 
         if (b->bitmap == s->bitmap) {
@@ -531,7 +539,7 @@ static void dirty_bitmap_load_complete(QEMUFile *f, DirtyBitmapLoadState *s)
 
     if (bdrv_dirty_bitmap_has_successor(s->bitmap)) {
         bdrv_dirty_bitmap_lock(s->bitmap);
-        if (enabled_bitmaps == NULL) {
+        if (dbm_load_state.enabled_bitmaps == NULL) {
             /* in postcopy */
             bdrv_reclaim_dirty_bitmap_locked(s->bitmap, &error_abort);
             bdrv_enable_dirty_bitmap_locked(s->bitmap);
@@ -550,11 +558,12 @@ static void dirty_bitmap_load_complete(QEMUFile *f, DirtyBitmapLoadState *s)
         bdrv_dirty_bitmap_unlock(s->bitmap);
     }
 
-    qemu_mutex_unlock(&finish_lock);
+    qemu_mutex_unlock(&dbm_load_state.finish_lock);
 }
 
-static int dirty_bitmap_load_bits(QEMUFile *f, DirtyBitmapLoadState *s)
+static int dirty_bitmap_load_bits(QEMUFile *f)
 {
+    DirtyBitmapLoadState *s = &dbm_load_state;
     uint64_t first_byte = qemu_get_be64(f) << BDRV_SECTOR_BITS;
     uint64_t nr_bytes = (uint64_t)qemu_get_be32(f) << BDRV_SECTOR_BITS;
     trace_dirty_bitmap_load_bits_enter(first_byte >> BDRV_SECTOR_BITS,
@@ -598,8 +607,9 @@ static int dirty_bitmap_load_bits(QEMUFile *f, DirtyBitmapLoadState *s)
     return 0;
 }
 
-static int dirty_bitmap_load_header(QEMUFile *f, DirtyBitmapLoadState *s)
+static int dirty_bitmap_load_header(QEMUFile *f)
 {
+    DirtyBitmapLoadState *s = &dbm_load_state;
     Error *local_err = NULL;
     bool nothing;
     s->flags = qemu_get_bitmap_flags(f);
@@ -647,7 +657,6 @@ static int dirty_bitmap_load_header(QEMUFile *f, DirtyBitmapLoadState *s)
 
 static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
 {
-    static DirtyBitmapLoadState s;
     int ret = 0;
 
     trace_dirty_bitmap_load_enter();
@@ -657,17 +666,17 @@ static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
     }
 
     do {
-        ret = dirty_bitmap_load_header(f, &s);
+        ret = dirty_bitmap_load_header(f);
         if (ret < 0) {
             return ret;
         }
 
-        if (s.flags & DIRTY_BITMAP_MIG_FLAG_START) {
-            ret = dirty_bitmap_load_start(f, &s);
-        } else if (s.flags & DIRTY_BITMAP_MIG_FLAG_COMPLETE) {
-            dirty_bitmap_load_complete(f, &s);
-        } else if (s.flags & DIRTY_BITMAP_MIG_FLAG_BITS) {
-            ret = dirty_bitmap_load_bits(f, &s);
+        if (dbm_load_state.flags & DIRTY_BITMAP_MIG_FLAG_START) {
+            ret = dirty_bitmap_load_start(f);
+        } else if (dbm_load_state.flags & DIRTY_BITMAP_MIG_FLAG_COMPLETE) {
+            dirty_bitmap_load_complete(f);
+        } else if (dbm_load_state.flags & DIRTY_BITMAP_MIG_FLAG_BITS) {
+            ret = dirty_bitmap_load_bits(f);
         }
 
         if (!ret) {
@@ -677,7 +686,7 @@ static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
         if (ret) {
             return ret;
         }
-    } while (!(s.flags & DIRTY_BITMAP_MIG_FLAG_EOS));
+    } while (!(dbm_load_state.flags & DIRTY_BITMAP_MIG_FLAG_EOS));
 
     trace_dirty_bitmap_load_success();
     return 0;
Signature · 1 line
2.21.0
#2 lore raw

Re: [PATCH 1/7] migration/block-dirty-bitmap: refactor incoming state to be one struct

Quoted 14 lines · Vladimir Sementsov-Ogievskiy

Move enabled_bitmaps and finish_lock, which are part of incoming state to DirtyBitmapLoadState, and make static global variable to store state instead of static local one.

---
 migration/block-dirty-bitmap.c | 77 +++++++++++++++++++---------------
 1 file changed, 43 insertions(+), 34 deletions(-)
migration/block-dirty-bitmap.c +0 -0
index 7eafface61..281d20f41d 100644

migration/block-dirty-bitmap.c typedef struct DirtyBitmapMigState { · line 128

Quoted 4 lines
migration/block-dirty-bitmap.c +1 -0
@@ -125,6 +125,13 @@ typedef struct DirtyBitmapMigState {
     BlockDriverState *prev_bs;
     BdrvDirtyBitmap *prev_bitmap;
 } DirtyBitmapMigState;
+static DirtyBitmapMigState dirty_bitmap_mig_state;

Missing new line.

Quoted 18 lines
migration/block-dirty-bitmap.c +6 -4
+
+typedef struct DirtyBitmapLoadBitmapState {
+    BlockDriverState *bs;
+    BdrvDirtyBitmap *bitmap;
+    bool migrated;
+} DirtyBitmapLoadBitmapState;
 
 typedef struct DirtyBitmapLoadState {
     uint32_t flags;
@@ -132,21 +139,15 @@ typedef struct DirtyBitmapLoadState {
     char bitmap_name[256];
     BlockDriverState *bs;
     BdrvDirtyBitmap *bitmap;
-} DirtyBitmapLoadState;
 
-static DirtyBitmapMigState dirty_bitmap_mig_state;
-
-typedef struct DirtyBitmapLoadBitmapState {

migration/block-dirty-bitmap.c typedef struct DirtyBitmapLoadState { · line 146

Quoted 10 lines
migration/block-dirty-bitmap.c +4 -6
-    BlockDriverState *bs;
-    BdrvDirtyBitmap *bitmap;
-    bool migrated;
-} DirtyBitmapLoadBitmapState;
-static GSList *enabled_bitmaps;
-QemuMutex finish_lock;
+    GSList *enabled_bitmaps;
+    QemuMutex finish_lock;
+} DirtyBitmapLoadState;
+static DirtyBitmapLoadState dbm_load_state;

You move two global variables to an struct (good)
But you create a even bigger global variable (i.e. state that was not shared before.)

migration/block-dirty-bitmap.c static void dirty_bitmap_save_pending(QEMUFile *f, void *opaque, · line 445

Quoted 5 lines
migration/block-dirty-bitmap.c +2 -1
 /* First occurrence of this bitmap. It should be created if doesn't exist */
-static int dirty_bitmap_load_start(QEMUFile *f, DirtyBitmapLoadState *s)
+static int dirty_bitmap_load_start(QEMUFile *f)
 {
+    DirtyBitmapLoadState *s = &dbm_load_state;

You create a local alias.

migration/block-dirty-bitmap.c static int dirty_bitmap_load_start(QEMUFile *f, DirtyBitmapLoadState *s) · line 488

Quoted 10 lines
migration/block-dirty-bitmap.c +2 -1
     Error *local_err = NULL;
     uint32_t granularity = qemu_get_be32(f);
     uint8_t flags = qemu_get_byte(f);
@@ -482,7 +484,8 @@ static int dirty_bitmap_load_start(QEMUFile *f, DirtyBitmapLoadState *s)
         b->bs = s->bs;
         b->bitmap = s->bitmap;
         b->migrated = false;
-        enabled_bitmaps = g_slist_prepend(enabled_bitmaps, b);
+        dbm_load_state.enabled_bitmaps =
+            g_slist_prepend(dbm_load_state.enabled_bitmaps, b);

And then you access it using the global variable?

migration/block-dirty-bitmap.c void dirty_bitmap_mig_before_vm_start(void) · line 522

Quoted 4 lines
migration/block-dirty-bitmap.c +2 -1
-static void dirty_bitmap_load_complete(QEMUFile *f, DirtyBitmapLoadState *s)
+static void dirty_bitmap_load_complete(QEMUFile *f)
 {
+    DirtyBitmapLoadState *s = &dbm_load_state;

Exactly the same on this function.

I still don't understand why you are removing the pass as parameters of this function.

migration/block-dirty-bitmap.c static int dirty_bitmap_load_header(QEMUFile *f, DirtyBitmapLoadState *s) · line 650

Quoted 1 line
migration/block-dirty-bitmap.c +0 -1
-    static DirtyBitmapLoadState s;

Aha, this is why you are moving it as a global.

But, why can't you put this inside dirty_bitmap_mig_state? Then you:
a- don't have to have "yet" another global variable
b- you can clean it up on save_cleanup handler.

not related to this patch, but to the file in general:

static void dirty_bitmap_save_cleanup(void *opaque)

{
    dirty_bitmap_mig_cleanup();
}

We have opaque here, that we can do:

DirtyBitmapMigBitmapState *dbms = opaque;

And then pass dbms to dirty_bitmap_mig_cleanup().

/* Called with iothread lock taken. */
static void dirty_bitmap_mig_cleanup(void)

{
    DirtyBitmapMigBitmapState *dbms;

    while ((dbms = QSIMPLEQ_FIRST(&dirty_bitmap_mig_state.dbms_list)) != NULL) {
        QSIMPLEQ_REMOVE_HEAD(&dirty_bitmap_mig_state.dbms_list, entry);
        bdrv_dirty_bitmap_set_busy(dbms->bitmap, false);
        bdrv_unref(dbms->bs);
        g_free(dbms);
    }
}

Because here we just use the global variable.

Later, Juan.

#3 lore raw

Re: [PATCH 1/7] migration/block-dirty-bitmap: refactor incoming state to be one struct

Quoted 87 lines · Juan Quintela
Quoted 14 lines · Vladimir Sementsov-Ogievskiy

Move enabled_bitmaps and finish_lock, which are part of incoming state to DirtyBitmapLoadState, and make static global variable to store state instead of static local one.

---
  migration/block-dirty-bitmap.c | 77 +++++++++++++++++++---------------
  1 file changed, 43 insertions(+), 34 deletions(-)
migration/block-dirty-bitmap.c +0 -0
index 7eafface61..281d20f41d 100644

migration/block-dirty-bitmap.c typedef struct DirtyBitmapMigState { · line 128

Quoted 4 lines
migration/block-dirty-bitmap.c +1 -0
@@ -125,6 +125,13 @@ typedef struct DirtyBitmapMigState {
      BlockDriverState *prev_bs;
      BdrvDirtyBitmap *prev_bitmap;
  } DirtyBitmapMigState;
+static DirtyBitmapMigState dirty_bitmap_mig_state;

Missing new line.

Quoted 18 lines
migration/block-dirty-bitmap.c +6 -4
+
+typedef struct DirtyBitmapLoadBitmapState {
+    BlockDriverState *bs;
+    BdrvDirtyBitmap *bitmap;
+    bool migrated;
+} DirtyBitmapLoadBitmapState;
 
  typedef struct DirtyBitmapLoadState {
      uint32_t flags;
@@ -132,21 +139,15 @@ typedef struct DirtyBitmapLoadState {
      char bitmap_name[256];
      BlockDriverState *bs;
      BdrvDirtyBitmap *bitmap;
-} DirtyBitmapLoadState;
 
-static DirtyBitmapMigState dirty_bitmap_mig_state;
-
-typedef struct DirtyBitmapLoadBitmapState {

migration/block-dirty-bitmap.c typedef struct DirtyBitmapLoadState { · line 146

Quoted 10 lines
migration/block-dirty-bitmap.c +4 -6
-    BlockDriverState *bs;
-    BdrvDirtyBitmap *bitmap;
-    bool migrated;
-} DirtyBitmapLoadBitmapState;
-static GSList *enabled_bitmaps;
-QemuMutex finish_lock;
+    GSList *enabled_bitmaps;
+    QemuMutex finish_lock;
+} DirtyBitmapLoadState;
+static DirtyBitmapLoadState dbm_load_state;

You move two global variables to an struct (good)
But you create a even bigger global variable (i.e. state that was not shared before.)

migration/block-dirty-bitmap.c static void dirty_bitmap_save_pending(QEMUFile *f, void *opaque, · line 445

Quoted 5 lines
migration/block-dirty-bitmap.c +2 -1
  /* First occurrence of this bitmap. It should be created if doesn't exist */
-static int dirty_bitmap_load_start(QEMUFile *f, DirtyBitmapLoadState *s)
+static int dirty_bitmap_load_start(QEMUFile *f)
  {
+    DirtyBitmapLoadState *s = &dbm_load_state;

You create a local alias.

migration/block-dirty-bitmap.c static int dirty_bitmap_load_start(QEMUFile *f, DirtyBitmapLoadState *s) · line 488

Quoted 10 lines
migration/block-dirty-bitmap.c +2 -1
      Error *local_err = NULL;
      uint32_t granularity = qemu_get_be32(f);
      uint8_t flags = qemu_get_byte(f);
@@ -482,7 +484,8 @@ static int dirty_bitmap_load_start(QEMUFile *f, DirtyBitmapLoadState *s)
          b->bs = s->bs;
          b->bitmap = s->bitmap;
          b->migrated = false;
-        enabled_bitmaps = g_slist_prepend(enabled_bitmaps, b);
+        dbm_load_state.enabled_bitmaps =
+            g_slist_prepend(dbm_load_state.enabled_bitmaps, b);

And then you access it using the global variable?

migration/block-dirty-bitmap.c void dirty_bitmap_mig_before_vm_start(void) · line 522

Quoted 4 lines
migration/block-dirty-bitmap.c +2 -1
-static void dirty_bitmap_load_complete(QEMUFile *f, DirtyBitmapLoadState *s)
+static void dirty_bitmap_load_complete(QEMUFile *f)
  {
+    DirtyBitmapLoadState *s = &dbm_load_state;

Exactly the same on this function.

I still don't understand why you are removing the pass as parameters of this function.

Quoted 7 lines

migration/block-dirty-bitmap.c static int dirty_bitmap_load_header(QEMUFile *f, DirtyBitmapLoadState *s) · line 650

Quoted 1 line
migration/block-dirty-bitmap.c +0 -1
-    static DirtyBitmapLoadState s;

Aha, this is why you are moving it as a global.

But, why can't you put this inside dirty_bitmap_mig_state? Then you:
a- don't have to have "yet" another global variable
b- you can clean it up on save_cleanup handler.

Because dirty_bitmap_mig_state is source state, and new dbm_load_state is destination state. So, at least [b] will not work...

Do you think it's good to combine both source and destination states into one struct, and use opaque everywhere?

It will look like

DirtyBitmapMigSourceState *s = ((DirtyBitmapMigState *)opaque)->source_state;

in save functions

and
DirtyBitmapIncomingState *s = ((DirtyBitmapMigState *)opaque)->incoming_state;

in load function

Quoted 32 lines

not related to this patch, but to the file in general:

static void dirty_bitmap_save_cleanup(void *opaque)

{
     dirty_bitmap_mig_cleanup();
}

We have opaque here, that we can do:

DirtyBitmapMigBitmapState *dbms = opaque;

And then pass dbms to dirty_bitmap_mig_cleanup().

/* Called with iothread lock taken. */
static void dirty_bitmap_mig_cleanup(void)

{
     DirtyBitmapMigBitmapState *dbms;

     while ((dbms = QSIMPLEQ_FIRST(&dirty_bitmap_mig_state.dbms_list)) != NULL) {
         QSIMPLEQ_REMOVE_HEAD(&dirty_bitmap_mig_state.dbms_list, entry);
         bdrv_dirty_bitmap_set_busy(dbms->bitmap, false);
         bdrv_unref(dbms->bs);
         g_free(dbms);
     }
}

Because here we just use the global variable.

Later, Juan.

Signature · 2 lines
Best regards,
Vladimir
#4 lore raw

PATCH 2/7 migration/block-dirty-bitmap: rename finish_lock to just lock

In reply to Vladimir Sementsov-Ogievskiy

finish_lock is bad name, as lock used not only on process end.

---
 migration/block-dirty-bitmap.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
migration/block-dirty-bitmap.c +6 -6
index 281d20f41d..502e858c31 100644
@@ -141,13 +141,13 @@ typedef struct DirtyBitmapLoadState {
     BdrvDirtyBitmap *bitmap;
 
     GSList *enabled_bitmaps;
-    QemuMutex finish_lock;
+    QemuMutex lock; /* protect enabled_bitmaps */
 } DirtyBitmapLoadState;
 static DirtyBitmapLoadState dbm_load_state;
 
 void init_dirty_bitmap_incoming_migration(void)
 {
-    qemu_mutex_init(&dbm_load_state.finish_lock);
+    qemu_mutex_init(&dbm_load_state.lock);
 }
 
 static uint32_t qemu_get_bitmap_flags(QEMUFile *f)
@@ -495,7 +495,7 @@ void dirty_bitmap_mig_before_vm_start(void)
 {
     GSList *item;
 
-    qemu_mutex_lock(&dbm_load_state.finish_lock);
+    qemu_mutex_lock(&dbm_load_state.lock);
 
     for (item = dbm_load_state.enabled_bitmaps; item;
          item = g_slist_next(item))
@@ -514,7 +514,7 @@ void dirty_bitmap_mig_before_vm_start(void)
     g_slist_free(dbm_load_state.enabled_bitmaps);
     dbm_load_state.enabled_bitmaps = NULL;
 
-    qemu_mutex_unlock(&dbm_load_state.finish_lock);
+    qemu_mutex_unlock(&dbm_load_state.lock);
 }
 
 static void dirty_bitmap_load_complete(QEMUFile *f)
@@ -524,7 +524,7 @@ static void dirty_bitmap_load_complete(QEMUFile *f)
     trace_dirty_bitmap_load_complete();
     bdrv_dirty_bitmap_deserialize_finish(s->bitmap);
 
-    qemu_mutex_lock(&dbm_load_state.finish_lock);
+    qemu_mutex_lock(&dbm_load_state.lock);
 
     for (item = dbm_load_state.enabled_bitmaps; item;
          item = g_slist_next(item))
@@ -558,7 +558,7 @@ static void dirty_bitmap_load_complete(QEMUFile *f)
         bdrv_dirty_bitmap_unlock(s->bitmap);
     }
 
-    qemu_mutex_unlock(&dbm_load_state.finish_lock);
+    qemu_mutex_unlock(&dbm_load_state.lock);
 }
 
 static int dirty_bitmap_load_bits(QEMUFile *f)
Signature · 1 line
2.21.0
#6 lore raw

PATCH 3/7 migration/block-dirty-bitmap: simplify dirty_bitmap_load_complete

In reply to Vladimir Sementsov-Ogievskiy

bdrv_enable_dirty_bitmap_locked() call does nothing, as if we are in postcopy, bitmap successor must be enabled, and reclaim operation will enable the bitmap.

So, actually we need just call _reclaim_ in both if branches, and making differences only to add an assertion seems not really good. The logic becomes simple: on load complete we do reclaim and that's all.

---
 migration/block-dirty-bitmap.c | 25 ++++---------------------
 1 file changed, 4 insertions(+), 21 deletions(-)
migration/block-dirty-bitmap.c +4 -21
index 502e858c31..eeaab2174e 100644
@@ -526,6 +526,10 @@ static void dirty_bitmap_load_complete(QEMUFile *f)
 
     qemu_mutex_lock(&dbm_load_state.lock);
 
+    if (bdrv_dirty_bitmap_has_successor(s->bitmap)) {
+        bdrv_reclaim_dirty_bitmap(s->bitmap, &error_abort);
+    }
+
     for (item = dbm_load_state.enabled_bitmaps; item;
          item = g_slist_next(item))
     {
@@ -537,27 +541,6 @@ static void dirty_bitmap_load_complete(QEMUFile *f)
         }
     }
 
-    if (bdrv_dirty_bitmap_has_successor(s->bitmap)) {
-        bdrv_dirty_bitmap_lock(s->bitmap);
-        if (dbm_load_state.enabled_bitmaps == NULL) {
-            /* in postcopy */
-            bdrv_reclaim_dirty_bitmap_locked(s->bitmap, &error_abort);
-            bdrv_enable_dirty_bitmap_locked(s->bitmap);
-        } else {
-            /* target not started, successor must be empty */
-            int64_t count = bdrv_get_dirty_count(s->bitmap);
-            BdrvDirtyBitmap *ret = bdrv_reclaim_dirty_bitmap_locked(s->bitmap,
-                                                                    NULL);
-            /* bdrv_reclaim_dirty_bitmap can fail only on no successor (it
-             * must be) or on merge fail, but merge can't fail when second
-             * bitmap is empty
-             */
-            assert(ret == s->bitmap &&
-                   count == bdrv_get_dirty_count(s->bitmap));
-        }
-        bdrv_dirty_bitmap_unlock(s->bitmap);
-    }
-
     qemu_mutex_unlock(&dbm_load_state.lock);
 }
 
Signature · 1 line
2.21.0
#7 lore raw

PATCH 4/7 migration/block-dirty-bitmap: keep bitmap state for all bitmaps

In reply to Vladimir Sementsov-Ogievskiy

Keep bitmap state for disabled bitmaps too. Keep the state until the end of the process. It's needed for the following commit to implement bitmap postcopy canceling.

---
 migration/block-dirty-bitmap.c | 59 ++++++++++++++++++++++------------
 1 file changed, 38 insertions(+), 21 deletions(-)
migration/block-dirty-bitmap.c +38 -21
index eeaab2174e..f96458113c 100644
@@ -131,6 +131,7 @@ typedef struct DirtyBitmapLoadBitmapState {
     BlockDriverState *bs;
     BdrvDirtyBitmap *bitmap;
     bool migrated;
+    bool enabled;
 } DirtyBitmapLoadBitmapState;
 
 typedef struct DirtyBitmapLoadState {
@@ -140,8 +141,11 @@ typedef struct DirtyBitmapLoadState {
     BlockDriverState *bs;
     BdrvDirtyBitmap *bitmap;
 
-    GSList *enabled_bitmaps;
-    QemuMutex lock; /* protect enabled_bitmaps */
+    bool bitmaps_enabled; /* set in dirty_bitmap_mig_before_vm_start */
+    bool stream_ended; /* set when all migrated data handled */
+
+    GSList *bitmaps;
+    QemuMutex lock; /* protect bitmaps */
 } DirtyBitmapLoadState;
 static DirtyBitmapLoadState dbm_load_state;
 
@@ -446,6 +450,7 @@ static int dirty_bitmap_load_start(QEMUFile *f)
     Error *local_err = NULL;
     uint32_t granularity = qemu_get_be32(f);
     uint8_t flags = qemu_get_byte(f);
+    DirtyBitmapLoadBitmapState *b;
 
     if (s->bitmap) {
         error_report("Bitmap with the same name ('%s') already exists on "
@@ -472,22 +477,23 @@ static int dirty_bitmap_load_start(QEMUFile *f)
 
     bdrv_disable_dirty_bitmap(s->bitmap);
     if (flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED) {
-        DirtyBitmapLoadBitmapState *b;
-
         bdrv_dirty_bitmap_create_successor(s->bitmap, &local_err);
         if (local_err) {
             error_report_err(local_err);
             return -EINVAL;
         }
-
-        b = g_new(DirtyBitmapLoadBitmapState, 1);
-        b->bs = s->bs;
-        b->bitmap = s->bitmap;
-        b->migrated = false;
-        dbm_load_state.enabled_bitmaps =
-            g_slist_prepend(dbm_load_state.enabled_bitmaps, b);
     }
 
+    b = g_new(DirtyBitmapLoadBitmapState, 1);
+    *b = (DirtyBitmapLoadBitmapState) {
+        .bs = s->bs,
+        .bitmap = s->bitmap,
+        .migrated = false,
+        .enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED,
+    };
+
+    dbm_load_state.bitmaps = g_slist_prepend(dbm_load_state.bitmaps, b);
+
     return 0;
 }
 
@@ -497,22 +503,25 @@ void dirty_bitmap_mig_before_vm_start(void)
 
     qemu_mutex_lock(&dbm_load_state.lock);
 
-    for (item = dbm_load_state.enabled_bitmaps; item;
-         item = g_slist_next(item))
-    {
+    for (item = dbm_load_state.bitmaps; item; item = g_slist_next(item)) {
         DirtyBitmapLoadBitmapState *b = item->data;
 
+        if (!b->enabled) {
+            continue;
+        }
+
         if (b->migrated) {
             bdrv_enable_dirty_bitmap_locked(b->bitmap);
         } else {
             bdrv_dirty_bitmap_enable_successor(b->bitmap);
         }
-
-        g_free(b);
     }
 
-    g_slist_free(dbm_load_state.enabled_bitmaps);
-    dbm_load_state.enabled_bitmaps = NULL;
+    dbm_load_state.bitmaps_enabled = true;
+    if (dbm_load_state.stream_ended) {
+        g_slist_free_full(dbm_load_state.bitmaps, g_free);
+        dbm_load_state.bitmaps = NULL;
+    }
 
     qemu_mutex_unlock(&dbm_load_state.lock);
 }
@@ -530,9 +539,7 @@ static void dirty_bitmap_load_complete(QEMUFile *f)
         bdrv_reclaim_dirty_bitmap(s->bitmap, &error_abort);
     }
 
-    for (item = dbm_load_state.enabled_bitmaps; item;
-         item = g_slist_next(item))
-    {
+    for (item = dbm_load_state.bitmaps; item; item = g_slist_next(item)) {
         DirtyBitmapLoadBitmapState *b = item->data;
 
         if (b->bitmap == s->bitmap) {
@@ -671,6 +678,16 @@ static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
         }
     } while (!(dbm_load_state.flags & DIRTY_BITMAP_MIG_FLAG_EOS));
 
+    qemu_mutex_lock(&dbm_load_state.lock);
+
+    dbm_load_state.stream_ended = true;
+    if (dbm_load_state.bitmaps_enabled) {
+        g_slist_free_full(dbm_load_state.bitmaps, g_free);
+        dbm_load_state.bitmaps = NULL;
+    }
+
+    qemu_mutex_unlock(&dbm_load_state.lock);
+
     trace_dirty_bitmap_load_success();
     return 0;
 }
Signature · 1 line
2.21.0
#8 lore raw

Re: [PATCH 4/7] migration/block-dirty-bitmap: keep bitmap state for all bitmaps

Quoted 11 lines · Vladimir Sementsov-Ogievskiy

Keep bitmap state for disabled bitmaps too. Keep the state until the end of the process. It's needed for the following commit to implement bitmap postcopy canceling.

migration/block-dirty-bitmap.c +0 -6
-
-        b = g_new(DirtyBitmapLoadBitmapState, 1);
-        b->bs = s->bs;
-        b->bitmap = s->bitmap;
-        b->migrated = false;
-        dbm_load_state.enabled_bitmaps =

migration/block-dirty-bitmap.c static int dirty_bitmap_load_start(QEMUFile *f) · line 493

Quoted 10 lines
migration/block-dirty-bitmap.c +7 -1
-            g_slist_prepend(dbm_load_state.enabled_bitmaps, b);
     }
 
+    b = g_new(DirtyBitmapLoadBitmapState, 1);
+    *b = (DirtyBitmapLoadBitmapState) {
+        .bs = s->bs,
+        .bitmap = s->bitmap,
+        .migrated = false,
+        .enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED,
+    };

What is wrong with:

     b->bs = s->bs;
     b->bitmap = s->bitmap;
     b->migrated = false;
     b->enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED;

???

Later, Juan.

#9 lore raw

Re: [PATCH 4/7] migration/block-dirty-bitmap: keep bitmap state for all bitmaps

Quoted 23 lines · Juan Quintela
Quoted 11 lines · Vladimir Sementsov-Ogievskiy

Keep bitmap state for disabled bitmaps too. Keep the state until the end of the process. It's needed for the following commit to implement bitmap postcopy canceling.

migration/block-dirty-bitmap.c +0 -6
-
-        b = g_new(DirtyBitmapLoadBitmapState, 1);
-        b->bs = s->bs;
-        b->bitmap = s->bitmap;
-        b->migrated = false;
-        dbm_load_state.enabled_bitmaps =

migration/block-dirty-bitmap.c static int dirty_bitmap_load_start(QEMUFile *f) · line 493

Quoted 10 lines
migration/block-dirty-bitmap.c +7 -1
-            g_slist_prepend(dbm_load_state.enabled_bitmaps, b);
      }
 
+    b = g_new(DirtyBitmapLoadBitmapState, 1);
+    *b = (DirtyBitmapLoadBitmapState) {
+        .bs = s->bs,
+        .bitmap = s->bitmap,
+        .migrated = false,
+        .enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED,
+    };
Quoted 7 lines

What is wrong with:

      b->bs = s->bs;
      b->bitmap = s->bitmap;
      b->migrated = false;
      b->enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED;

???

Nothing wrong. Compound literal is a bit better, as it will initialize to zero all skipped fields. Still nothing missed here. The change is actually unrelated to the patch, I can drop it.

Signature · 2 lines
Best regards,
Vladimir
#10 lore raw

Re: [PATCH 4/7] migration/block-dirty-bitmap: keep bitmap state for all bitmaps

In reply to Vladimir Sementsov-Ogievskiy
Quoted 121 lines · Vladimir Sementsov-Ogievskiy

Keep bitmap state for disabled bitmaps too. Keep the state until the end of the process. It's needed for the following commit to implement bitmap postcopy canceling.

---
  migration/block-dirty-bitmap.c | 59 ++++++++++++++++++++++------------
  1 file changed, 38 insertions(+), 21 deletions(-)
migration/block-dirty-bitmap.c +28 -21
index eeaab2174e..f96458113c 100644
@@ -131,6 +131,7 @@ typedef struct DirtyBitmapLoadBitmapState {
      BlockDriverState *bs;
      BdrvDirtyBitmap *bitmap;
      bool migrated;
+    bool enabled;
  } DirtyBitmapLoadBitmapState;
 
  typedef struct DirtyBitmapLoadState {
@@ -140,8 +141,11 @@ typedef struct DirtyBitmapLoadState {
      BlockDriverState *bs;
      BdrvDirtyBitmap *bitmap;
 
-    GSList *enabled_bitmaps;
-    QemuMutex lock; /* protect enabled_bitmaps */
+    bool bitmaps_enabled; /* set in dirty_bitmap_mig_before_vm_start */
+    bool stream_ended; /* set when all migrated data handled */
+
+    GSList *bitmaps;
+    QemuMutex lock; /* protect bitmaps */
  } DirtyBitmapLoadState;
  static DirtyBitmapLoadState dbm_load_state;
 
@@ -446,6 +450,7 @@ static int dirty_bitmap_load_start(QEMUFile *f)
      Error *local_err = NULL;
      uint32_t granularity = qemu_get_be32(f);
      uint8_t flags = qemu_get_byte(f);
+    DirtyBitmapLoadBitmapState *b;
 
      if (s->bitmap) {
          error_report("Bitmap with the same name ('%s') already exists on "
@@ -472,22 +477,23 @@ static int dirty_bitmap_load_start(QEMUFile *f)
 
      bdrv_disable_dirty_bitmap(s->bitmap);
      if (flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED) {
-        DirtyBitmapLoadBitmapState *b;
-
          bdrv_dirty_bitmap_create_successor(s->bitmap, &local_err);
          if (local_err) {
              error_report_err(local_err);
              return -EINVAL;
          }
-
-        b = g_new(DirtyBitmapLoadBitmapState, 1);
-        b->bs = s->bs;
-        b->bitmap = s->bitmap;
-        b->migrated = false;
-        dbm_load_state.enabled_bitmaps =
-            g_slist_prepend(dbm_load_state.enabled_bitmaps, b);
      }
 
+    b = g_new(DirtyBitmapLoadBitmapState, 1);
+    *b = (DirtyBitmapLoadBitmapState) {
+        .bs = s->bs,
+        .bitmap = s->bitmap,
+        .migrated = false,
+        .enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED,
+    };
+
+    dbm_load_state.bitmaps = g_slist_prepend(dbm_load_state.bitmaps, b);
+
      return 0;
  }
 
@@ -497,22 +503,25 @@ void dirty_bitmap_mig_before_vm_start(void)
 
      qemu_mutex_lock(&dbm_load_state.lock);
 
-    for (item = dbm_load_state.enabled_bitmaps; item;
-         item = g_slist_next(item))
-    {
+    for (item = dbm_load_state.bitmaps; item; item = g_slist_next(item)) {
          DirtyBitmapLoadBitmapState *b = item->data;
 
+        if (!b->enabled) {
+            continue;
+        }
+
          if (b->migrated) {
              bdrv_enable_dirty_bitmap_locked(b->bitmap);
          } else {
              bdrv_dirty_bitmap_enable_successor(b->bitmap);
          }
-
-        g_free(b);
      }
 
-    g_slist_free(dbm_load_state.enabled_bitmaps);
-    dbm_load_state.enabled_bitmaps = NULL;
+    dbm_load_state.bitmaps_enabled = true;
+    if (dbm_load_state.stream_ended) {
+        g_slist_free_full(dbm_load_state.bitmaps, g_free);
+        dbm_load_state.bitmaps = NULL;
+    }
 
      qemu_mutex_unlock(&dbm_load_state.lock);
  }
@@ -530,9 +539,7 @@ static void dirty_bitmap_load_complete(QEMUFile *f)
          bdrv_reclaim_dirty_bitmap(s->bitmap, &error_abort);
      }
 
-    for (item = dbm_load_state.enabled_bitmaps; item;
-         item = g_slist_next(item))
-    {
+    for (item = dbm_load_state.bitmaps; item; item = g_slist_next(item)) {
          DirtyBitmapLoadBitmapState *b = item->data;
 
          if (b->bitmap == s->bitmap) {

migration/block-dirty-bitmap.c static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id) · line 683

Quoted 6 lines
migration/block-dirty-bitmap.c +3 -0
@@ -671,6 +678,16 @@ static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
          }
      } while (!(dbm_load_state.flags & DIRTY_BITMAP_MIG_FLAG_EOS));
 
+    qemu_mutex_lock(&dbm_load_state.lock);
+
+    dbm_load_state.stream_ended = true;

This is wrong. EOS doesn't mean that bitmap migration finished: only one set of sequential bitmap chunks finished. EOS may be sent several times during migration and it is needed to switch to other things migration.

Quoted 11 lines
migration/block-dirty-bitmap.c +7 -0
+    if (dbm_load_state.bitmaps_enabled) {
+        g_slist_free_full(dbm_load_state.bitmaps, g_free);
+        dbm_load_state.bitmaps = NULL;
+    }
+
+    qemu_mutex_unlock(&dbm_load_state.lock);
+
      trace_dirty_bitmap_load_success();
      return 0;
  }
Signature · 2 lines
Best regards,
Vladimir
#11 lore raw

PATCH 5/7 migration/block-dirty-bitmap: cancel migration on shutdown

In reply to Vladimir Sementsov-Ogievskiy

If target is turned of prior to postcopy finished, we crash because busy bitmaps are found at shutdown.

Let's fix it by removing all unfinished bitmaps on shutdown.

---
 migration/migration.h          |  1 +
 migration/block-dirty-bitmap.c | 44 ++++++++++++++++++++++++++++++----
 migration/migration.c          |  7 ++++++
 3 files changed, 48 insertions(+), 4 deletions(-)
migration/migration.h +1 -0
index aa9ff6f27b..a3927b93bb 100644
@@ -331,6 +331,7 @@ void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis,
 void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value);
 
 void dirty_bitmap_mig_before_vm_start(void);
+void dirty_bitmap_mig_cancel_incoming(void);
 void init_dirty_bitmap_incoming_migration(void);
 void migrate_add_address(SocketAddress *address);
 
migration/block-dirty-bitmap.c +40 -4

80 lines not on this page · raw message

migration/migration.c +7 -0
index 990bff00c0..12d161165d 100644
@@ -182,6 +182,13 @@ void migration_shutdown(void)
      */
     migrate_fd_cancel(current_migration);
     object_unref(OBJECT(current_migration));
+
+    /*
+     * Cancel incoming migration of dirty bitmaps. Dirty bitmaps
+     * are non-critical data, and their loss never considered as
+     * something serious.
+     */
+    dirty_bitmap_mig_cancel_incoming();
 }
 
 /* For outgoing */
Signature · 1 line
2.21.0
#12 lore raw

PATCH 6/7 migration: handle to_src_file on target only for ram postcopy

In reply to Vladimir Sementsov-Ogievskiy

If only bitmaps postcopy migration enabled and not ram, this assertion will fire, as we don't have to_src_file for bitmaps postcopy migration.

migrate_postcopy_ram() accesses migrations state, which may be freed in main thread, so, we should ref/unref it in postcopy incoming thread.

---
 migration/savevm.c | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)
migration/savevm.c +13 -6
index adfdca26ac..143755389e 100644
@@ -1832,6 +1832,9 @@ static void *postcopy_ram_listen_thread(void *opaque)
     MigrationIncomingState *mis = migration_incoming_get_current();
     QEMUFile *f = mis->from_src_file;
     int load_res;
+    MigrationState *migr = migrate_get_current();
+
+    object_ref(OBJECT(migr));
 
     migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
                                    MIGRATION_STATUS_POSTCOPY_ACTIVE);
@@ -1898,6 +1901,8 @@ static void *postcopy_ram_listen_thread(void *opaque)
     mis->have_listen_thread = false;
     postcopy_state_set(POSTCOPY_INCOMING_END);
 
+    object_unref(OBJECT(migr));
+
     return NULL;
 }
 
@@ -2457,12 +2462,14 @@ static bool postcopy_pause_incoming(MigrationIncomingState *mis)
     qemu_fclose(mis->from_src_file);
     mis->from_src_file = NULL;
 
-    assert(mis->to_src_file);
-    qemu_file_shutdown(mis->to_src_file);
-    qemu_mutex_lock(&mis->rp_mutex);
-    qemu_fclose(mis->to_src_file);
-    mis->to_src_file = NULL;
-    qemu_mutex_unlock(&mis->rp_mutex);
+    if (migrate_postcopy_ram()) {
+        assert(mis->to_src_file);
+        qemu_file_shutdown(mis->to_src_file);
+        qemu_mutex_lock(&mis->rp_mutex);
+        qemu_fclose(mis->to_src_file);
+        mis->to_src_file = NULL;
+        qemu_mutex_unlock(&mis->rp_mutex);
+    }
 
     migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
                       MIGRATION_STATUS_POSTCOPY_PAUSED);
Signature · 1 line
2.21.0
#13 lore raw

PATCH 7/7 qemu-iotests/199: add early shutdown case to bitmaps postcopy

In reply to Vladimir Sementsov-Ogievskiy

Previous patches fixed two crashes which may occur on shutdown prior to bitmaps postcopy finished. Check that it works now.

---
 tests/qemu-iotests/199     | 12 +++++++++++-
 tests/qemu-iotests/199.out |  4 ++--
 2 files changed, 13 insertions(+), 3 deletions(-)
tests/qemu-iotests/199 +11 -1
index a2c8ecab5a..a3f6c73aed 100755
@@ -47,7 +47,7 @@ class TestDirtyBitmapPostcopyMigration(iotests.QMPTestCase):
         self.vm_a.launch()
         self.vm_b.launch()
 
-    def test_postcopy(self):
+    def do_test_postcopy(self, early_shutdown):
         write_size = 0x40000000
         granularity = 512
         chunk = 4096
@@ -99,6 +99,10 @@ class TestDirtyBitmapPostcopyMigration(iotests.QMPTestCase):
             if event['data']['status'] == 'completed':
                 break
 
+        if early_shutdown:
+            self.vm_b.qmp('quit')
+            return
+
         s = 0x8000
         while s < write_size:
             self.vm_b.hmp_qemu_io('drive0', 'write %d %d' % (s, chunk))
@@ -114,6 +118,12 @@ class TestDirtyBitmapPostcopyMigration(iotests.QMPTestCase):
 
         self.assert_qmp(result, 'return/sha256', sha256);
 
+    def test_postcopy(self):
+        self.do_test_postcopy(False)
+
+    def test_postcopy_early_shutdown(self):
+        self.do_test_postcopy(True)
+
 if __name__ == '__main__':
     iotests.main(supported_fmts=['qcow2'], supported_cache_modes=['none'],
                  supported_protocols=['file'])
tests/qemu-iotests/199.out +2 -2
index ae1213e6f8..fbc63e62f8 100644
@@ -1,5 +1,5 @@
-.
+..
 ----------------------------------------------------------------------
-Ran 1 tests
+Ran 2 tests
 
 OK
Signature · 1 line
2.21.0