Skip to contentSkip to conversation

migration: introduce failed-unrecovarable status

RFC In review Vladimir Sementsov-Ogievskiy 1 participant 2 messages Dec 18, 2019 – Jan 20, 2020

lore
Share & tools
lore
#0 lore raw

RFC 1/1 migration: introduce failed-unrecovarable status

We should not start source vm automatically, if the error occured after target accessed disks, or if we failed to invalidate nodes.

Also, fix, that we need invalidate even if bdrv_inactivate_all() failed, as in this case it still may successfully inactivate some of the nodes.

---

Hi all!

It's an investigation on top of old thread
https://lists.gnu.org/archive/html/qemu-devel/2018-06/msg02355.html

Either I'm missing something, or we need this patch. It's a draft, may be need to split it into 2-3 small patches. Still I'd like to get general approval at first, may be I'm doing something wrong.

Also, there may be other migration failure cases like this.

 qapi/migration.json   |  7 +++++--
 migration/migration.c | 36 ++++++++++++++++++++++++------------
 2 files changed, 29 insertions(+), 14 deletions(-)
qapi/migration.json +5 -2
index b7348d0c8b..90fa625cbb 100644
@@ -125,6 +125,9 @@
 #
 # @failed: some error occurred during migration process.
 #
+# @failed-unrecoverable: postcopy failed after no return point, when disks may
+#                        already be accessed by target Qemu process. (since 5.0)
+#
 # @colo: VM is in the process of fault tolerance, VM can not get into this
 #        state unless colo capability is enabled for migration. (since 2.8)
 #
@@ -142,8 +145,8 @@
 { 'enum': 'MigrationStatus',
   'data': [ 'none', 'setup', 'cancelling', 'cancelled',
             'active', 'postcopy-active', 'postcopy-paused',
-            'postcopy-recover', 'completed', 'failed', 'colo',
-            'pre-switchover', 'device', 'wait-unplug' ] }
+            'postcopy-recover', 'completed', 'failed', 'failed-unrecoverable',
+            'colo', 'pre-switchover', 'device', 'wait-unplug' ] }
 
 ##
 # @MigrationInfo:
migration/migration.c +24 -12
index 354ad072fa..00684fdef8 100644
@@ -2576,7 +2576,14 @@ static int postcopy_start(MigrationState *ms)
     QEMUFile *fb;
     int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
     int64_t bandwidth = migrate_max_postcopy_bandwidth();
-    bool restart_block = false;
+
+    /*
+     * recoverable_failure
+     * A failure happened early enough that we know the destination hasn't
+     * accessed block devices, so we're safe to recover.
+     */
+    bool recoverable_failure = true;
+    bool inactivated = false;
     int cur_state = MIGRATION_STATUS_ACTIVE;
     if (!migrate_pause_before_switchover()) {
         migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
@@ -2600,11 +2607,11 @@ static int postcopy_start(MigrationState *ms)
         goto fail;
     }
 
+    inactivated = true;
     ret = bdrv_inactivate_all();
     if (ret < 0) {
         goto fail;
     }
-    restart_block = true;
 
     /*
      * Cause any non-postcopiable, but iterative devices to
@@ -2682,7 +2689,7 @@ static int postcopy_start(MigrationState *ms)
         goto fail_closefb;
     }
 
-    restart_block = false;
+    recoverable_failure = false;
 
     /* Now send that blob */
     if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
@@ -2716,26 +2723,28 @@ static int postcopy_start(MigrationState *ms)
     ret = qemu_file_get_error(ms->to_dst_file);
     if (ret) {
         error_report("postcopy_start: Migration stream errored");
-        migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
-                              MIGRATION_STATUS_FAILED);
+        goto fail;
     }
 
-    return ret;
+    return 0;
 
 fail_closefb:
     qemu_fclose(fb);
 fail:
     migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
-                          MIGRATION_STATUS_FAILED);
-    if (restart_block) {
-        /* A failure happened early enough that we know the destination hasn't
-         * accessed block devices, so we're safe to recover.
-         */
+                      recoverable_failure ? MIGRATION_STATUS_FAILED :
+                      MIGRATION_STATUS_FAILED_UNRECOVERABLE);
+    if (recoverable_failure && inactivated) {
         Error *local_err = NULL;
 
         bdrv_invalidate_cache_all(&local_err);
         if (local_err) {
             error_report_err(local_err);
+            /*
+             * We failed to invalidate, so we must not start vm automatically.
+             * User may retry invalidation and start by cont qmp command.
+             */
+            ms->vm_was_running = false;
         }
     }
     qemu_mutex_unlock_iothread();
@@ -3194,9 +3203,12 @@ static void migration_iteration_finish(MigrationState *s)
         s->vm_was_running = true;
         /* Fallthrough */
     case MIGRATION_STATUS_FAILED:
+    case MIGRATION_STATUS_FAILED_UNRECOVERABLE:
     case MIGRATION_STATUS_CANCELLED:
     case MIGRATION_STATUS_CANCELLING:
-        if (s->vm_was_running) {
+        if (s->vm_was_running &&
+            s->state != MIGRATION_STATUS_FAILED_UNRECOVERABLE)
+        {
             vm_start();
         } else {
             if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
Signature · 1 line
2.21.0
#1 lore raw

ping

Quoted 144 lines · Vladimir Sementsov-Ogievskiy

We should not start source vm automatically, if the error occured after target accessed disks, or if we failed to invalidate nodes.

Also, fix, that we need invalidate even if bdrv_inactivate_all() failed, as in this case it still may successfully inactivate some of the nodes.

---

Hi all!

It's an investigation on top of old thread
https://lists.gnu.org/archive/html/qemu-devel/2018-06/msg02355.html

Either I'm missing something, or we need this patch. It's a draft, may be need to split it into 2-3 small patches. Still I'd like to get general approval at first, may be I'm doing something wrong.

Also, there may be other migration failure cases like this.

  qapi/migration.json   |  7 +++++--
  migration/migration.c | 36 ++++++++++++++++++++++++------------
  2 files changed, 29 insertions(+), 14 deletions(-)
qapi/migration.json +5 -2
index b7348d0c8b..90fa625cbb 100644
@@ -125,6 +125,9 @@
  #
  # @failed: some error occurred during migration process.
  #
+# @failed-unrecoverable: postcopy failed after no return point, when disks may
+#                        already be accessed by target Qemu process. (since 5.0)
+#
  # @colo: VM is in the process of fault tolerance, VM can not get into this
  #        state unless colo capability is enabled for migration. (since 2.8)
  #
@@ -142,8 +145,8 @@
  { 'enum': 'MigrationStatus',
    'data': [ 'none', 'setup', 'cancelling', 'cancelled',
              'active', 'postcopy-active', 'postcopy-paused',
-            'postcopy-recover', 'completed', 'failed', 'colo',
-            'pre-switchover', 'device', 'wait-unplug' ] }
+            'postcopy-recover', 'completed', 'failed', 'failed-unrecoverable',
+            'colo', 'pre-switchover', 'device', 'wait-unplug' ] }
 
  ##
  # @MigrationInfo:
migration/migration.c +24 -12
index 354ad072fa..00684fdef8 100644
@@ -2576,7 +2576,14 @@ static int postcopy_start(MigrationState *ms)
      QEMUFile *fb;
      int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
      int64_t bandwidth = migrate_max_postcopy_bandwidth();
-    bool restart_block = false;
+
+    /*
+     * recoverable_failure
+     * A failure happened early enough that we know the destination hasn't
+     * accessed block devices, so we're safe to recover.
+     */
+    bool recoverable_failure = true;
+    bool inactivated = false;
      int cur_state = MIGRATION_STATUS_ACTIVE;
      if (!migrate_pause_before_switchover()) {
          migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
@@ -2600,11 +2607,11 @@ static int postcopy_start(MigrationState *ms)
          goto fail;
      }
 
+    inactivated = true;
      ret = bdrv_inactivate_all();
      if (ret < 0) {
          goto fail;
      }
-    restart_block = true;
 
      /*
       * Cause any non-postcopiable, but iterative devices to
@@ -2682,7 +2689,7 @@ static int postcopy_start(MigrationState *ms)
          goto fail_closefb;
      }
 
-    restart_block = false;
+    recoverable_failure = false;
 
      /* Now send that blob */
      if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
@@ -2716,26 +2723,28 @@ static int postcopy_start(MigrationState *ms)
      ret = qemu_file_get_error(ms->to_dst_file);
      if (ret) {
          error_report("postcopy_start: Migration stream errored");
-        migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
-                              MIGRATION_STATUS_FAILED);
+        goto fail;
      }
 
-    return ret;
+    return 0;
 
  fail_closefb:
      qemu_fclose(fb);
  fail:
      migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
-                          MIGRATION_STATUS_FAILED);
-    if (restart_block) {
-        /* A failure happened early enough that we know the destination hasn't
-         * accessed block devices, so we're safe to recover.
-         */
+                      recoverable_failure ? MIGRATION_STATUS_FAILED :
+                      MIGRATION_STATUS_FAILED_UNRECOVERABLE);
+    if (recoverable_failure && inactivated) {
          Error *local_err = NULL;
 
          bdrv_invalidate_cache_all(&local_err);
          if (local_err) {
              error_report_err(local_err);
+            /*
+             * We failed to invalidate, so we must not start vm automatically.
+             * User may retry invalidation and start by cont qmp command.
+             */
+            ms->vm_was_running = false;
          }
      }
      qemu_mutex_unlock_iothread();
@@ -3194,9 +3203,12 @@ static void migration_iteration_finish(MigrationState *s)
          s->vm_was_running = true;
          /* Fallthrough */
      case MIGRATION_STATUS_FAILED:
+    case MIGRATION_STATUS_FAILED_UNRECOVERABLE:
      case MIGRATION_STATUS_CANCELLED:
      case MIGRATION_STATUS_CANCELLING:
-        if (s->vm_was_running) {
+        if (s->vm_was_running &&
+            s->state != MIGRATION_STATUS_FAILED_UNRECOVERABLE)
+        {
              vm_start();
          } else {
              if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
Signature · 2 lines
Best regards,
Vladimir