Skip to contentSkip to conversation

colo migration cleanups

PATCH In review Lukas Straub 3 participants 12 messages Jun 22, 2023 – Jun 28, 2023

lore
Share & tools
lore
#1 lore raw

PATCH 1/8 colo: Only support the same qemu version on source and destination

In reply to Lukas Straub
---
 docs/COLO-FT.txt | 2 ++
 1 file changed, 2 insertions(+)
docs/COLO-FT.txt +2 -0
index 2e760a4aee..8e64480dbd 100644
@@ -148,6 +148,8 @@ in test procedure.
 Note: Here we are running both instances on the same host for testing,
 change the IP Addresses if you want to run it on two hosts. Initially
 127.0.0.1 is the Primary Host and 127.0.0.2 is the Secondary Host.
+Note: COLO is a experimental feature, so currently is should only be used
+with the same qemu version on sourcee and target.
 
 == Startup qemu ==
 1. Primary:
Signature · 1 line
2.39.2
#0 lore raw

cover colo migration cleanups

Hello Everyone,
Here are some cleanups around the colo migration code. Notably, we only support the same qemu version on both sides for now. Though in this patchset I still try to be support migration to a newer qemu version.

Lukas Straub (8):
colo: Only support the same qemu version on source and destination
colo: Setup ram cache in normal migration path
colo: Replace migration_incoming_colo_enabled() with migrate_colo()
colo: Remove ENABLE_COLO loadvm command functions
colo: Don't send ENABLE_COLO command
colo: Reject colo with postcopy capability enabled
colo: Reject colo with block migration capability enabled
ram: Remove useless colo special-casing

 docs/COLO-FT.txt         |  2 ++
 include/migration/colo.h |  3 --
 migration/colo.c         |  2 +-
 migration/migration.c    | 63 ++++++++++------------------------------
 migration/options.c      | 14 +++++++++
 migration/ram.c          | 14 ++++-----
 migration/savevm.c       | 21 +-------------
 migration/savevm.h       |  1 -
 migration/trace-events   |  1 -
 9 files changed, 41 insertions(+), 80 deletions(-)
Signature · 1 line
2.39.2
#4 lore raw

PATCH 2/8 colo: Setup ram cache in normal migration path

Now that x-colo capability needs to be always enabled on the incoming side we can use that to initialize the ram cache in the normal migration path.

---
 migration/migration.c | 16 ++++++++++++----
 migration/savevm.c    | 10 +---------
 2 files changed, 13 insertions(+), 13 deletions(-)
migration/migration.c +12 -4
index dc05c6f6ea..050bd8ffc8 100644
@@ -404,10 +404,6 @@ int migration_incoming_enable_colo(void)
         return -EINVAL;
     }
 
-    if (ram_block_discard_disable(true)) {
-        error_report("COLO: cannot disable RAM discard");
-        return -EBUSY;
-    }
     migration_colo_enabled = true;
     return 0;
 }
@@ -519,6 +515,18 @@ process_incoming_migration_co(void *opaque)
         goto fail;
     }
 
+    if (migrate_colo()) {
+        if (ram_block_discard_disable(true)) {
+            error_report("COLO: cannot disable RAM discard");
+            goto fail;
+        }
+
+        if (colo_init_ram_cache() < 0) {
+            error_report("Init ram cache failed");
+            goto fail;
+        }
+    }
+
     mis->largest_page_size = qemu_ram_pagesize_largest();
     postcopy_state_set(POSTCOPY_INCOMING_NONE);
     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
migration/savevm.c +1 -9
index bc284087f9..155abb0fda 100644
@@ -2302,15 +2302,7 @@ static int loadvm_handle_recv_bitmap(MigrationIncomingState *mis,
 
 static int loadvm_process_enable_colo(MigrationIncomingState *mis)
 {
-    int ret = migration_incoming_enable_colo();
-
-    if (!ret) {
-        ret = colo_init_ram_cache();
-        if (ret) {
-            migration_incoming_disable_colo();
-        }
-    }
-    return ret;
+    return migration_incoming_enable_colo();
 }
 
 /*
Signature · 1 line
2.39.2
#8 lore raw

PATCH 5/8 colo: Don't send ENABLE_COLO command

In reply to Lukas Straub

We should only migrate to qemu with newer version, and there it's not needed anymore.

---
 migration/migration.c  | 5 -----
 migration/savevm.c     | 6 ------
 migration/savevm.h     | 1 -
 migration/trace-events | 1 -
 4 files changed, 13 deletions(-)
migration/migration.c +0 -5
index 1d347533f9..a954ff4f7d 100644
@@ -2929,11 +2929,6 @@ static void *migration_thread(void *opaque)
         qemu_savevm_send_postcopy_advise(s->to_dst_file);
     }
 
-    if (migrate_colo()) {
-        /* Notify migration destination that we enable COLO */
-        qemu_savevm_send_colo_enable(s->to_dst_file);
-    }
-
     qemu_savevm_state_setup(s->to_dst_file);
 
     qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP,
migration/savevm.c +0 -6
index 3a1de15bd0..5986f852b2 100644
@@ -1035,12 +1035,6 @@ static void qemu_savevm_command_send(QEMUFile *f,
     qemu_fflush(f);
 }
 
-void qemu_savevm_send_colo_enable(QEMUFile *f)
-{
-    trace_savevm_send_colo_enable();
-    qemu_savevm_command_send(f, MIG_CMD_ENABLE_COLO, 0, NULL);
-}
-
 void qemu_savevm_send_ping(QEMUFile *f, uint32_t value)
 {
     uint32_t buf;
migration/savevm.h +0 -1
index fb636735f0..cb3318e9e2 100644
@@ -57,7 +57,6 @@ void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
                                            uint16_t len,
                                            uint64_t *start_list,
                                            uint64_t *length_list);
-void qemu_savevm_send_colo_enable(QEMUFile *f);
 void qemu_savevm_live_state(QEMUFile *f);
 int qemu_save_device_state(QEMUFile *f);
 
migration/trace-events +0 -1
index cdaef7a1ea..bd3ec40e31 100644
@@ -36,7 +36,6 @@ savevm_send_ping(uint32_t val) "0x%x"
 savevm_send_postcopy_listen(void) ""
 savevm_send_postcopy_run(void) ""
 savevm_send_postcopy_resume(void) ""
-savevm_send_colo_enable(void) ""
 savevm_send_recv_bitmap(char *name) "%s"
 savevm_state_setup(void) ""
 savevm_state_resume_prepare(void) ""
Signature · 1 line
2.39.2
#7 lore raw

PATCH 4/8 colo: Remove ENABLE_COLO loadvm command functions

In reply to Lukas Straub

No need for it anymore now that x-colo capability is required on incoming side. Still accept the command as noop, for compatibility with older qemu.

---
 include/migration/colo.h |  2 --
 migration/migration.c    | 26 --------------------------
 migration/savevm.c       |  7 +------
 3 files changed, 1 insertion(+), 34 deletions(-)
include/migration/colo.h +0 -2
index 4a1955067b..addbc24fcf 100644
@@ -25,8 +25,6 @@ void migrate_start_colo_process(MigrationState *s);
 bool migration_in_colo_state(void);
 
 /* loadvm */
-int migration_incoming_enable_colo(void);
-void migration_incoming_disable_colo(void);
 bool migration_incoming_in_colo_state(void);
 
 COLOMode get_colo_mode(void);
migration/migration.c +0 -26
index 2506fd63f7..1d347533f9 100644
@@ -378,31 +378,6 @@ int migrate_send_rp_req_pages(MigrationIncomingState *mis,
     return migrate_send_rp_message_req_pages(mis, rb, start);
 }
 
-static bool migration_colo_enabled;
-void migration_incoming_disable_colo(void)
-{
-    ram_block_discard_disable(false);
-    migration_colo_enabled = false;
-}
-
-int migration_incoming_enable_colo(void)
-{
-#ifndef CONFIG_REPLICATION
-    error_report("ENABLE_COLO command come in migration stream, but COLO "
-                 "module is not built in");
-    return -ENOTSUP;
-#endif
-
-    if (!migrate_colo()) {
-        error_report("ENABLE_COLO command come in migration stream, but c-colo "
-                     "capability is not set");
-        return -EINVAL;
-    }
-
-    migration_colo_enabled = true;
-    return 0;
-}
-
 void migrate_add_address(SocketAddress *address)
 {
     MigrationIncomingState *mis = migration_incoming_get_current();
@@ -480,7 +455,6 @@ static void process_incoming_migration_bh(void *opaque)
             runstate_set(RUN_STATE_PAUSED);
         }
     } else if (migrate_colo()) {
-        migration_incoming_disable_colo();
         vm_start();
     } else {
         runstate_set(global_state_get_runstate());
migration/savevm.c +1 -6
index 155abb0fda..3a1de15bd0 100644
@@ -2300,11 +2300,6 @@ static int loadvm_handle_recv_bitmap(MigrationIncomingState *mis,
     return 0;
 }
 
-static int loadvm_process_enable_colo(MigrationIncomingState *mis)
-{
-    return migration_incoming_enable_colo();
-}
-
 /*
  * Process an incoming 'QEMU_VM_COMMAND'
  * 0           just a normal return
@@ -2387,7 +2382,7 @@ static int loadvm_process_command(QEMUFile *f)
         return loadvm_handle_recv_bitmap(mis, len);
 
     case MIG_CMD_ENABLE_COLO:
-        return loadvm_process_enable_colo(mis);
+        return 0;
     }
 
     return 0;
Signature · 1 line
2.39.2
#6 lore raw

PATCH 3/8 colo: Replace migration_incoming_colo_enabled() with migrate_colo()

In reply to Lukas Straub
---
 include/migration/colo.h | 1 -
 migration/colo.c         | 2 +-
 migration/migration.c    | 7 +------
 migration/ram.c          | 2 +-
 4 files changed, 3 insertions(+), 9 deletions(-)
include/migration/colo.h +0 -1
index eaac07f26d..4a1955067b 100644
@@ -27,7 +27,6 @@ bool migration_in_colo_state(void);
 /* loadvm */
 int migration_incoming_enable_colo(void);
 void migration_incoming_disable_colo(void);
-bool migration_incoming_colo_enabled(void);
 bool migration_incoming_in_colo_state(void);
 
 COLOMode get_colo_mode(void);
migration/colo.c +1 -1
index 72f4f7b37e..8b0e7c9af3 100644
@@ -927,7 +927,7 @@ int coroutine_fn colo_incoming_co(void)
 
     assert(qemu_mutex_iothread_locked());
 
-    if (!migration_incoming_colo_enabled()) {
+    if (!migrate_colo()) {
         return 0;
     }
 
migration/migration.c +1 -6
index 050bd8ffc8..2506fd63f7 100644
@@ -379,11 +379,6 @@ int migrate_send_rp_req_pages(MigrationIncomingState *mis,
 }
 
 static bool migration_colo_enabled;
-bool migration_incoming_colo_enabled(void)
-{
-    return migration_colo_enabled;
-}
-
 void migration_incoming_disable_colo(void)
 {
     ram_block_discard_disable(false);
@@ -484,7 +479,7 @@ static void process_incoming_migration_bh(void *opaque)
         } else {
             runstate_set(RUN_STATE_PAUSED);
         }
-    } else if (migration_incoming_colo_enabled()) {
+    } else if (migrate_colo()) {
         migration_incoming_disable_colo();
         vm_start();
     } else {
migration/ram.c +1 -1
index 5283a75f02..e3eadd08cd 100644
@@ -3892,7 +3892,7 @@ static int ram_load_precopy(QEMUFile *f)
              * speed of the migration, but it obviously reduce the downtime of
              * back-up all SVM'S memory in COLO preparing stage.
              */
-            if (migration_incoming_colo_enabled()) {
+            if (migrate_colo()) {
                 if (migration_incoming_in_colo_state()) {
                     /* In COLO stage, put all pages into cache temporarily */
                     host = colo_cache_from_block_offset(block, addr, true);
Signature · 1 line
2.39.2
#11 lore raw

PATCH 8/8 ram: Remove useless colo special-casing

In reply to Lukas Straub

This is not needed, as colo primary side everything is more or less a new migration for every checkpoint.

Also, we only enter colo mode after the precopy migration is finished so this if is always taken. Still add an assert just in case.

---
 migration/ram.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
migration/ram.c +6 -6
index e3eadd08cd..e5c1146360 100644
@@ -2994,17 +2994,17 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
     RAMBlock *block;
     int ret;
 
+    assert(!migration_in_colo_state());
+
     if (compress_threads_save_setup()) {
         return -1;
     }
 
-    /* migration has already setup the bitmap, reuse it. */
-    if (!migration_in_colo_state()) {
-        if (ram_init_all(rsp) != 0) {
-            compress_threads_save_cleanup();
-            return -1;
-        }
+    if (ram_init_all(rsp) != 0) {
+        compress_threads_save_cleanup();
+        return -1;
     }
+
     (*rsp)->pss[RAM_CHANNEL_PRECOPY].pss_channel = f;
 
     WITH_RCU_READ_LOCK_GUARD() {
Signature · 1 line
2.39.2
#10 lore raw

PATCH 7/8 colo: Reject colo with block migration capability enabled

In reply to Lukas Straub
---
 migration/migration.c | 4 ----
 migration/options.c   | 5 +++++
 2 files changed, 5 insertions(+), 4 deletions(-)
migration/migration.c +0 -4
index 9860f960f2..270130579f 100644
@@ -1571,10 +1571,6 @@ static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc,
     }
 
     if (blk || blk_inc) {
-        if (migrate_colo()) {
-            error_setg(errp, "No disk migration is required in COLO mode");
-            return false;
-        }
         if (migrate_block() || migrate_block_incremental()) {
             error_setg(errp, "Command options are incompatible with "
                        "current migration capabilities");
migration/options.c +5 -0
index d3d4525d40..1e9659fcb3 100644
@@ -451,6 +451,11 @@ bool migrate_caps_check(bool *old_caps, bool *new_caps, Error **errp)
             error_setg(errp, "COLO is not compatible with postcopy");
             return false;
         }
+
+        if (new_caps[MIGRATION_CAPABILITY_BLOCK]) {
+            error_setg(errp, "COLO is not compatible with block migration");
+            return false;
+        }
     }
 #endif
 
Signature · 1 line
2.39.2
#9 lore raw

PATCH 6/8 colo: Reject colo with postcopy capability enabled

In reply to Lukas Straub
---
 migration/migration.c | 5 +++--
 migration/options.c   | 9 +++++++++
 2 files changed, 12 insertions(+), 2 deletions(-)
migration/migration.c +3 -2
index a954ff4f7d..9860f960f2 100644
@@ -2333,9 +2333,10 @@ static void migration_completion(MigrationState *s)
         goto fail;
     }
 
-    if (migrate_colo() && s->state == MIGRATION_STATUS_ACTIVE) {
+    if (migrate_colo()) {
         /* COLO does not support postcopy */
-        migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE,
+        assert(s->state != MIGRATION_STATUS_POSTCOPY_ACTIVE);
+        migrate_set_state(&s->state, current_active_state,
                           MIGRATION_STATUS_COLO);
     } else {
         migrate_set_state(&s->state, current_active_state,
migration/options.c +9 -0
index b62ab30cd5..d3d4525d40 100644
@@ -445,8 +445,17 @@ bool migrate_caps_check(bool *old_caps, bool *new_caps, Error **errp)
         error_append_hint(errp, "Please enable replication before COLO.\n");
         return false;
     }
+#else
+    if (new_caps[MIGRATION_CAPABILITY_X_COLO]) {
+        if (new_caps[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
+            error_setg(errp, "COLO is not compatible with postcopy");
+            return false;
+        }
+    }
 #endif
 
+
+
     if (new_caps[MIGRATION_CAPABILITY_POSTCOPY_RAM]) {
         /* This check is reasonably expensive, so only when it's being
          * set the first time, also it's only the destination that needs
Signature · 1 line
2.39.2
#2 lore raw

RE: [PATCH 1/8] colo: Only support the same qemu version on source and destination

In reply to Lukas Straub
Quoted 25 lines

-----Original Message-----
From: [email protected] <qemu- [email protected]> On Behalf Of Lukas Straub
Sent: Thursday, June 22, 2023 5:15 AM
To: qemu-devel <[email protected]>
Cc: Zhang, Hailiang <[email protected]>; Juan Quintela <[email protected]>; Peter Xu <[email protected]>; Leonardo Bras <[email protected]>; Zhang, Chen <[email protected]>
Subject: [PATCH 1/8] colo: Only support the same qemu version on source and destination

---
 docs/COLO-FT.txt | 2 ++
 1 file changed, 2 insertions(+)
docs/COLO-FT.txt index renamed from docs/COLO-FT.txt +0 -0

2e760a4aee..8e64480dbd 100644

docs/COLO-FT.txt +0 -0
@@ -148,6 +148,8 @@ in test procedure.
 Note: Here we are running both instances on the same host for testing,
Quoted 3 lines

change the IP Addresses if you want to run it on two hosts. Initially
127.0.0.1 is the Primary Host and 127.0.0.2 is the Secondary Host.
+Note: COLO is a experimental feature,

an experimental feature

Quoted 1 line

so currently is should only be

it should ...

Quoted 6 lines

+used with the same qemu version on sourcee and target.

== Startup qemu ==
1. Primary:
--
2.39.2

#3 lore raw

RE: [PATCH 1/8] colo: Only support the same qemu version on source and destination

Quoted 44 lines

-----Original Message-----
From: Dong, Eddie <[email protected]>
Sent: Friday, June 23, 2023 1:17 AM
To: Lukas Straub <[email protected]>; qemu-devel <qemu- [email protected]>
Cc: Zhang, Hailiang <[email protected]>; Juan Quintela <[email protected]>; Peter Xu <[email protected]>; Leonardo Bras <[email protected]>; Zhang, Chen <[email protected]>
Subject: RE: [PATCH 1/8] colo: Only support the same qemu version on source and destination

Quoted 23 lines

-----Original Message-----
From: [email protected] <qemu- [email protected]> On Behalf Of Lukas Straub
Sent: Thursday, June 22, 2023 5:15 AM
To: qemu-devel <[email protected]>
Cc: Zhang, Hailiang <[email protected]>; Juan Quintela <[email protected]>; Peter Xu <[email protected]>; Leonardo Bras <[email protected]>; Zhang, Chen <[email protected]>
Subject: [PATCH 1/8] colo: Only support the same qemu version on source and destination

---
 docs/COLO-FT.txt | 2 ++
 1 file changed, 2 insertions(+)
docs/COLO-FT.txt index renamed from docs/COLO-FT.txt +0 -0

2e760a4aee..8e64480dbd 100644

docs/COLO-FT.txt +0 -0
@@ -148,6 +148,8 @@ in test procedure.
 Note: Here we are running both instances on the same host for
Quoted 4 lines

testing, change the IP Addresses if you want to run it on two hosts. Initially
127.0.0.1 is the Primary Host and 127.0.0.2 is the Secondary Host.
+Note: COLO is a experimental feature,

an experimental feature

Quoted 4 lines
Quoted 1 line

so currently is should only be

it should ...

Quoted 1 line

+used with the same qemu version on sourcee and target.

S/sourcee/source

Thanks
Chen

Quoted 5 lines
Quoted 5 lines

== Startup qemu ==
1. Primary:
--
2.39.2

#5 lore raw

RE: [PATCH 2/8] colo: Setup ram cache in normal migration path

In reply to Lukas Straub
Quoted 25 lines

-----Original Message-----
From: Lukas Straub <[email protected]>
Sent: Thursday, June 22, 2023 8:15 PM
To: qemu-devel <[email protected]>
Cc: Zhang, Hailiang <[email protected]>; Juan Quintela <[email protected]>; Peter Xu <[email protected]>; Leonardo Bras <[email protected]>; Zhang, Chen <[email protected]>
Subject: [PATCH 2/8] colo: Setup ram cache in normal migration path

Now that x-colo capability needs to be always enabled on the incoming side we can use that to initialize the ram cache in the normal migration path.

---
 migration/migration.c | 16 ++++++++++++----
 migration/savevm.c    | 10 +---------
 2 files changed, 13 insertions(+), 13 deletions(-)
migration/migration.c index renamed from migration/migration.c +0 -0

dc05c6f6ea..050bd8ffc8 100644

migration/migration.c int migration_incoming_enable_colo(void) · line 410

Quoted 7 lines
migration/migration.c +0 -4
@@ -404,10 +404,6 @@ int migration_incoming_enable_colo(void)
         return -EINVAL;
     }
 
-    if (ram_block_discard_disable(true)) {
-        error_report("COLO: cannot disable RAM discard");
-        return -EBUSY;
-    }

COLO may call here when occur each checkpoint to pin the guest memory, in the Colo_incoming_process_checkpoint(). Is it still working after move to the "process_incoming_migration_co()" ?

Quoted 5 lines
migration/migration.c +0 -0
     migration_colo_enabled = true;
     return 0;
 }
@@ -519,6 +515,18 @@ process_incoming_migration_co(void *opaque)
         goto fail;

migration/migration.c process_incoming_migration_co(void *opaque) · line 525

Quoted 10 lines
migration/migration.c +8 -0
     }
 
+    if (migrate_colo()) {
+        if (ram_block_discard_disable(true)) {
+            error_report("COLO: cannot disable RAM discard");
+            goto fail;
+        }
+
+        if (colo_init_ram_cache() < 0) {
+            error_report("Init ram cache failed");

Change the code path to here need to add COLO tag to the error:
error_report("Init COLO ram cache failed");

It looks here removed the original migration_incoming_disable_colo(), But maybe it's OK for goto fail directly.

Thanks
Chen

Quoted 30 lines
migration/migration.c +4 -0
+            goto fail;
+        }
+    }
+
     mis->largest_page_size = qemu_ram_pagesize_largest();
     postcopy_state_set(POSTCOPY_INCOMING_NONE);
     migrate_set_state(&mis->state, MIGRATION_STATUS_NONE, diff --git

a/migration/savevm.c b/migration/savevm.c index bc284087f9..155abb0fda 100644

migration/savevm.c +1 -10
@@ -2302,15 +2302,7 @@ static int
 loadvm_handle_recv_bitmap(MigrationIncomingState *mis,
 
 static int loadvm_process_enable_colo(MigrationIncomingState *mis)  {
-    int ret = migration_incoming_enable_colo();
-
-    if (!ret) {
-        ret = colo_init_ram_cache();
-        if (ret) {
-            migration_incoming_disable_colo();
-        }
-    }
-    return ret;
+    return migration_incoming_enable_colo();
 }
 
 /*
--

2.39.2