Skip to contentSkip to conversation

Implement dirty ring for pre-copy migration

PATCH In review Shota Imamura 2 participants 6 messages Jun 20, 2024 – Jun 25, 2024 KVM

lore
Share & tools
lore
#0 lore raw

cover Implement dirty ring for pre-copy migration

This patch series introduces the dirty ring as an additional method for dirty tracking, alongside the existing dirty bitmap.

Shota Imamura (2):
migration: Implement dirty ring
qtest/migration: Add dirty ring tests

 accel/kvm/kvm-all.c            |  36 ++++++++-
 include/exec/ram_addr.h        | 131 +++++++++++++++++++++++++++++++--
 include/exec/ramlist.h         |  48 ++++++++++++
 include/migration/misc.h       |   4 +-
 include/qemu/bitops.h          |  23 ++++++
 migration/migration-hmp-cmds.c |   2 +
 migration/migration.c          |  27 ++++++-
 migration/migration.h          |   6 ++
 migration/ram.c                | 127 ++++++++++++++++++++++++++++----
 qemu-options.hx                |  29 ++++++++
 system/physmem.c               | 128 +++++++++++++++++++++++++++++++-
 system/vl.c                    |  63 +++++++++++++++-
 tests/qtest/migration-test.c   |  78 ++++++++++++++++++--
 13 files changed, 667 insertions(+), 35 deletions(-)
Signature · 1 line
2.34.1
#1 lore raw

PATCH 1/2 migration: Implement dirty ring

This commit implements the dirty ring as an alternative dirty tracking method to the dirty bitmap.

While the dirty ring has already been implemented in accel/kvm using KVM's dirty ring, it was designed to set bits in the ramlist and ramblock bitmap. This commit introduces a new dirty ring to replace the bitmap, allowing the use of the dirty ring even without KVM. When using KVM's dirty ring, this implementation maximizes its effectiveness.

To enable the dirty ring, specify the startup option "-migration dirty-logging=ring,dirty-ring-size=N". To use the bitmap, either specify nothing or "-migration dirty-logging=bitmap". If the dirty ring becomes full, it falls back to the bitmap for that round.

---
 accel/kvm/kvm-all.c            |  36 ++++++++-
 include/exec/ram_addr.h        | 131 +++++++++++++++++++++++++++++++--
 include/exec/ramlist.h         |  48 ++++++++++++
 include/migration/misc.h       |   4 +-
 include/qemu/bitops.h          |  23 ++++++
 migration/migration-hmp-cmds.c |   2 +
 migration/migration.c          |  27 ++++++-
 migration/migration.h          |   6 ++
 migration/ram.c                | 127 ++++++++++++++++++++++++++++----
 qemu-options.hx                |  29 ++++++++
 system/physmem.c               | 128 +++++++++++++++++++++++++++++++-
 system/vl.c                    |  63 +++++++++++++++-
 12 files changed, 597 insertions(+), 27 deletions(-)
accel/kvm/kvm-all.c +35 -1
index 854cb86b22..91410d682f 100644
@@ -667,7 +667,13 @@ static void kvm_dirty_ring_mark_page(KVMState *s, uint32_t as_id,
         return;
     }
 
-    set_bit(offset, mem->dirty_bmap);
+    if (!test_and_set_bit(offset, mem->dirty_bmap) &&
+        mem->flags & KVM_MEM_LOG_DIRTY_PAGES &&
+        migration_has_dirty_ring()) {
+        unsigned long pfn =
+            (mem->ram_start_offset >> TARGET_PAGE_BITS) + offset;
+        ram_list_enqueue_dirty(pfn);
+    }
 }
 
 static bool dirty_gfn_is_dirtied(struct kvm_dirty_gfn *gfn)
@@ -1675,6 +1681,34 @@ static void kvm_log_sync_global(MemoryListener *l, bool last_stage)
     /* Flush all kernel dirty addresses into KVMSlot dirty bitmap */
     kvm_dirty_ring_flush();
 
+    if (!ram_list_enqueue_dirty_full()) {
+        cpu_physical_memory_set_dirty_ring(ram_list_get_enqueue_dirty());
+
+        if (s->kvm_dirty_ring_with_bitmap && last_stage) {
+            kvm_slots_lock();
+            for (i = 0; i < s->nr_slots; i++) {
+                mem = &kml->slots[i];
+                if (mem->memory_size &&
+                    mem->flags & KVM_MEM_LOG_DIRTY_PAGES &&
+                    kvm_slot_get_dirty_log(s, mem)) {
+                    kvm_slot_sync_dirty_pages(mem);
+                }
+            }
+            kvm_slots_unlock();
+        }
+
+        kvm_slots_lock();
+        for (i = 0; i < s->nr_slots; i++) {
+            mem = &kml->slots[i];
+            if (mem->memory_size && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
+                kvm_slot_reset_dirty_pages(mem);
+            }
+        }
+        kvm_slots_unlock();
+
+        return;
+    }
+
     /*
      * TODO: make this faster when nr_slots is big while there are
      * only a few used slots (small VMs).
include/exec/ram_addr.h +126 -5

170 lines not on this page · raw message

include/exec/ramlist.h +48 -0

78 lines not on this page · raw message

include/migration/misc.h +3 -1
index bfadc5613b..1335e0958e 100644
@@ -51,13 +51,15 @@ AnnounceParameters *migrate_announce_params(void);
 void dump_vmstate_json_to_file(FILE *out_fp);
 
 /* migration/migration.c */
-void migration_object_init(void);
+void migration_object_init(QDict *qdict);
 void migration_shutdown(void);
 bool migration_is_idle(void);
 bool migration_is_active(void);
 bool migration_is_device(void);
 bool migration_thread_is_self(void);
 bool migration_is_setup_or_active(void);
+bool migration_has_dirty_ring(void);
+unsigned long migration_get_dirty_ring_size(void);
 
 typedef enum MigrationEventType {
     MIG_EVENT_PRECOPY_SETUP,
include/qemu/bitops.h +23 -0
index 2c0a2fe751..ea5d02d0ec 100644
@@ -108,6 +108,29 @@ static inline int test_and_set_bit(long nr, unsigned long *addr)
     return (old & mask) != 0;
 }
 
+/**
+ * test_and_set_bit_atomic - Set a bit and return its old value atomically
+ * @nr: Bit to set
+ * @addr: Address to count from
+ */
+static inline int test_and_set_bit_atomic(long nr, unsigned long *addr)
+{
+    unsigned long mask = BIT_MASK(nr);
+    unsigned long *p = addr + BIT_WORD(nr);
+    unsigned long old;
+    unsigned long desired;
+
+    do {
+        old = qatomic_read(p);
+        if (old & mask) {
+            return false;
+        }
+        desired = old | mask;
+    } while (qatomic_cmpxchg(p, old, desired) != desired);
+
+    return true;
+}
+
 /**
  * test_and_clear_bit - Clear a bit and return its old value
  * @nr: Bit to clear
migration/migration-hmp-cmds.c +2 -0
index 9f0e8029e0..02b1fa7b29 100644
@@ -48,6 +48,8 @@ static void migration_global_dump(Monitor *mon)
                    ms->send_section_footer ? "on" : "off");
     monitor_printf(mon, "clear-bitmap-shift: %u\n",
                    ms->clear_bitmap_shift);
+    monitor_printf(mon, "dirty-logging-method: %s\n",
+                   migration_has_dirty_ring() ? "ring" : "bitmap");
 }
 
 void hmp_info_migrate(Monitor *mon, const QDict *qdict)
migration/migration.c +26 -1
index e1b269624c..a059ac7626 100644
@@ -44,6 +44,7 @@
 #include "qapi/qapi-events-migration.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/qmp/qnull.h"
+#include "qapi/qmp/qdict.h"
 #include "qemu/rcu.h"
 #include "postcopy-ram.h"
 #include "qemu/thread.h"
@@ -207,12 +208,20 @@ static int migration_stop_vm(MigrationState *s, RunState state)
     return ret;
 }
 
-void migration_object_init(void)
+void migration_object_init(QDict *qdict)
 {
     /* This can only be called once. */
     assert(!current_migration);
     current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION));
 
+    const char *logging_method = qdict_get_try_str(qdict, "dirty-logging");
+    if (logging_method == NULL || strcmp(logging_method, "bitmap") == 0) {
+        current_migration->dirty_ring_size = 0;
+    } else if (strcmp(logging_method, "ring") == 0) {
+        current_migration->dirty_ring_size = qdict_get_int(qdict,
+                                                           "dirty-ring-size");
+    }
+
     /*
      * Init the migrate incoming object as well no matter whether
      * we'll use it or not.
@@ -1632,6 +1641,20 @@ bool migrate_mode_is_cpr(MigrationState *s)
     return s->parameters.mode == MIG_MODE_CPR_REBOOT;
 }
 
+bool migration_has_dirty_ring(void)
+{
+    MigrationState *s = current_migration;
+
+    return s->dirty_ring_size != 0;
+}
+
+unsigned long migration_get_dirty_ring_size(void)
+{
+    MigrationState *s = current_migration;
+
+    return s->dirty_ring_size;
+}
+
 int migrate_init(MigrationState *s, Error **errp)
 {
     int ret;
@@ -3806,6 +3829,8 @@ static void migration_instance_init(Object *obj)
     qemu_sem_init(&ms->wait_unplug_sem, 0);
     qemu_sem_init(&ms->postcopy_qemufile_src_sem, 0);
     qemu_mutex_init(&ms->qemu_file_lock);
+
+    ms->dirty_ring_size = 0;
 }
 
 /*
migration/migration.h +6 -0
index 6af01362d4..4dfdf98acc 100644
@@ -457,6 +457,12 @@ struct MigrationState {
     bool switchover_acked;
     /* Is this a rdma migration */
     bool rdma_migration;
+
+    /*
+     * The number of elements in the dirty-ring.
+     * It is 0 if the dirty-ring is not enabled.
+     */
+    unsigned long dirty_ring_size;
 };
 
 void migrate_set_state(int *state, int old_state, int new_state);
migration/ram.c +111 -16

169 lines not on this page · raw message

qemu-options.hx +29 -0
index 8ca7f34ef0..f1ec592ec0 100644
@@ -556,6 +556,35 @@ SRST
         -numa hmat-cache,node-id=1,size=10K,level=1,associativity=direct,policy=write-back,line=8
 ERST
 
+DEF("migration", HAS_ARG, QEMU_OPTION_migration,
+    "-migration [dirty-logging=method][,dirty-ring-size=n]\n"
+    "                Configure migration settings.\n"
+    "                dirty-logging: Specify the dirty logging method.\n"
+    "                    bitmap: Use bitmap-based dirty logging (default).\n"
+    "                    ring: Use ring-based dirty logging.\n"
+    "                dirty-ring-size: Specify the size of the dirty ring buffer.\n"
+    "                    This option is only applicable if dirty-logging is set to ring.\n"
+    "                    The size must be a power of 2.\n"
+    "                    Example: -migration dirty-logging=ring,dirty-ring-size=1024\n", QEMU_ARCH_ALL)
+SRST
+``-migration [dirty-logging=method,dirty-ring-size=n]``
+    Configure migration settings.
+
+    ``dirty-logging=bitmap``
+        Use bitmap-based dirty logging. This is the default method for tracking changes to memory pages during migration.
+
+    ``dirty-logging=ring``
+        Use ring-based dirty logging. This method uses a ring buffer to track changes to memory pages, which can be more efficient in some scenarios.
+
+    ``dirty-ring-size=n``
+        Specify the size of the dirty ring buffer when using ring-based dirty logging. The size must be a power of 2 (e.g., 1024, 2048, 4096). This option is only applicable if dirty-logging is set to ring.
+
+    Example:
+    ::
+
+        -migration dirty-logging=ring,dirty-ring-size=1024
+ERST
+
 DEF("add-fd", HAS_ARG, QEMU_OPTION_add_fd,
     "-add-fd fd=fd,set=set[,opaque=opaque]\n"
     "                Add 'fd' to fd 'set'\n", QEMU_ARCH_ALL)
system/physmem.c +127 -1

149 lines not on this page · raw message

system/vl.c +61 -2

105 lines not on this page · raw message

Signature · 1 line
2.34.1
#2 lore raw

Re: [PATCH 1/2] migration: Implement dirty ring

Hi, Shota,

Quoted 8 lines · Shota Imamura

This commit implements the dirty ring as an alternative dirty tracking method to the dirty bitmap.

While the dirty ring has already been implemented in accel/kvm using KVM's dirty ring, it was designed to set bits in the ramlist and ramblock bitmap. This commit introduces a new dirty ring to replace the bitmap, allowing the use of the dirty ring even without KVM. When using KVM's dirty ring, this implementation maximizes its effectiveness.

It looks like this patch will introduce a ring but still it keeps the bitmaps around.

Could you elaborate your motivation of this work? It'll be interesting to know whether you did any kind of measurement around it.

Quoted 5 lines

To enable the dirty ring, specify the startup option "-migration dirty-logging=ring,dirty-ring-size=N". To use the bitmap, either specify nothing or "-migration dirty-logging=bitmap". If the dirty ring becomes full, it falls back to the bitmap for that round.

I remember adding such option is not suggested. We may consider using either QMP to setup a migration parameter, or something else.

Thanks,

Signature · 1 line
Peter Xu
#3 lore raw

Re: [PATCH 1/2] migration: Implement dirty ring

Dear Peter Xu,

Thank you for your feedback.

Quoted 5 lines

It looks like this patch will introduce a ring but still it keeps the bitmaps around.

Could you elaborate your motivation of this work? It’ll be interesting to know whether you did any kind of measurement around it.

First of all, I apologize for the lack of explanation.
To provide more details, the motivation for this work stems from the current QEMU implementation, where pages obtained from the KVM ring are set into the KVMSlot/RAMList/RAMBlock bitmaps. Consequently, when the migration thread sends data, it ends up scanning the bitmap (resulting in O(N) time complexity). I aimed to improve this situation.

Here are the steps and considerations in my implementation plan:

1. Call MigrationOps::ram_save_target_page inside kvm_dirty_ring_reap_one.

The approach involves QEMU holding neither bitmaps nor rings and sending immediately. However, this would require non-migration threads (like accel threads) to send pages, necessitating a synchronization mechanism with the migration thread, which I deemed would increase code complexity. Additionally, if future non-KVM accels provided their rings, we would have to write similar code in different places, increasing future maintenance costs. For these reasons, I decided against an implementation where page sending occurs within accel/kvm and opted for a separate ring within QEMU.

2. Use a ring as an alternative to bitmaps.

The approach involves implementing a ring within QEMU and inserting pages into the ring instead of setting them into bitmaps in functions like kvm_dirty_ring_mark_page, cpu_physical_memory_set_dirty_lebitmap, and cpu_physical_memory_set_dirty_range. Then, pages are retrieved from the ring in ram_find_and_save_block. However, this approach necessitates immediate sending of pages when the ring is full, which might involve non-migration threads sending pages, leading to the same issues as mentioned in step 1. Given the ring has a limited capacity, if there are enough dirty pages to fill the ring, the cost difference between operating the ring and scanning the entire bitmap would be negligible. Hence, I decided to fall back to bitmaps when the ring is full.

3. Use bitmaps when the ring is full.

The approach involves setting bitmaps while simultaneously inserting pages into the ring in functions like kvm_dirty_ring_mark_page, cpu_physical_memory_set_dirty_lebitmap, and cpu_physical_memory_set_dirty_range. If the ring is not full, it is used in ram_find_and_save_block; otherwise, bitmaps are used. This way, only the migration thread sends pages. Additionally, by checking if a page is already in the ring (O(1) complexity), redundant entries are avoided. However, enqueuing and dequeuing are handled by different threads, which could result in a situation where pages exist in the bitmap but not in the ring once it is full. Identifying these pages would require locking and scanning the entire bitmap.

4. Use two rings to revert to the ring after falling back to the bitmap within a round.

As mentioned earlier, once the fallback to the bitmap occurs, pages that get dirty after the ring is full cannot be captured by the ring. This would necessitate using bitmaps until the final round or periodically locking and scanning the entire bitmap to synchronize with the ring. To improve this, I considered using two rings: one for enqueueing and one for dequeuing. Pages are inserted into the enqueue ring in functions like kvm_dirty_ring_mark_page and cpu_physical_memory_set_dirty_range, and the rings are swapped in migration_sync_bitmap, with pages being retrieved in ram_find_and_save_block. This way, each call to migration_sync_bitmap (or ram_state_pending_exact) determines whether to use the ring or the bitmap in subsequent rounds.

Based on this reasoning, I implemented a system that combines bitmaps and rings.

Regarding performance, my local environment might be insufficient for proper measurement, but I obtained the following results by migrating after booting the latest Linux and Buildroot with an open login shell:

Commands used:

```
# src
sudo ./qemu-system-x86_64 \
-accel kvm,dirty-ring-size=1024 \
-m 8G \
-boot c \
-kernel ~/path/to/linux/arch/x86/boot/bzImage \
-hda ~/path/to/buildroot/output/images/rootfs.ext4 \
-append "root=/dev/sda rw console=ttyS0,115200 acpi=off" \
-nographic \
-migration dirty-logging=ring,dirty-ring-size=1024

# dst
sudo ./qemu-system-x86_64 \
-accel kvm,dirty-ring-size=1024 \
-m 8G \
-boot c \
-kernel ~/path/to/linux/arch/x86/boot/bzImage \
-hda ~/path/to/buildroot/output/images/rootfs.ext4 \
-append "root=/dev/sda rw console=ttyS0,115200 acpi=off" \
-nographic \
-incoming tcp:0:4444

# hmp
migrate_set_parameter max-bandwidth 1250
migrate tcp:0:4444
info migrate
```

Results for each memory size, measured 5 times:

```
# ring -m 8G
total time: 418 ms
total time: 416 ms
total time: 415 ms
total time: 416 ms
total time: 416 ms

# bitmap -m 8G
total time: 434 ms
total time: 421 ms
total time: 423 ms
total time: 430 ms
total time: 429 ms

# ring -m 16G
total time: 847 ms
total time: 852 ms
total time: 850 ms
total time: 848 ms
total time: 852 ms

# bitmap -m 16G
total time: 860 ms
total time: 862 ms
total time: 858 ms
total time: 859 ms
total time: 861 ms

# ring -m 32G
total time: 1616 ms
total time: 1625 ms
total time: 1612 ms
total time: 1612 ms
total time: 1630 ms

# bitmap -m 32G
total time: 1714 ms
total time: 1724 ms
total time: 1718 ms
total time: 1714 ms
total time: 1714 ms

# ring -m 64G
total time: 3451 ms
total time: 3452 ms
total time: 3449 ms
total time: 3451 ms
total time: 3450 ms

# bitmap -m 64G
total time: 3550 ms
total time: 3553 ms
total time: 3552 ms
total time: 3550 ms
total time: 3553 ms

# ring -m 96G
total time: 5185 ms
total time: 5186 ms
total time: 5183 ms
total time: 5191 ms
total time: 5191 ms

# bitmap -m 96G
total time: 5385 ms
total time: 5388 ms
total time: 5386 ms
total time: 5392 ms
total time: 5592 ms
```

It is natural that the implemented ring completes migration faster for all memory sizes, given that the conditions favor the ring due to the minimal memory workload. By the way, these are total migration times, with much of the overhead attributed to page sending and other IO operations.

I plan to conduct more detailed measurements going forward, but if you have any recommendations for good measurement methods, please let me know.

Quoted 2 lines

I remember adding such option is not suggested. We may consider using either QMP to setup a migration parameter, or something else.

I apologize for implementing this without knowledge of QEMU's policy. I will remove this option and instead implement it using migrate_set_parameter or migrate_set_capability. Is this approach acceptable?

This is my first time contributing to QEMU, so I appreciate your guidance.

Thank you and best regards,

--
Shota Imamura

2024年6月25日(火) 4:08 Peter Xu <[email protected]>:

Quoted 33 lines

Hi, Shota,

Quoted 8 lines · Shota Imamura

This commit implements the dirty ring as an alternative dirty tracking method to the dirty bitmap.

While the dirty ring has already been implemented in accel/kvm using KVM's dirty ring, it was designed to set bits in the ramlist and ramblock bitmap. This commit introduces a new dirty ring to replace the bitmap, allowing the use of the dirty ring even without KVM. When using KVM's dirty ring, this implementation maximizes its effectiveness.

It looks like this patch will introduce a ring but still it keeps the bitmaps around.

Could you elaborate your motivation of this work? It'll be interesting to know whether you did any kind of measurement around it.

Quoted 5 lines

To enable the dirty ring, specify the startup option "-migration dirty-logging=ring,dirty-ring-size=N". To use the bitmap, either specify nothing or "-migration dirty-logging=bitmap". If the dirty ring becomes full, it falls back to the bitmap for that round.

I remember adding such option is not suggested. We may consider using either QMP to setup a migration parameter, or something else.

Thanks,

--
Peter Xu

#4 lore raw

Re: [PATCH 1/2] migration: Implement dirty ring

Quoted 10 lines · Shota Imamura

Dear Peter Xu,

Thank you for your feedback.

Quoted 5 lines

It looks like this patch will introduce a ring but still it keeps the bitmaps around.

Could you elaborate your motivation of this work? It’ll be interesting to know whether you did any kind of measurement around it.

Quoted 6 lines

First of all, I apologize for the lack of explanation.
To provide more details, the motivation for this work stems from the current QEMU implementation, where pages obtained from the KVM ring are set into the KVMSlot/RAMList/RAMBlock bitmaps. Consequently, when the migration thread sends data, it ends up scanning the bitmap (resulting in O(N) time complexity). I aimed to improve this situation.

So is this a research project? Or do you have explicit goals, e.g. on reducing migration time, or make migration easier to converge?

These information might be helpful for reviewers to understand the ultimate goal of this series.

Quoted 5 lines

Here are the steps and considerations in my implementation plan:

1. Call MigrationOps::ram_save_target_page inside kvm_dirty_ring_reap_one.

Quoted 8 lines

The approach involves QEMU holding neither bitmaps nor rings and sending immediately. However, this would require non-migration threads (like accel threads) to send pages, necessitating a synchronization mechanism with the migration thread, which I deemed would increase code complexity. Additionally, if future non-KVM accels provided their rings, we would have to write similar code in different places, increasing future maintenance costs. For these reasons, I decided against an implementation where page sending occurs within accel/kvm and opted for a separate ring within QEMU.

Yes this won't trivially work cleanly, as ram_save_target_page() requires migration context. It may also not be thread-safe too. E.g. qemufile isn't thread-safe.

"maybe" it'll be easier to do it the other way round: allowing migration to fetch entry from a ring by calling a registered ring op.

Quoted 8 lines

2. Use a ring as an alternative to bitmaps.

The approach involves implementing a ring within QEMU and inserting pages into the ring instead of setting them into bitmaps in functions like kvm_dirty_ring_mark_page, cpu_physical_memory_set_dirty_lebitmap, and cpu_physical_memory_set_dirty_range. Then, pages are retrieved from the ring in ram_find_and_save_block.

(I think this might be what I mentioned above)

Quoted 7 lines

However, this approach necessitates immediate sending of pages when the ring is full, which might involve non-migration threads sending pages, leading to the same issues as mentioned in step 1. Given the ring has a limited capacity, if there are enough dirty pages to fill the ring, the cost difference between operating the ring and scanning the entire bitmap would be negligible. Hence, I decided to fall back to bitmaps when the ring is full.

Right, that was also what I found - the ring may not work as well when the guest is very busy. Here the question is migration is normally more challenging when that is the case.. and when with a pretty idle guest it's easier to migrate anyway even with bitmap-only. I think I noticed that pretty late.

Quoted 14 lines

3. Use bitmaps when the ring is full.

The approach involves setting bitmaps while simultaneously inserting pages into the ring in functions like kvm_dirty_ring_mark_page, cpu_physical_memory_set_dirty_lebitmap, and cpu_physical_memory_set_dirty_range. If the ring is not full, it is used in ram_find_and_save_block; otherwise, bitmaps are used. This way, only the migration thread sends pages. Additionally, by checking if a page is already in the ring (O(1) complexity), redundant entries are avoided. However, enqueuing and dequeuing are handled by different threads, which could result in a situation where pages exist in the bitmap but not in the ring once it is full. Identifying these pages would require locking and scanning the entire bitmap.

Yes, is this series using this approach?

I think the ring effect may be minimum if there is the bitmap already, because as long as the ring can fallback (to avoid hanging a vcpu for a long time) it means the bitmap can contain useful data and the bitmap scans will be needed, then it kind of invalidates the ring's benefit, IMHO.

What I was thinking a long time ago was something like this: we only use ring, as you said in (2) above, so no bitmap. Then the migration core can look like this:

  migration_thread():
    count=0
    while () {
        count++
        if ((count % ENFORCE_BACKGROUND_COUNT == 0) ||
            ring_empty())
            send_page(background.pop())
        else
            send_page(ring.pop())
    }

To explain it a bit:

- we will have two page iterators:

- background.pop() means we pop one background page. Here since we don't have a bitmap, we can't scan anything, the background page will be an iterator that will migrate the guest memory from page 0 to page N-1 (considering there's N guest pages), once and for all. It means we pop guest page 0, then 1, ... until N-1, then it's all done. After this iteration, nothing will be needed in background, all dirty will only reside in the ring.

- ring.pop() means we fetch one entry from the ring.

- here ENFORCE_BACKGROUND_COUNT will guarantee the background work can always make some progress over time, so that at some point it'll flush empty and only ring entries left. Otherwise if we do things like "if (!ring_empty()) send_page(ring.pop())" it can happen that we keep sending ring pages but we never sent the initial round of RAMs, so it will never completes.

Then the background migration doesn't need to keep iterating but only happen once, and then we know we've finished.

That will be a totally different way to describe migration. I don't know whether it'll make sense at all. There're a lot of hard questions to answer, e.g. besides MIGRATION we also have TCG/VGA bitmap consumers, and we need to know how to handle them. It's also just challenging to remove all bitmap layers because there're just a huge lot of them in QEMU..

I'm not sure whether above would help at all, but let me still put this here, maybe it'll let you think about something useful.

Quoted 126 lines

4. Use two rings to revert to the ring after falling back to the bitmap within a round.

As mentioned earlier, once the fallback to the bitmap occurs, pages that get dirty after the ring is full cannot be captured by the ring. This would necessitate using bitmaps until the final round or periodically locking and scanning the entire bitmap to synchronize with the ring. To improve this, I considered using two rings: one for enqueueing and one for dequeuing. Pages are inserted into the enqueue ring in functions like kvm_dirty_ring_mark_page and cpu_physical_memory_set_dirty_range, and the rings are swapped in migration_sync_bitmap, with pages being retrieved in ram_find_and_save_block. This way, each call to migration_sync_bitmap (or ram_state_pending_exact) determines whether to use the ring or the bitmap in subsequent rounds.

Based on this reasoning, I implemented a system that combines bitmaps and rings.

Regarding performance, my local environment might be insufficient for proper measurement, but I obtained the following results by migrating after booting the latest Linux and Buildroot with an open login shell:

Commands used:
```

# src
sudo ./qemu-system-x86_64 \
(quoted diff) +0 -8
-accel kvm,dirty-ring-size=1024 \
-m 8G \
-boot c \
-kernel ~/path/to/linux/arch/x86/boot/bzImage \
-hda ~/path/to/buildroot/output/images/rootfs.ext4 \
-append "root=/dev/sda rw console=ttyS0,115200 acpi=off" \
-nographic \
-migration dirty-logging=ring,dirty-ring-size=1024
# dst
sudo ./qemu-system-x86_64 \
(quoted diff) +0 -8
-accel kvm,dirty-ring-size=1024 \
-m 8G \
-boot c \
-kernel ~/path/to/linux/arch/x86/boot/bzImage \
-hda ~/path/to/buildroot/output/images/rootfs.ext4 \
-append "root=/dev/sda rw console=ttyS0,115200 acpi=off" \
-nographic \
-incoming tcp:0:4444
# hmp
migrate_set_parameter max-bandwidth 1250
migrate tcp:0:4444
info migrate
```

Results for each memory size, measured 5 times:

```
# ring -m 8G
total time: 418 ms
total time: 416 ms
total time: 415 ms
total time: 416 ms
total time: 416 ms

# bitmap -m 8G
total time: 434 ms
total time: 421 ms
total time: 423 ms
total time: 430 ms
total time: 429 ms

# ring -m 16G
total time: 847 ms
total time: 852 ms
total time: 850 ms
total time: 848 ms
total time: 852 ms

# bitmap -m 16G
total time: 860 ms
total time: 862 ms
total time: 858 ms
total time: 859 ms
total time: 861 ms

# ring -m 32G
total time: 1616 ms
total time: 1625 ms
total time: 1612 ms
total time: 1612 ms
total time: 1630 ms

# bitmap -m 32G
total time: 1714 ms
total time: 1724 ms
total time: 1718 ms
total time: 1714 ms
total time: 1714 ms

# ring -m 64G
total time: 3451 ms
total time: 3452 ms
total time: 3449 ms
total time: 3451 ms
total time: 3450 ms

# bitmap -m 64G
total time: 3550 ms
total time: 3553 ms
total time: 3552 ms
total time: 3550 ms
total time: 3553 ms

# ring -m 96G
total time: 5185 ms
total time: 5186 ms
total time: 5183 ms
total time: 5191 ms
total time: 5191 ms

# bitmap -m 96G
total time: 5385 ms
total time: 5388 ms
total time: 5386 ms
total time: 5392 ms
total time: 5592 ms
```
Quoted 4 lines

It is natural that the implemented ring completes migration faster for all memory sizes, given that the conditions favor the ring due to the minimal memory workload. By the way, these are total migration times, with much of the overhead attributed to page sending and other IO operations.

It's interesting to know there's already a perf difference. I wonder how that happened, though, if the bitmap is still there, and it looks like the scan will still be needed anyway.

I didn't look where a ring full event in kvm can imply halting the vcpu threads, if with that, and if the vcpu is dirtying something it can even dirty it slower, it can contributes to a less total dirty pages to move. Then maybe the total migration time is reduced because of that. But I'm not sure.

Quoted 7 lines

I plan to conduct more detailed measurements going forward, but if you have any recommendations for good measurement methods, please let me know.

Quoted 2 lines

I remember adding such option is not suggested. We may consider using either QMP to setup a migration parameter, or something else.

Quoted 6 lines

I apologize for implementing this without knowledge of QEMU's policy. I will remove this option and instead implement it using migrate_set_parameter or migrate_set_capability. Is this approach acceptable?

This is my first time contributing to QEMU, so I appreciate your guidance.

Not sure how you nail this, but IMHO this can be a very research-level projects. I'm not sure what's your goal, if it's about contributing to qemu there can be more solid problems to solve. Let me know if it is the case. At least I am not crystal clear on how to best leverage the rings yet, I guess, even though it feels like a pretty neat design.

Thanks,

Signature · 1 line
Peter Xu
#5 lore raw

PATCH 2/2 qtest/migration: Add dirty ring tests

In reply to Shota Imamura

This commit adds tests for migration using the dirty ring. To avoid confusion with KVM's dirty ring, use_dirty_ring has been changed to use_kvm_dirty_ring, and use_qemu_dirty_ring has been added.

---
 tests/qtest/migration-test.c | 78 ++++++++++++++++++++++++++++++++----
 1 file changed, 70 insertions(+), 8 deletions(-)
tests/qtest/migration-test.c +70 -8
index 0dccb4beff..a8151b9470 100644
@@ -556,7 +556,8 @@ typedef struct {
     /* only launch the target process */
     bool only_target;
     /* Use dirty ring if true; dirty logging otherwise */
-    bool use_dirty_ring;
+    bool use_kvm_dirty_ring;
+    bool use_qemu_dirty_ring;
     const char *opts_source;
     const char *opts_target;
     /* suspend the src before migrating to dest. */
@@ -675,6 +676,7 @@ static int test_migrate_start(QTestState **from, QTestState **to,
     g_autofree char *shmem_opts = NULL;
     g_autofree char *shmem_path = NULL;
     const char *kvm_opts = NULL;
+    const char *migration_ops = NULL;
     const char *arch = qtest_get_arch();
     const char *memory_size;
     const char *machine_alias, *machine_opts = "";
@@ -754,10 +756,16 @@ static int test_migrate_start(QTestState **from, QTestState **to,
             memory_size, shmem_path);
     }
 
-    if (args->use_dirty_ring) {
+    if (args->use_kvm_dirty_ring) {
         kvm_opts = ",dirty-ring-size=4096";
     }
 
+    if (args->use_qemu_dirty_ring) {
+        migration_ops = "dirty-logging=ring,dirty-ring-size=32768";
+    } else {
+        migration_ops = "dirty-logging=bitmap";
+    }
+
     if (!qtest_has_machine(machine_alias)) {
         g_autofree char *msg = g_strdup_printf("machine %s not supported", machine_alias);
         g_test_skip(msg);
@@ -774,10 +782,12 @@ static int test_migrate_start(QTestState **from, QTestState **to,
                                  "-name source,debug-threads=on "
                                  "-m %s "
                                  "-serial file:%s/src_serial "
+                                 "-migration %s "
                                  "%s %s %s %s %s",
                                  kvm_opts ? kvm_opts : "",
                                  machine, machine_opts,
                                  memory_size, tmpfs,
+                                 migration_ops,
                                  arch_opts ? arch_opts : "",
                                  arch_source ? arch_source : "",
                                  shmem_opts ? shmem_opts : "",
@@ -1796,12 +1806,27 @@ static void test_precopy_unix_suspend_notlive(void)
     test_precopy_common(&args);
 }
 
-static void test_precopy_unix_dirty_ring(void)
+static void test_precopy_unix_qemu_dirty_ring(void)
 {
     g_autofree char *uri = g_strdup_printf("unix:%s/migsocket", tmpfs);
     MigrateCommon args = {
         .start = {
-            .use_dirty_ring = true,
+            .use_qemu_dirty_ring = true,
+        },
+        .listen_uri = uri,
+        .connect_uri = uri,
+        .live = true,
+    };
+
+    test_precopy_common(&args);
+}
+
+static void test_precopy_unix_kvm_dirty_ring(void)
+{
+    g_autofree char *uri = g_strdup_printf("unix:%s/migsocket", tmpfs);
+    MigrateCommon args = {
+        .start = {
+            .use_kvm_dirty_ring = true,
         },
         .listen_uri = uri,
         .connect_uri = uri,
@@ -1815,6 +1840,22 @@ static void test_precopy_unix_dirty_ring(void)
     test_precopy_common(&args);
 }
 
+static void test_precopy_unix_kvm_and_qemu_dirty_ring(void)
+{
+    g_autofree char *uri = g_strdup_printf("unix:%s/migsocket", tmpfs);
+    MigrateCommon args = {
+        .start = {
+            .use_kvm_dirty_ring = true,
+            .use_qemu_dirty_ring = true,
+        },
+        .listen_uri = uri,
+        .connect_uri = uri,
+        .live = true,
+    };
+
+    test_precopy_common(&args);
+}
+
 #ifdef CONFIG_GNUTLS
 static void test_precopy_unix_tls_psk(void)
 {
@@ -1942,6 +1983,21 @@ static void test_precopy_file(void)
     test_file_common(&args, true);
 }
 
+static void test_precopy_file_dirty_ring(void)
+{
+    g_autofree char *uri = g_strdup_printf("file:%s/%s", tmpfs,
+                                           FILE_TEST_FILENAME);
+    MigrateCommon args = {
+        .start = {
+            .use_qemu_dirty_ring = true,
+        },
+        .connect_uri = uri,
+        .listen_uri = "defer",
+    };
+
+    test_file_common(&args, true);
+}
+
 static void file_offset_finish_hook(QTestState *from, QTestState *to,
                                     void *opaque)
 {
@@ -3298,7 +3354,7 @@ static void test_migrate_dirty_limit(void)
     MigrateCommon args = {
         .start = {
             .hide_stderr = true,
-            .use_dirty_ring = true,
+            .use_kvm_dirty_ring = true,
         },
         .listen_uri = uri,
         .connect_uri = uri,
@@ -3342,7 +3398,7 @@ static void test_migrate_dirty_limit(void)
     args = (MigrateCommon) {
         .start = {
             .only_target = true,
-            .use_dirty_ring = true,
+            .use_kvm_dirty_ring = true,
         },
         .listen_uri = uri,
         .connect_uri = uri,
@@ -3504,10 +3560,14 @@ int main(int argc, char **argv)
 
     migration_test_add("/migration/precopy/unix/plain",
                        test_precopy_unix_plain);
+    migration_test_add("/migration/precopy/unix/dirty_ring",
+                       test_precopy_unix_qemu_dirty_ring);
     migration_test_add("/migration/precopy/unix/xbzrle",
                        test_precopy_unix_xbzrle);
     migration_test_add("/migration/precopy/file",
                        test_precopy_file);
+    migration_test_add("/migration/precopy/file/dirty_ring",
+                       test_precopy_file_dirty_ring);
     migration_test_add("/migration/precopy/file/offset",
                        test_precopy_file_offset);
     migration_test_add("/migration/precopy/file/offset/bad",
@@ -3660,8 +3720,10 @@ int main(int argc, char **argv)
 #endif /* CONFIG_GNUTLS */
 
     if (g_str_equal(arch, "x86_64") && has_kvm && kvm_dirty_ring_supported()) {
-        migration_test_add("/migration/dirty_ring",
-                           test_precopy_unix_dirty_ring);
+        migration_test_add("/migration/precopy/unix/kvm_dirty_ring",
+                           test_precopy_unix_kvm_dirty_ring);
+        migration_test_add("/migration/precopy/unix/kvm_and_qemu_dirty_ring",
+                           test_precopy_unix_kvm_and_qemu_dirty_ring);
         migration_test_add("/migration/vcpu_dirty_limit",
                            test_vcpu_dirty_limit);
     }
Signature · 1 line
2.34.1