Obtain dirty bitmap via VM logging
v5 cover Obtain dirty bitmap via VM logging
Hi,
The following patches introduce the support of the dirty bitmap logging and dumping to a specified file. This patch series addresses the previous issues raised by David and Eric.
v4 --> v5
* Removed the runstates code (Eric's and David's advice).
* Now, using global variable to resolve the issue between migration and bitmap dump (David's advice).
* Merged the hmp and qmp interface's patch (Eric's advice).
* Modified the code to support the dynamic change in last_ram_offset (David's advice).
* Formatted the strings in the python code i.e. length followed by text (David's advice).
* Added some code to generate black and white figure based on the bitmap.
* Modified the shared function to be more generic (David's advice).
v3 --> v4
* Added new qmp interface for information extraction from the bitmap process
v2 --> v3
* Reformatted the code and removed some unnecessary parts.
* Printing block info along with length and offset.
* Changed the functions that were directly using RUN_STATE_RUNNING as state.
v1 --> v2:
* Added two new run states to avoid simultaneous execution of both migration and bitmap dump process.
* Removed FILE pointer usage.
* Dumping the data only in machine-readable format.
* Tried to rectify mistakes of the previous version.
Sanidhya Kashyap (6):
generic function between migration and bitmap dump
BitmapLog: bitmap dump code
BitmapLog: get the information about the parameters
BitmapLog: cancel mechanism for an already running dump bitmap process
BitmapLog: set the period of the dump bitmap process
BitmapLog: python script for extracting bitmap from a binary file
arch_init.c | 21 ++- hmp-commands.hx | 47 ++++++ hmp.c | 54 ++++++ hmp.h | 4 + include/exec/cpu-all.h | 5 +- include/exec/ram_addr.h | 4 + include/sysemu/sysemu.h | 5 + migration.c | 12 ++ monitor.c | 7 + qapi-schema.json | 84 +++++++++ qmp-commands.hx | 103 +++++++++++ savevm.c | 423 ++++++++++++++++++++++++++++++++++++++++++++++ scripts/extract-bitmap.py | 144 ++++++++++++++++ vl.c | 24 +++ 14 files changed, 928 insertions(+), 9 deletions(-) create mode 100755 scripts/extract-bitmap.py
Signature · 1 line
1.9.3
PATCH v5 1/6 generic function between migration and bitmap dump
I have modified the functions to be more generic i.e. I have used the counter variable which stores the required value. If the value of counter is 0, it means bitmap dump process whereas the value of 1 means migration.
--- arch_init.c | 21 +++++++++++++-------- include/exec/ram_addr.h | 4 ++++ 2 files changed, 17 insertions(+), 8 deletions(-)
arch_init.c +13 -8
@@ -434,20 +434,22 @@ ram_addr_t migration_bitmap_find_and_reset_dirty(MemoryRegion *mr,return (next - base) <<TARGET_PAGE_BITS ; } -static inline bool migration_bitmap_set_dirty (ram_addr_t addr) +static inline bool qemu_bitmap_set_dirty (ram_addr_t addr,unsigned long *bitmap, +uint64_t *counter) {bool ret;int nr = addr >>TARGET_PAGE_BITS ; - ret =test_and_set_bit (nr, migration_bitmap); + ret =test_and_set_bit (nr, bitmap); -if (!ret) { +if (!ret && *counter) { migration_dirty_pages++; }return ret; } -static void migration_bitmap_sync_range (ram_addr_t start,ram_addr_t length) +void qemu_bitmap_sync_range (ram_addr_t start,ram_addr_t length, +unsigned long *bitmap,uint64_t *counter) {ram_addr_t addr;unsigned long page =BIT_WORD (start >>TARGET_PAGE_BITS );
@@ -461,8 +463,8 @@ static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)for (k = page; k < page + nr; k++) {if (src[k]) {unsigned long new_dirty; - new_dirty = ~migration_bitmap[k]; - migration_bitmap[k] |= src[k]; + new_dirty = ~bitmap[k]; + bitmap[k] |= src[k]; new_dirty &= src[k]; migration_dirty_pages +=ctpopl (new_dirty); src[k] =0 ;
@@ -476,7 +478,7 @@ static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)cpu_physical_memory_reset_dirty (start + addr,TARGET_PAGE_SIZE ,DIRTY_MEMORY_MIGRATION ); -migration_bitmap_set_dirty (start + addr); +qemu_bitmap_set_dirty (start + addr, bitmap, counter); } } }
@@ -497,6 +499,8 @@ static void migration_bitmap_sync(void)int64_t bytes_xfer_now;static uint64_t xbzrle_cache_miss_prev;static uint64_t iterations_prev; +/* counter = 1 indicates, this will be used for migration */ +uint64_t counter =1 ; bitmap_sync_count++;
@@ -512,7 +516,8 @@ static void migration_bitmap_sync(void)address_space_sync_dirty_bitmap (&address_space_memory); QTAILQ_FOREACH(block, &ram_list.blocks, next) { -migration_bitmap_sync_range (block->mr->ram_addr, block->length); +qemu_bitmap_sync_range (block->mr->ram_addr, block->length, + migration_bitmap, &counter); }trace_migration_bitmap_sync_end (migration_dirty_pages - num_dirty_pages_init);
include/exec/ram_addr.h +4 -0
@@ -162,5 +162,9 @@ static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,void cpu_physical_memory_reset_dirty (ram_addr_t start,ram_addr_t length,unsigned client); + +void qemu_bitmap_sync_range (ram_addr_t start,ram_addr_t length, +unsigned long *bitmap,uint64_t *counter); +#endif #endif
Signature · 1 line
1.9.3
Re: [Qemu-devel] [PATCH v5 1/6] generic function between migration and bitmap dump
Quoted 3 lines · * Sanidhya Kashyap ([email protected]) wrote:
I have modified the functions to be more generic i.e. I have used the counter variable which stores the required value. If the value of counter is 0, it means bitmap dump process whereas the value of 1 means migration.
OK, but not quite what I meant:
Quoted 20 lines
Signed-off-by Sanidhya Kashyap <[email protected]>--- arch_init.c | 21 +++++++++++++-------- include/exec/ram_addr.h | 4 ++++ 2 files changed, 17 insertions(+), 8 deletions(-)arch_init.c +2
-1@@ -434,20 +434,22 @@ ram_addr_t migration_bitmap_find_and_reset_dirty(MemoryRegion *mr,return (next - base) <<TARGET_PAGE_BITS ; } -static inline bool migration_bitmap_set_dirty (ram_addr_t addr) +static inline bool qemu_bitmap_set_dirty (ram_addr_t addr,unsigned long *bitmap, +uint64_t *counter) {bool ret;int nr = addr >>TARGET_PAGE_BITS ;
arch_init.c ram_addr_t migration_bitmap_find_and_reset_dirty(MemoryRegion *mr, · line 449
Quoted 10 lines
arch_init.c +2
-2- ret =test_and_set_bit (nr, migration_bitmap); + ret =test_and_set_bit (nr, bitmap); -if (!ret) { +if (!ret && *counter) { migration_dirty_pages++; }return ret; }
If you made this:
if (!ret && *counter) {
*counter++;
}Quoted 36 lines
arch_init.c +7
-4-static void migration_bitmap_sync_range (ram_addr_t start,ram_addr_t length) +void qemu_bitmap_sync_range (ram_addr_t start,ram_addr_t length, +unsigned long *bitmap,uint64_t *counter) {ram_addr_t addr;unsigned long page =BIT_WORD (start >>TARGET_PAGE_BITS );@@ -461,8 +463,8 @@ static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)for (k = page; k < page + nr; k++) {if (src[k]) {unsigned long new_dirty; - new_dirty = ~migration_bitmap[k]; - migration_bitmap[k] |= src[k]; + new_dirty = ~bitmap[k]; + bitmap[k] |= src[k]; new_dirty &= src[k]; migration_dirty_pages +=ctpopl (new_dirty); src[k] =0 ;@@ -476,7 +478,7 @@ static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)cpu_physical_memory_reset_dirty (start + addr,TARGET_PAGE_SIZE ,DIRTY_MEMORY_MIGRATION ); -migration_bitmap_set_dirty (start + addr); +qemu_bitmap_set_dirty (start + addr, bitmap, counter); } } }@@ -497,6 +499,8 @@ static void migration_bitmap_sync(void)int64_t bytes_xfer_now;static uint64_t xbzrle_cache_miss_prev;static uint64_t iterations_prev; +/* counter = 1 indicates, this will be used for migration */ +uint64_t counter =1 ; bitmap_sync_count++;
arch_init.c static void migration_bitmap_sync(void) · line 520
Quoted 6 lines
arch_init.c +2
-1@@ -512,7 +516,8 @@ static void migration_bitmap_sync(void)address_space_sync_dirty_bitmap (&address_space_memory); QTAILQ_FOREACH(block, &ram_list.blocks, next) { -migration_bitmap_sync_range (block->mr->ram_addr, block->length); +qemu_bitmap_sync_range (block->mr->ram_addr, block->length, + migration_bitmap, &counter);
Then this could become:
qemu_bitmap_sync_range(block->mr->ram_addr, block->length,
migration_bitmap, &migration_dirty_pages);so you don't have the dummy 'counter' varaible.
Dave
Quoted 20 lines
arch_init.c +0
-0}trace_migration_bitmap_sync_end (migration_dirty_pages - num_dirty_pages_init);include/exec/ram_addr.h +4
-0@@ -162,5 +162,9 @@ static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,void cpu_physical_memory_reset_dirty (ram_addr_t start,ram_addr_t length,unsigned client); + +void qemu_bitmap_sync_range (ram_addr_t start,ram_addr_t length, +unsigned long *bitmap,uint64_t *counter); +#endif #endif Signature · 1 line
1.9.3
--
Dr. David Alan Gilbert / [email protected] / Manchester, UK
PATCH v5 2/6 BitmapLog: bitmap dump code
In reply to Sanidhya KashyapIn this patch, I have incorporated an enum named QemuProcess which defines what kind of process is being executed i.e. none --> no other process except the VM execution migration --> migration is being executed bitmap-dump --> bitmap dump process is undergoing
Besides this, I have tried to incorporate the dynamic change of the last_ram_offset, if it gets change. The downside is that I am holding lock for a longer period of time and I don't know whether that should be done or not. I am also doing some allocation when locked.
I am not sure whether last_ram_offset gets changed when a device is hot plugged or hot unplugged.
I have modified the variables name as:
current-iteration: for the current iteration under process
iterations: total iterations that will be done which is constant
period: the delay in each iteration.
--- hmp-commands.hx | 16 ++ hmp.c | 18 +++ hmp.h | 1 + include/exec/cpu-all.h | 5 +- include/sysemu/sysemu.h | 5 + migration.c | 12 ++ qapi-schema.json | 35 +++++ qmp-commands.hx | 34 +++++ savevm.c | 378 ++++++++++++++++++++++++++++++++++++++++++++++++ vl.c | 24 +++ 10 files changed, 527 insertions(+), 1 deletion(-)
hmp-commands.hx +16 -0
@@ -1788,6 +1788,22 @@ STEXI show available trace events and their state ETEXI + { + .name = "ldb|log_dirty_bitmap", + .args_type = "filename:s,iterations:i?,period:i?", + .params = "filename iterations period", + .help = "dumps the memory's dirty bitmap to file\n\t\t\t" + "filename: name of the file in which the bitmap will be saved\n\t\t\t" + "iterations: number of times, the memory will be logged\n\t\t\t" + "period: time difference in milliseconds between each iteration", + .mhandler.cmd = hmp_log_dirty_bitmap, + }, +STEXI +@item ldb or log_dirty_bitmap @var{filename} +@findex log_dirty_bitmap +dumps the writable working set of a VM's memory to a file +ETEXI + STEXI @end table ETEXI
hmp.c +18 -0
@@ -1318,6 +1318,24 @@ void hmp_device_del(Monitor *mon, const QDict *qdict)hmp_handle_error (mon, &err); } +void hmp_log_dirty_bitmap (Monitor *mon,const QDict *qdict) +{ +const char *filename =qdict_get_str (qdict,"filename" ); +bool has_iterations =qdict_haskey (qdict,"iterations" ); +int64_t iterations =qdict_get_try_int (qdict,"iterations" ,3 ); +bool has_period =qdict_haskey (qdict,"period" ); +int64_t period =qdict_get_try_int (qdict,"period" ,10 ); +Error *err =NULL ; + +qmp_log_dirty_bitmap (filename, has_iterations, iterations, + has_period, period, &err); +if (err) { +monitor_printf (mon,"log_dirty_bitmap: %s\n" ,error_get_pretty (err)); +error_free (err); +return ; + } +} +void hmp_dump_guest_memory (Monitor *mon,const QDict *qdict) {Error *err =NULL ;
hmp.h +1 -0
@@ -94,6 +94,7 @@ void hmp_cpu_add(Monitor *mon, const QDict *qdict);void hmp_object_add (Monitor *mon,const QDict *qdict);void hmp_object_del (Monitor *mon,const QDict *qdict);void hmp_info_memdev (Monitor *mon,const QDict *qdict); +void hmp_log_dirty_bitmap (Monitor *mon,const QDict *qdict);void object_add_completion (ReadLineState *rs,int nb_args,const char *str);void object_del_completion (ReadLineState *rs,int nb_args,const char *str);void device_add_completion (ReadLineState *rs,int nb_args,const char *str);
include/exec/cpu-all.h +4 -1
@@ -297,13 +297,16 @@ CPUArchState *cpu_copy(CPUArchState *env);/* memory API */ +/* global name which is used with both migration and bitmap dump */ +#define RAMBLOCK_NAME_LENGTH 256 + typedef struct RAMBlock { struct MemoryRegion *mr; uint8_t *host; ram_addr_t offset; ram_addr_t length; uint32_t flags; - char idstr[256]; + char idstr[RAMBLOCK_NAME_LENGTH]; /* Reads can take either the iothread or the ramlist lock. * Writes must take both locks. */
include/sysemu/sysemu.h +5 -0
@@ -227,4 +227,9 @@ extern QemuOptsList qemu_net_opts;extern QemuOptsList qemu_global_opts;extern QemuOptsList qemu_mon_opts; +/* migration vs dirty bitmap process */ +bool qemu_process_check (QemuProcess user); +void qemu_process_set (QemuProcess user); +const char *get_qemu_process_as_string (void ); +#endif
migration.c +12 -0
@@ -117,6 +117,7 @@ static void process_incoming_migration_co(void *opaque) }else {runstate_set (RUN_STATE_PAUSED ); } +qemu_process_set (QEMU_PROCESS_NONE ); }void process_incoming_migration (QEMUFile *f)
@@ -317,6 +318,7 @@ static void migrate_fd_cleanup(void *opaque) }notifier_list_notify (&migration_state_notifiers, s); +qemu_process_set (QEMU_PROCESS_NONE ); }void migrate_fd_error (MigrationState *s)
@@ -326,6 +328,7 @@ void migrate_fd_error(MigrationState *s) s->state =MIG_STATE_ERROR ;trace_migrate_set_state (MIG_STATE_ERROR );notifier_list_notify (&migration_state_notifiers, s); +qemu_process_set (QEMU_PROCESS_NONE ); }static void migrate_fd_cancel (MigrationState *s)
@@ -436,6 +439,15 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,return ; } +if (!qemu_process_check (QEMU_PROCESS_NONE ) && + !qemu_process_check (QEMU_PROCESS_MIGRATION )) { +error_setg (errp,"Migration not possible, since %s " +"is in progress.\n" ,get_qemu_process_as_string ()); +return ; + } + +qemu_process_set (QEMU_PROCESS_MIGRATION ); + s =migrate_init (¶ms);if (strstart (uri,"tcp:" , &p)) {
qapi-schema.json +35 -0
@@ -3480,3 +3480,38 @@ # Since: 2.1 ## { 'command': 'rtc-reset-reinjection' } + +## +# QemuProcess +# +# @none: no other process is being executed besides a simple VM execution. +# +# @migration: migration process is going on. +# +# @bitmap-dump: bitmap dump process is being executed. +# +# Since 2.2 +## +{ 'enum': 'QemuProcess', + 'data': [ 'none', 'migration', 'bitmap-dump' ] } + +## +# @log-dirty-bitmap +# +# This command will dump the dirty bitmap to a file by logging the +# memory for a specified number of times with a defined time difference +# +# @filename: name of the file in which the bitmap will be saved. +# +# @iterations: number of times the memory will be logged (optional). The +# and max values are 3 and 100000 respectively. +# +# @period: time difference in milliseconds between each iteration (optional). +# The min and max values are 10 and 100000 respectively. +# +# Since 2.2 +## +{ 'command' : 'log-dirty-bitmap', + 'data' : { 'filename' : 'str', + '*iterations' : 'int', + '*period' : 'int' } }
qmp-commands.hx +34 -0
@@ -3753,5 +3753,39 @@ Example: -> { "execute": "rtc-reset-reinjection" } <- { "return": {} } +EQMP + + { + .name = "log-dirty-bitmap", + .args_type = "filename:s,iterations:i?,period:i?", + .mhandler.cmd_new = qmp_marshal_input_log_dirty_bitmap, + }, + +SQMP +log-dirty-bitmap +---------------- + +start logging the memory of the VM for writable working set + +Arguments: + +- "filename": name of the file, in which the bitmap will be saved. + +- "iterations": number of times, the memory will be logged (optional). + The min and max values are 3 and 100000 respectively. + +- "period": time difference in milliseconds between each iteration (optional). + The min and max values are 10 and 100000 respectively. + +Examples: +-> { "execute": "log-dirty-bitmap", + "arguments": { + "filename": "/tmp/fileXXX", + "iterations": 3, + "period": 10 } } + +<- { "return": {} } +Note: The iterations, and period parameters are optional. iterations default +value is 3 while that of period is 10. EQMP
savevm.c +378 -0
390 lines not on this page · raw message
vl.c +24 -0
@@ -205,6 +205,8 @@ bool qemu_uuid_set;static QEMUBootSetHandler *boot_set_handler;static void *boot_set_opaque; +int dirty_bitmap_user; +static NotifierList exit_notifiers =NOTIFIER_LIST_INITIALIZER (exit_notifiers);
@@ -751,6 +753,27 @@ void vm_start(void)qapi_event_send_resume (&error_abort); } +/* +* A global variable to decide which process will only +* execute migration or bitmap dump +*/ + +static QemuProcess dbu =QEMU_PROCESS_NONE ; + +bool qemu_process_check (QemuProcess user) +{ +return user == dbu; +} + +void qemu_process_set (QemuProcess user) +{ + dbu = user; +} + +const char *get_qemu_process_as_string (void ) +{ +return QemuProcess_lookup[dbu]; +}/***********************************************************/ /* real time host monotonic timer */
@@ -4518,6 +4541,7 @@ int main(int argc, char **argv, char **envp) }if (incoming) { +qemu_process_set (QEMU_PROCESS_MIGRATION );Error *local_err =NULL ;qemu_start_incoming_migration (incoming, &local_err);if (local_err) {
Signature · 1 line
1.9.3
Re: [Qemu-devel] [PATCH v5 2/6] BitmapLog: bitmap dump code
Quoted 5 lines · * Sanidhya Kashyap ([email protected]) wrote:
In this patch, I have incorporated an enum named QemuProcess which defines what kind of process is being executed i.e. none --> no other process except the VM execution migration --> migration is being executed bitmap-dump --> bitmap dump process is undergoing
OK, so that's the right idea; I suspect 'process' is probably the wrong word - and we use it to mean too many things already. Why don't you stick with the 'dbu':
instead of:
static QemuProcess dbu = QEMU_PROCESS_NONE;
I'd use:
static QemuDirtyBitmapUser dbu = QEMU_DBU_NONE;
doesn't look too bad to me, and is more explicit about what it's for.
Quoted 160 lines
Besides this, I have tried to incorporate the dynamic change of the
last_ram_offset, if it gets change. The downside is that I am holding lock for a longer period of time and I don't know whether that should be done or not. I am also doing some allocation when locked.
I am not sure whetherlast_ram_offsetgets changed when a device is hot plugged or hot unplugged.I have modified the variables name as:
current-iteration: for the current iteration under process
iterations: total iterations that will be done which is constant
period: the delay in each iteration.Signed-off-by Sanidhya Kashyap <[email protected]>--- hmp-commands.hx | 16 ++ hmp.c | 18 +++ hmp.h | 1 + include/exec/cpu-all.h | 5 +- include/sysemu/sysemu.h | 5 + migration.c | 12 ++ qapi-schema.json | 35 +++++ qmp-commands.hx | 34 +++++ savevm.c | 378 ++++++++++++++++++++++++++++++++++++++++++++++++ vl.c | 24 +++ 10 files changed, 527 insertions(+), 1 deletion(-)hmp-commands.hx +16
-0@@ -1788,6 +1788,22 @@ STEXI show available trace events and their state ETEXI + { + .name = "ldb|log_dirty_bitmap", + .args_type = "filename:s,iterations:i?,period:i?", + .params = "filename iterations period", + .help = "dumps the memory's dirty bitmap to file\n\t\t\t" + "filename: name of the file in which the bitmap will be saved\n\t\t\t" + "iterations: number of times, the memory will be logged\n\t\t\t" + "period: time difference in milliseconds between each iteration", + .mhandler.cmd = hmp_log_dirty_bitmap, + }, +STEXI +@item ldb or log_dirty_bitmap @var{filename} +@findex log_dirty_bitmap +dumps the writable working set of a VM's memory to a file +ETEXI + STEXI @end table ETEXIhmp.c +18
-0@@ -1318,6 +1318,24 @@ void hmp_device_del(Monitor *mon, const QDict *qdict)hmp_handle_error (mon, &err); } +void hmp_log_dirty_bitmap (Monitor *mon,const QDict *qdict) +{ +const char *filename =qdict_get_str (qdict,"filename" ); +bool has_iterations =qdict_haskey (qdict,"iterations" ); +int64_t iterations =qdict_get_try_int (qdict,"iterations" ,3 ); +bool has_period =qdict_haskey (qdict,"period" ); +int64_t period =qdict_get_try_int (qdict,"period" ,10 ); +Error *err =NULL ; + +qmp_log_dirty_bitmap (filename, has_iterations, iterations, + has_period, period, &err); +if (err) { +monitor_printf (mon,"log_dirty_bitmap: %s\n" ,error_get_pretty (err)); +error_free (err); +return ; + } +} +void hmp_dump_guest_memory (Monitor *mon,const QDict *qdict) {Error *err =NULL ;hmp.h +1
-0@@ -94,6 +94,7 @@ void hmp_cpu_add(Monitor *mon, const QDict *qdict);void hmp_object_add (Monitor *mon,const QDict *qdict);void hmp_object_del (Monitor *mon,const QDict *qdict);void hmp_info_memdev (Monitor *mon,const QDict *qdict); +void hmp_log_dirty_bitmap (Monitor *mon,const QDict *qdict);void object_add_completion (ReadLineState *rs,int nb_args,const char *str);void object_del_completion (ReadLineState *rs,int nb_args,const char *str);void device_add_completion (ReadLineState *rs,int nb_args,const char *str);include/exec/cpu-all.h +4
-1@@ -297,13 +297,16 @@ CPUArchState *cpu_copy(CPUArchState *env);/* memory API */ +/* global name which is used with both migration and bitmap dump */ +#define RAMBLOCK_NAME_LENGTH 256 + typedef struct RAMBlock { struct MemoryRegion *mr; uint8_t *host; ram_addr_t offset; ram_addr_t length; uint32_t flags; - char idstr[256]; + char idstr[RAMBLOCK_NAME_LENGTH]; /* Reads can take either the iothread or the ramlist lock. * Writes must take both locks. */include/sysemu/sysemu.h +5
-0@@ -227,4 +227,9 @@ extern QemuOptsList qemu_net_opts;extern QemuOptsList qemu_global_opts;extern QemuOptsList qemu_mon_opts; +/* migration vs dirty bitmap process */ +bool qemu_process_check (QemuProcess user); +void qemu_process_set (QemuProcess user); +const char *get_qemu_process_as_string (void ); +#endif migration.c +3
-0@@ -117,6 +117,7 @@ static void process_incoming_migration_co(void *opaque) }else {runstate_set (RUN_STATE_PAUSED ); } +qemu_process_set (QEMU_PROCESS_NONE ); }void process_incoming_migration (QEMUFile *f)@@ -317,6 +318,7 @@ static void migrate_fd_cleanup(void *opaque) }notifier_list_notify (&migration_state_notifiers, s); +qemu_process_set (QEMU_PROCESS_NONE ); }void migrate_fd_error (MigrationState *s)@@ -326,6 +328,7 @@ void migrate_fd_error(MigrationState *s) s->state =MIG_STATE_ERROR ;trace_migrate_set_state (MIG_STATE_ERROR );notifier_list_notify (&migration_state_notifiers, s); +qemu_process_set (QEMU_PROCESS_NONE ); }static void migrate_fd_cancel (MigrationState *s)
migration.c void qmp_migrate(const char *uri, bool has_blk, bool blk, · line 445
Quoted 7 lines
migration.c +4
-0@@ -436,6 +439,15 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,return ; } +if (!qemu_process_check (QEMU_PROCESS_NONE ) && + !qemu_process_check (QEMU_PROCESS_MIGRATION )) { +error_setg (errp,"Migration not possible, since %s " +"is in progress.\n" ,get_qemu_process_as_string ());
Don't need that '\n'
Quoted 543 lines
migration.c +5
-0+return ; + } + +qemu_process_set (QEMU_PROCESS_MIGRATION ); + s =migrate_init (¶ms);if (strstart (uri,"tcp:" , &p)) {qapi-schema.json +35
-0@@ -3480,3 +3480,38 @@ # Since: 2.1 ## { 'command': 'rtc-reset-reinjection' } + +## +# QemuProcess +# +# @none: no other process is being executed besides a simple VM execution. +# +# @migration: migration process is going on. +# +# @bitmap-dump: bitmap dump process is being executed. +# +# Since 2.2 +## +{ 'enum': 'QemuProcess', + 'data': [ 'none', 'migration', 'bitmap-dump' ] } + +## +# @log-dirty-bitmap +# +# This command will dump the dirty bitmap to a file by logging the +# memory for a specified number of times with a defined time difference +# +# @filename: name of the file in which the bitmap will be saved. +# +# @iterations: number of times the memory will be logged (optional). The +# and max values are 3 and 100000 respectively. +# +# @period: time difference in milliseconds between each iteration (optional). +# The min and max values are 10 and 100000 respectively. +# +# Since 2.2 +## +{ 'command' : 'log-dirty-bitmap', + 'data' : { 'filename' : 'str', + '*iterations' : 'int', + '*period' : 'int' } }qmp-commands.hx +34
-0@@ -3753,5 +3753,39 @@ Example: -> { "execute": "rtc-reset-reinjection" } <- { "return": {} } +EQMP + + { + .name = "log-dirty-bitmap", + .args_type = "filename:s,iterations:i?,period:i?", + .mhandler.cmd_new = qmp_marshal_input_log_dirty_bitmap, + }, + +SQMP +log-dirty-bitmap +---------------- + +start logging the memory of the VM for writable working set + +Arguments: + +- "filename": name of the file, in which the bitmap will be saved. + +- "iterations": number of times, the memory will be logged (optional). + The min and max values are 3 and 100000 respectively. + +- "period": time difference in milliseconds between each iteration (optional). + The min and max values are 10 and 100000 respectively. + +Examples: +-> { "execute": "log-dirty-bitmap", + "arguments": { + "filename": "/tmp/fileXXX", + "iterations": 3, + "period": 10 } } + +<- { "return": {} } +Note: The iterations, and period parameters are optional. iterations default +value is 3 while that of period is 10. EQMPsavevm.c +378
-0@@ -42,6 +42,9 @@#include "qemu/iov.h" #include "block/snapshot.h" #include "block/qapi.h" +#include "exec/address-spaces.h" +#include "exec/ram_addr.h" +#include "qemu/bitmap.h" #ifndef ETH_P_RARP @@ -1137,6 +1140,381 @@ void do_savevm(Monitor *mon, const QDict *qdict) } } +/* +* Adding the functionality of continuous logging of the +* dirty bitmap which is almost similar to the migration +* thread +*/ + +enum { +LOG_BITMAP_STATE_ERROR =-1 , +LOG_BITMAP_STATE_NONE , +LOG_BITMAP_STATE_ACTIVE , +LOG_BITMAP_STATE_CANCELING , +LOG_BITMAP_STATE_COMPLETED +}; + +typedef struct BitmapLogState BitmapLogState ; +static int64_t MIN_ITERATION_VALUE =3 ; +static int64_t MIN_PERIOD_VALUE =10 ; +static int64_t MAX_ITERATION_VALUE =100000 ; +static int64_t MAX_PERIOD_VALUE =100000 ; + +struct BitmapLogState { +int state; +int fd; +int64_t current_period; +int64_t current_iteration; +int64_t iterations; +unsigned long *log_bitmap_array; +QemuThread thread; +}; + +/* +* helper functions +*/ + +static inline void log_bitmap_lock (void ) +{ +qemu_mutex_lock_iothread (); +qemu_mutex_lock_ramlist (); +} + +static inline void log_bitmap_unlock (void ) +{ +qemu_mutex_unlock_ramlist (); +qemu_mutex_unlock_iothread (); +} + +static inline void log_bitmap_set_dirty (ram_addr_t addr, +unsigned long *log_bitmap_array) +{ +long nr = addr >>TARGET_PAGE_BITS ; +set_bit (nr, log_bitmap_array); +} + +static bool log_bitmap_set_status (BitmapLogState *b, +int old_state, +int new_state) +{ +return atomic_cmpxchg (&b->state, old_state, new_state); +} + +/* +* inspired from migration mechanism +*/ + +static BitmapLogState *log_bitmap_get_current_state (void ) +{ +static BitmapLogState current_bitmaplogstate = { + .state =LOG_BITMAP_STATE_NONE , + .log_bitmap_array =NULL , + }; + +return ¤t_bitmaplogstate; +} + +/* +* syncing the log_bitmap with the ram_list dirty bitmap +*/ + +static void log_bitmap_dirty_bitmap_sync (unsigned long *log_bitmap_array) +{ +RAMBlock *block; +uint64_t counter =0 ;/* 0 means log bitmap */ +address_space_sync_dirty_bitmap (&address_space_memory); + QTAILQ_FOREACH(block, &ram_list.blocks, next) { +qemu_bitmap_sync_range (block->mr->ram_addr, block->length, + log_bitmap_array, &counter); + } +} + +static inline bool value_in_range (int64_t value,int64_t min_value, +int64_t max_value,const char *str, +Error **errp) +{ +if (value < min_value) { +error_setg (errp,"%s's value must be greater than %ld" , + str, min_value); +return false; + } +if (value > max_value) { +error_setg (errp,"%s's value must be less than %ld" , + str, max_value); +return false; + } +return true; +} + +static inline void log_bitmap_close (BitmapLogState *b) +{ +log_bitmap_lock (); +memory_global_dirty_log_stop (); +log_bitmap_unlock (); + +g_free (b->log_bitmap_array); + b->log_bitmap_array =NULL ; +qemu_close (b->fd); + b->fd =-1 ; +} + +static bool log_bitmap_ram_block_info_dump (int fd,int64_t ram_bitmap_pages, +bool dump_blocks_info) +{ +int block_count =0 ; +int block_name_length; +RAMBlock *block; +int ret; + +if (qemu_write_full (fd, &ram_bitmap_pages,sizeof (int64_t )) <0 ) { +return true; + } + +if (dump_blocks_info) { + + QTAILQ_FOREACH(block, &ram_list.blocks, next) { + block_count++; + } + + ret =qemu_write_full (fd, &block_count,sizeof (int )); +if (ret <sizeof (int )) { +return true; + } + + QTAILQ_FOREACH(block, &ram_list.blocks, next) { + block_name_length =strlen (block->idstr) +1 ; + ret =qemu_write_full (fd, &block_name_length,sizeof (int )); +if (ret <sizeof (int )) { +return true; + } + + ret =qemu_write_full (fd, &(block->idstr),sizeof (char ) * + block_name_length); +if (ret <sizeof (char ) * block_name_length) { +return true; + } + + ret =qemu_write_full (fd, &(block->offset),sizeof (ram_addr_t)); +if (ret <sizeof (ram_addr_t)) { +return true; + } + + ret =qemu_write_full (fd, &(block->length),sizeof (ram_addr_t)); +if (ret <sizeof (ram_addr_t)) { +return true; + } + } + } +return false; +} + +static void log_bitmap_update_status (BitmapLogState *b) +{ +int s = b->state; +switch (s) { +case LOG_BITMAP_STATE_ACTIVE : +case LOG_BITMAP_STATE_CANCELING : +case LOG_BITMAP_STATE_ERROR : +log_bitmap_set_status (b, s,LOG_BITMAP_STATE_COMPLETED ); + } +return ; +} + +static void *bitmap_logging_thread (void *opaque) +{ +/* +* setup basic structures +*/ + +BitmapLogState *b = opaque; +int fd = b->fd; +int64_t current_ram_bitmap_pages, prev_ram_bitmap_pages; +size_t bitmap_size =0 ; +unsigned long *temp_log_bitmap_array =NULL ; +char marker ='M' ; +int ret; + + b->current_iteration =1 ; +log_bitmap_set_status (b,LOG_BITMAP_STATE_NONE , +LOG_BITMAP_STATE_ACTIVE ); + + current_ram_bitmap_pages =0 ; + prev_ram_bitmap_pages =1 ; + +/* +* start the logging period +*/ + +/* +* need lock for getting the information about the ram pages. +* This does not change on acquiring the lock +*/ +log_bitmap_lock (); + current_ram_bitmap_pages =last_ram_offset () >>TARGET_PAGE_BITS ; + bitmap_size =BITS_TO_LONGS (current_ram_bitmap_pages) * +sizeof (unsigned long ); + b->log_bitmap_array =bitmap_new (current_ram_bitmap_pages); +if (b->log_bitmap_array ==NULL ) { + b->state =LOG_BITMAP_STATE_ERROR ; +log_bitmap_unlock (); + goto log_thread_end; + } + +memory_global_dirty_log_start (); +log_bitmap_dirty_bitmap_sync (b->log_bitmap_array); +log_bitmap_unlock (); + +/* +* sync the dirty bitmap along with saving it +* using the QEMUFile pointer. +*/ +while (b->current_iteration <= b->iterations) { +if (!runstate_is_running () || + b->state !=LOG_BITMAP_STATE_ACTIVE ) { + goto log_thread_end; + } + +/* +* Need to calculate the ram pages again as there is a +* possibility of the change in the memory +*/ +log_bitmap_lock (); + current_ram_bitmap_pages =last_ram_offset () >>TARGET_PAGE_BITS ; +if (current_ram_bitmap_pages != prev_ram_bitmap_pages) { + temp_log_bitmap_array =bitmap_new (current_ram_bitmap_pages); +if (temp_log_bitmap_array ==NULL ) { + b->state =LOG_BITMAP_STATE_ERROR ; +log_bitmap_unlock (); + goto log_thread_end; + } +log_bitmap_dirty_bitmap_sync (temp_log_bitmap_array); + }else { +log_bitmap_dirty_bitmap_sync (b->log_bitmap_array); + } +log_bitmap_unlock (); + +if (current_ram_bitmap_pages != prev_ram_bitmap_pages) { + prev_ram_bitmap_pages = current_ram_bitmap_pages; + bitmap_size =BITS_TO_LONGS (current_ram_bitmap_pages) * +sizeof (unsigned long ); +if (b->log_bitmap_array) { +g_free (b->log_bitmap_array); + } + b->log_bitmap_array = temp_log_bitmap_array; + temp_log_bitmap_array =NULL ; +if (log_bitmap_ram_block_info_dump (fd, current_ram_bitmap_pages, + true)) { + b->state =LOG_BITMAP_STATE_ERROR ; + goto log_thread_end; + } + }else { +if (log_bitmap_ram_block_info_dump (fd, current_ram_bitmap_pages, + false)) { + b->state =LOG_BITMAP_STATE_ERROR ; + goto log_thread_end; + } + } + + ret =qemu_write_full (fd, b->log_bitmap_array, bitmap_size); +if (ret < bitmap_size) { + b->state =LOG_BITMAP_STATE_ERROR ; + goto log_thread_end; + } + + ret =qemu_write_full (fd, &marker,sizeof (char )); +if (ret <sizeof (char )) { + b->state =LOG_BITMAP_STATE_ERROR ; + goto log_thread_end; + } +g_usleep (b->current_period *1000 ); + b->current_iteration++; +bitmap_zero (b->log_bitmap_array, current_ram_bitmap_pages); + } + +/* +* stop the logging period. +*/ + log_thread_end: +log_bitmap_close (b); +log_bitmap_update_status (b); +qemu_process_set (QEMU_PROCESS_NONE ); +return NULL ; +} + +void qmp_log_dirty_bitmap (const char *filename,bool has_iterations, +int64_t iterations,bool has_period, +int64_t period,Error **errp) +{ +int fd =-1 ; +BitmapLogState *b =log_bitmap_get_current_state (); +Error *local_err =NULL ; + +if (!runstate_is_running ()) { +error_setg (errp,"Guest is not in a running state" ); +return ; + } + +if (!qemu_process_check (QEMU_PROCESS_NONE ) && + !qemu_process_check (QEMU_PROCESS_BITMAP_DUMP )) { +error_setg (errp,"Dirty bitmap dumping not possible, since %s " +"is in progress.\n" ,get_qemu_process_as_string ()); +return ; + } + +qemu_process_set (QEMU_PROCESS_BITMAP_DUMP ); + +if (b->state ==LOG_BITMAP_STATE_ACTIVE || + b->state ==LOG_BITMAP_STATE_CANCELING ) { +error_setg (errp,"dirty bitmap dump in progress" ); +return ; + } + + b->state =LOG_BITMAP_STATE_NONE ; + +/* +* checking the iteration range +*/ +if (!has_iterations) { + b->iterations =MIN_ITERATION_VALUE ; + }else if (!value_in_range (iterations,MIN_ITERATION_VALUE , +MAX_ITERATION_VALUE ,"iterations" , &local_err)) { +if (local_err) { +error_propagate (errp, local_err); + } +return ; + }else { + b->iterations = iterations; + } + +/* +* checking the period range +*/ +if (!has_period) { + b->current_period =MIN_PERIOD_VALUE ; + }else if (!value_in_range (period,MIN_PERIOD_VALUE , +MAX_PERIOD_VALUE ,"period" , &local_err)) { +if (local_err) { +error_propagate (errp, local_err); + } +return ; + }else { + b->current_period = period; + } + + fd =qemu_open (filename,O_WRONLY |O_CREAT |O_TRUNC |O_BINARY ,S_IRUSR ); +if (fd <0 ) { +error_setg_file_open (errp, errno, filename); +return ; + } + + b->fd = fd; +qemu_thread_create (&b->thread,"dirty-bitmap-dump" , + bitmap_logging_thread, b, +QEMU_THREAD_JOINABLE ); + +return ; +} +void qmp_xen_save_devices_state (const char *filename,Error **errp) {QEMUFile *f;vl.c +24
-0@@ -205,6 +205,8 @@ bool qemu_uuid_set;static QEMUBootSetHandler *boot_set_handler;static void *boot_set_opaque; +int dirty_bitmap_user; +static NotifierList exit_notifiers =NOTIFIER_LIST_INITIALIZER (exit_notifiers);@@ -751,6 +753,27 @@ void vm_start(void)qapi_event_send_resume (&error_abort); } +/* +* A global variable to decide which process will only +* execute migration or bitmap dump +*/ + +static QemuProcess dbu =QEMU_PROCESS_NONE ; + +bool qemu_process_check (QemuProcess user) +{ +return user == dbu; +} + +void qemu_process_set (QemuProcess user) +{ + dbu = user; +} + +const char *get_qemu_process_as_string (void ) +{ +return QemuProcess_lookup[dbu]; +}/***********************************************************/ /* real time host monotonic timer */ @@ -4518,6 +4541,7 @@ int main(int argc, char **argv, char **envp) }if (incoming) { +qemu_process_set (QEMU_PROCESS_MIGRATION );Error *local_err =NULL ;qemu_start_incoming_migration (incoming, &local_err);if (local_err) {Signature · 1 line
1.9.3
--
Dr. David Alan Gilbert / [email protected] / Manchester, UK
Re: [Qemu-devel] [PATCH v5 2/6] BitmapLog: bitmap dump code
In reply to Sanidhya KashyapQuoted 36 lines · Sanidhya Kashyap
In this patch, I have incorporated an enum named QemuProcess which defines what kind of process is being executed i.e. none --> no other process except the VM execution migration --> migration is being executed bitmap-dump --> bitmap dump process is undergoing
Besides this, I have tried to incorporate the dynamic change of the
last_ram_offset, if it gets change. The downside is that I am holding lock for a longer period of time and I don't know whether that should be done or not. I am also doing some allocation when locked.
I am not sure whetherlast_ram_offsetgets changed when a device is hot plugged or hot unplugged.I have modified the variables name as:
current-iteration: for the current iteration under process
iterations: total iterations that will be done which is constant
period: the delay in each iteration.Signed-off-by Sanidhya Kashyap <[email protected]>--- hmp-commands.hx | 16 ++ hmp.c | 18 +++ hmp.h | 1 + include/exec/cpu-all.h | 5 +- include/sysemu/sysemu.h | 5 + migration.c | 12 ++ qapi-schema.json | 35 +++++ qmp-commands.hx | 34 +++++ savevm.c | 378 ++++++++++++++++++++++++++++++++++++++++++++++++ vl.c | 24 +++ 10 files changed, 527 insertions(+), 1 deletion(-)+++ b/hmp-commands.hx
hmp-commands.hx STEXI · line 1792
Quoted 5 lines
hmp-commands.hx +2
-0@@ -1788,6 +1788,22 @@ STEXI show available trace events and their state ETEXI + { + .name = "ldb|log_dirty_bitmap",
The only other instances of a .name with an | are for giving a single-letter option name synonym to a long option, but ldb is not a single-letter option. I don't think you can create 'ldb' in the HMP interface. Just stick with the long name.
hmp-commands.hx STEXI · line 1797
Quoted 5 lines
hmp-commands.hx +5
-0+ .args_type = "filename:s,iterations:i?,period:i?", + .params = "filename iterations period", + .help = "dumps the memory's dirty bitmap to file\n\t\t\t" + "filename: name of the file in which the bitmap will be saved\n\t\t\t" + "iterations: number of times, the memory will be logged\n\t\t\t"
s/times,/times/
qapi-schema.json line 3485
Quoted 8 lines
qapi-schema.json +4
-0+++ b/qapi-schema.json@@ -3480,3 +3480,38 @@ # Since: 2.1 ## { 'command': 'rtc-reset-reinjection' } + +## +# QemuProcess
I agree with David's advice to name this QemuBitmapUser.
qapi-schema.json line 3507
Quoted 10 lines
qapi-schema.json +10
-0+## +# @log-dirty-bitmap +# +# This command will dump the dirty bitmap to a file by logging the +# memory for a specified number of times with a defined time difference +# +# @filename: name of the file in which the bitmap will be saved. +# +# @iterations: number of times the memory will be logged (optional). The +# and max values are 3 and 100000 respectively.
s/and/min and/
For consistency, the optional markup should look like:
@iterations: #optional number of times the memory will be logged.
Also, you should document what value is used when the option is omitted.
qapi-schema.json line 3510
Quoted 3 lines
qapi-schema.json +3
-0+# +# @period: time difference in milliseconds between each iteration (optional). +# The min and max values are 10 and 100000 respectively.
Again, wrong markup for #optional, and missing the default value.
qmp-commands.hx Example: · line 3779
Quoted 10 lines
qmp-commands.hx +10
-0+Arguments: + +- "filename": name of the file, in which the bitmap will be saved. + +- "iterations": number of times, the memory will be logged (optional). + The min and max values are 3 and 100000 respectively. + +- "period": time difference in milliseconds between each iteration (optional). + The min and max values are 10 and 100000 respectively. +
[1]
qmp-commands.hx Example: · line 3790
Quoted 11 lines
qmp-commands.hx +10
-0+Examples: +-> { "execute": "log-dirty-bitmap", + "arguments": { + "filename": "/tmp/fileXXX", + "iterations": 3, + "period": 10 } } + +<- { "return": {} } +Note: The iterations, and period parameters are optional. iterations default +value is 3 while that of period is 10.
This note is redundant, and it is not good to split information. Instead, drop the note paragraph, and mention the optional values of 3 and 10 up above at the point [1] where you mention the arguments are optional.
Signature · 2 lines
Eric Blake eblake redhat com +1-919-301-3266 Libvirt virtualization library http://libvirt.org
PATCH v5 3/6 BitmapLog: get the information about the parameters
In reply to Sanidhya KashyapNo functional change except the variable name.
--- hmp-commands.hx | 2 ++ hmp.c | 19 +++++++++++++++++++ hmp.h | 1 + monitor.c | 7 +++++++ qapi-schema.json | 28 ++++++++++++++++++++++++++++ qmp-commands.hx | 25 +++++++++++++++++++++++++ savevm.c | 17 +++++++++++++++++ 7 files changed, 99 insertions(+)
hmp-commands.hx +2 -0
@@ -1780,6 +1780,8 @@ show qdev device model list show roms @item info tpm show the TPM device +@item info log_dirty_bitmap +show the current parameters values @end table ETEXI
hmp.c +19 -0
@@ -1732,3 +1732,22 @@ void hmp_info_memdev(Monitor *mon, const QDict *qdict)monitor_printf (mon,"\n" ); } + +void hmp_info_log_dirty_bitmap (Monitor *mon,const QDict *qdict) +{ +Error *err =NULL ; +BitmapLogStateInfo *info =qmp_query_log_dirty_bitmap (&err); + +if (info) { +monitor_printf (mon,"current iteration: %ld\n" , + info->current_iteration); +monitor_printf (mon,"total iterations: %ld\n" , info->iterations); +monitor_printf (mon,"current period value: %ld\n" , info->period); + } + +if (err) { +hmp_handle_error (mon, &err); + } + +qapi_free_BitmapLogStateInfo (info); +}
hmp.h +1 -0
@@ -38,6 +38,7 @@ void hmp_info_balloon(Monitor *mon, const QDict *qdict);void hmp_info_pci (Monitor *mon,const QDict *qdict);void hmp_info_block_jobs (Monitor *mon,const QDict *qdict);void hmp_info_tpm (Monitor *mon,const QDict *qdict); +void hmp_info_log_dirty_bitmap (Monitor *mon,const QDict *qdict);void hmp_quit (Monitor *mon,const QDict *qdict);void hmp_stop (Monitor *mon,const QDict *qdict);void hmp_system_reset (Monitor *mon,const QDict *qdict);
monitor.c +7 -0
@@ -2918,6 +2918,13 @@ static mon_cmd_t info_cmds[] = { .mhandler.cmd = hmp_info_memdev, }, { + .name = "log_dirty_bitmap", + .args_type = "", + .params = "", + .help = "show the current parameters values", + .mhandler.cmd = hmp_info_log_dirty_bitmap, + }, + { .name = NULL, }, };
qapi-schema.json +28 -0
@@ -3515,3 +3515,31 @@ 'data' : { 'filename' : 'str', '*iterations' : 'int', '*period' : 'int' } } +## +# @BitmapLogStateInfo +# +# Provides information for the bitmap logging process +# +# @current-iteration: stores current iteration value +# +# @iterations: total iterations value +# +# @period: the time difference in milliseconds between each iteration +# +# Since 2.2 +## +{ 'type': 'BitmapLogStateInfo', + 'data': { 'current-iteration' : 'int', + 'iterations' : 'int', + 'period' : 'int' } } + +## +# @query-log-dirty-bitmap +# +# Get the current values of the parameters involved in bitmap logging process +# +# This command returns the BitmapLogStateInfo +# +# Since 2.2 +## +{ 'command': 'query-log-dirty-bitmap', 'returns': 'BitmapLogStateInfo' }
qmp-commands.hx +25 -0
@@ -3789,3 +3789,28 @@ Examples: Note: The iterations, and period parameters are optional. iterations default value is 3 while that of period is 10. EQMP + + { + .name = "query-log-dirty-bitmap", + .args_type = "", + .mhandler.cmd_new = qmp_marshal_input_query_log_dirty_bitmap, + }, + +SQMP +query-log-dirty-bitmap +---------------------- + +Get the parameters information + +- "current-iteration": stores current iteration value +- "iterations": total iterations value +- "period": the time difference in milliseconds between each iteration + +Example: + +-> { "execute": "query-log-dirty-bitmap" } +<- { "return": { + "current-iteration": 3 + "iterations": 10 + "period": 100 } } +EQMP
savevm.c +17 -0
@@ -1515,6 +1515,23 @@ void qmp_log_dirty_bitmap(const char *filename, bool has_iterations,return ; } +BitmapLogStateInfo *qmp_query_log_dirty_bitmap (Error **errp) +{ +BitmapLogState *b =log_bitmap_get_current_state (); +BitmapLogStateInfo *info =NULL ; + +if (b->state !=LOG_BITMAP_STATE_ACTIVE ) { +return info; + } + + info =g_malloc0 (sizeof (BitmapLogStateInfo)); + info->current_iteration = b->current_iteration; + info->iterations = b->iterations; + info->period = b->current_period; + +return info; +} +void qmp_xen_save_devices_state (const char *filename,Error **errp) {QEMUFile *f;
Signature · 1 line
1.9.3
Re: [Qemu-devel] [PATCH v5 3/6] BitmapLog: get the information about the parameters
Quoted 35 lines · * Sanidhya Kashyap ([email protected]) wrote:
No functional change except the variable name.
Signed-off-by Sanidhya Kashyap <[email protected]>--- hmp-commands.hx | 2 ++ hmp.c | 19 +++++++++++++++++++ hmp.h | 1 + monitor.c | 7 +++++++ qapi-schema.json | 28 ++++++++++++++++++++++++++++ qmp-commands.hx | 25 +++++++++++++++++++++++++ savevm.c | 17 +++++++++++++++++ 7 files changed, 99 insertions(+)hmp-commands.hx +2
-0@@ -1780,6 +1780,8 @@ show qdev device model list show roms @item info tpm show the TPM device +@item info log_dirty_bitmap +show the current parameters values @end table ETEXIhmp.c +1
-0@@ -1732,3 +1732,22 @@ void hmp_info_memdev(Monitor *mon, const QDict *qdict)monitor_printf (mon,"\n" ); } +
hmp.c void hmp_info_memdev(Monitor *mon, const QDict *qdict) · line 1745
Quoted 10 lines
hmp.c +10
-0+void hmp_info_log_dirty_bitmap (Monitor *mon,const QDict *qdict) +{ +Error *err =NULL ; +BitmapLogStateInfo *info =qmp_query_log_dirty_bitmap (&err); + +if (info) { +monitor_printf (mon,"current iteration: %ld\n" , + info->current_iteration); +monitor_printf (mon,"total iterations: %ld\n" , info->iterations); +monitor_printf (mon,"current period value: %ld\n" , info->period);
I think you need to use the PRId64 macros there rather than %ld; because long is not always 64bits (there are some other uses in hmp.c)
Quoted 138 lines
hmp.c +8
-0+ } + +if (err) { +hmp_handle_error (mon, &err); + } + +qapi_free_BitmapLogStateInfo (info); +}hmp.h +1
-0@@ -38,6 +38,7 @@ void hmp_info_balloon(Monitor *mon, const QDict *qdict);void hmp_info_pci (Monitor *mon,const QDict *qdict);void hmp_info_block_jobs (Monitor *mon,const QDict *qdict);void hmp_info_tpm (Monitor *mon,const QDict *qdict); +void hmp_info_log_dirty_bitmap (Monitor *mon,const QDict *qdict);void hmp_quit (Monitor *mon,const QDict *qdict);void hmp_stop (Monitor *mon,const QDict *qdict);void hmp_system_reset (Monitor *mon,const QDict *qdict);monitor.c +7
-0@@ -2918,6 +2918,13 @@ static mon_cmd_t info_cmds[] = { .mhandler.cmd = hmp_info_memdev, }, { + .name = "log_dirty_bitmap", + .args_type = "", + .params = "", + .help = "show the current parameters values", + .mhandler.cmd = hmp_info_log_dirty_bitmap, + }, + { .name = NULL, }, };qapi-schema.json +28
-0@@ -3515,3 +3515,31 @@ 'data' : { 'filename' : 'str', '*iterations' : 'int', '*period' : 'int' } } +## +# @BitmapLogStateInfo +# +# Provides information for the bitmap logging process +# +# @current-iteration: stores current iteration value +# +# @iterations: total iterations value +# +# @period: the time difference in milliseconds between each iteration +# +# Since 2.2 +## +{ 'type': 'BitmapLogStateInfo', + 'data': { 'current-iteration' : 'int', + 'iterations' : 'int', + 'period' : 'int' } } + +## +# @query-log-dirty-bitmap +# +# Get the current values of the parameters involved in bitmap logging process +# +# This command returns the BitmapLogStateInfo +# +# Since 2.2 +## +{ 'command': 'query-log-dirty-bitmap', 'returns': 'BitmapLogStateInfo' }qmp-commands.hx +25
-0@@ -3789,3 +3789,28 @@ Examples: Note: The iterations, and period parameters are optional. iterations default value is 3 while that of period is 10. EQMP + + { + .name = "query-log-dirty-bitmap", + .args_type = "", + .mhandler.cmd_new = qmp_marshal_input_query_log_dirty_bitmap, + }, + +SQMP +query-log-dirty-bitmap +---------------------- + +Get the parameters information + +- "current-iteration": stores current iteration value +- "iterations": total iterations value +- "period": the time difference in milliseconds between each iteration + +Example: + +-> { "execute": "query-log-dirty-bitmap" } +<- { "return": { + "current-iteration": 3 + "iterations": 10 + "period": 100 } } +EQMPsavevm.c +17
-0@@ -1515,6 +1515,23 @@ void qmp_log_dirty_bitmap(const char *filename, bool has_iterations,return ; } +BitmapLogStateInfo *qmp_query_log_dirty_bitmap (Error **errp) +{ +BitmapLogState *b =log_bitmap_get_current_state (); +BitmapLogStateInfo *info =NULL ; + +if (b->state !=LOG_BITMAP_STATE_ACTIVE ) { +return info; + } + + info =g_malloc0 (sizeof (BitmapLogStateInfo)); + info->current_iteration = b->current_iteration; + info->iterations = b->iterations; + info->period = b->current_period; + +return info; +} +void qmp_xen_save_devices_state (const char *filename,Error **errp) {QEMUFile *f;Signature · 1 line
1.9.3
--
Dr. David Alan Gilbert / [email protected] / Manchester, UK
Re: [Qemu-devel] [PATCH v5 3/6] BitmapLog: get the information about the parameters
In reply to Sanidhya KashyapQuoted 1 line · Sanidhya Kashyap
No functional change except the variable name.
This comment feels more like it is a changelog of what is different from v4. If so, it belongs...
Quoted 3 lines
Signed-off-by Sanidhya Kashyap <[email protected]>---
...here, after the --- separator. It makes no sense in isolation in qemu.git (where v1 through v4 do not appear), but is there only to aid reviewers on list (who DO see prior versions, and want to see if you took into account earlier review comments).
hmp.c void hmp_info_memdev(Monitor *mon, const QDict *qdict) · line 1743
Quoted 3 lines
hmp.c +3
-0+if (info) { +monitor_printf (mon,"current iteration: %ld\n" , + info->current_iteration);
Won't compile on 32-bit. Per patch 2/6, info->current_iteration is int64_t, but %ld might be 32-bit. Furthermore, patch 2/6 had an (arbitrary?) limit of 100,000 as the maximum iteration request, which fits in a 32-bit value to begin with, so using int64_t to hold the value is overkill.
qmp-commands.hx Examples: · line 3815
Quoted 7 lines
qmp-commands.hx +7
-0+Example: + +-> { "execute": "query-log-dirty-bitmap" } +<- { "return": { + "current-iteration": 3 + "iterations": 10 + "period": 100 } }
That's not valid JSON. You are missing two commas. It's best to paste an actual QMP result, rather than trying to write it by hand.
Signature · 2 lines
Eric Blake eblake redhat com +1-919-301-3266 Libvirt virtualization library http://libvirt.org
PATCH v5 4/6 BitmapLog: cancel mechanism for an already running dump bitmap process
In reply to Sanidhya Kashyap--- hmp-commands.hx | 14 ++++++++++++++ hmp.c | 5 +++++ hmp.h | 1 + qapi-schema.json | 9 +++++++++ qmp-commands.hx | 20 ++++++++++++++++++++ savevm.c | 14 ++++++++++++++ 6 files changed, 63 insertions(+)
hmp-commands.hx +14 -0
@@ -1806,6 +1806,20 @@ STEXI dumps the writable working set of a VM's memory to a file ETEXI + { + .name = "ldbc|log_dirty_bitmap_cancel", + .args_type = "", + .params = "", + .help = "cancel the current bitmap dump process", + .mhandler.cmd = hmp_log_dirty_bitmap_cancel, +}, + +STEXI +@item ldbc or log_dirty_bitmap_cancel +@findex log_dirty_bitmap_cancel +Cancel the current bitmap dump process +ETEXI + STEXI @end table ETEXI
hmp.c +5 -0
@@ -1336,6 +1336,11 @@ void hmp_log_dirty_bitmap(Monitor *mon, const QDict *qdict) } } +void hmp_log_dirty_bitmap_cancel (Monitor *mon,const QDict *qdict) +{ +qmp_log_dirty_bitmap_cancel (NULL ); +} +void hmp_dump_guest_memory (Monitor *mon,const QDict *qdict) {Error *err =NULL ;
hmp.h +1 -0
@@ -96,6 +96,7 @@ void hmp_object_add(Monitor *mon, const QDict *qdict);void hmp_object_del (Monitor *mon,const QDict *qdict);void hmp_info_memdev (Monitor *mon,const QDict *qdict);void hmp_log_dirty_bitmap (Monitor *mon,const QDict *qdict); +void hmp_log_dirty_bitmap_cancel (Monitor *mon,const QDict *qdict);void object_add_completion (ReadLineState *rs,int nb_args,const char *str);void object_del_completion (ReadLineState *rs,int nb_args,const char *str);void device_add_completion (ReadLineState *rs,int nb_args,const char *str);
qapi-schema.json +9 -0
@@ -3543,3 +3543,12 @@ # Since 2.2 ## { 'command': 'query-log-dirty-bitmap', 'returns': 'BitmapLogStateInfo' } + +## +# @log-dirty-bitmap-cancel +# +# cancel the dirty bitmap logging process +# +# Since 2.2 +## +{ 'command': 'log-dirty-bitmap-cancel' }
qmp-commands.hx +20 -0
@@ -3814,3 +3814,23 @@ Example: "iterations": 10 "period": 100 } } EQMP + + { + .name = "log-dirty-bitmap-cancel", + .args_type = "", + .mhandler.cmd_new = qmp_marshal_input_log_dirty_bitmap_cancel, + }, + +SQMP +log-dirty-bitmap-cancel +----------------------- + +Cancel the current bitmap dump process. + +Arguments: None. + +Example: + +-> { "execute": "log-dirty-bitmap-cancel" } +<- { "return": {} } +EQMP
savevm.c +14 -0
@@ -1532,6 +1532,20 @@ BitmapLogStateInfo *qmp_query_log_dirty_bitmap(Error **errp)return info; } +void qmp_log_dirty_bitmap_cancel (Error **errp) +{ +BitmapLogState *b =log_bitmap_get_current_state (); +int old_state; +do { + old_state = b->state; +if (old_state !=LOG_BITMAP_STATE_ACTIVE ) { +break ; + } +log_bitmap_set_status (b, old_state, +LOG_BITMAP_STATE_CANCELING ); + }while (b->state !=LOG_BITMAP_STATE_CANCELING ); +} +void qmp_xen_save_devices_state (const char *filename,Error **errp) {QEMUFile *f;
Signature · 1 line
1.9.3
Re: [Qemu-devel] [PATCH v5 4/6] BitmapLog: cancel mechanism for an already running dump bitmap process
Quoted 15 lines · Sanidhya Kashyap
Signed-off-by Sanidhya Kashyap <[email protected]>--- hmp-commands.hx | 14 ++++++++++++++ hmp.c | 5 +++++ hmp.h | 1 + qapi-schema.json | 9 +++++++++ qmp-commands.hx | 20 ++++++++++++++++++++ savevm.c | 14 ++++++++++++++ 6 files changed, 63 insertions(+)hmp-commands.hx +0
-0
hmp-commands.hx STEXI · line 1810
Quoted 5 lines
hmp-commands.hx +2
-0@@ -1806,6 +1806,20 @@ STEXI dumps the writable working set of a VM's memory to a file ETEXI + { + .name = "ldbc|log_dirty_bitmap_cancel",
As in 2/6, trying to use 'ldbc' as a short name is not going to work.
Signature · 2 lines
Eric Blake eblake redhat com +1-919-301-3266 Libvirt virtualization library http://libvirt.org
PATCH v5 5/6 BitmapLog: set the period of the dump bitmap process
In reply to Sanidhya KashyapNo functional change except the variable name -- frequency has been modified to period.
--- hmp-commands.hx | 15 +++++++++++++++ hmp.c | 12 ++++++++++++ hmp.h | 1 + qapi-schema.json | 12 ++++++++++++ qmp-commands.hx | 24 ++++++++++++++++++++++++ savevm.c | 14 ++++++++++++++ 6 files changed, 78 insertions(+)
hmp-commands.hx +15 -0
@@ -1820,6 +1820,21 @@ STEXI Cancel the current bitmap dump process ETEXI + { + .name = "ldbsf|log_dirty_bitmap_set_period", + .args_type = "period:i", + .params = "period", + .help = "set the period for bitmap dump process\n\t\t\t" + "period: the new period value to replace the existing", + .mhandler.cmd = hmp_log_dirty_bitmap_set_period, + }, + +STEXI +@item ldbsf or log_dirty_bitmap_set_period @var{period} +@findex log_dirty_bitmap_set_period +Set the period to @var{period} (int) for bitmap dump process. +ETEXI + STEXI @end table ETEXI
hmp.c +12 -0
@@ -1341,6 +1341,18 @@ void hmp_log_dirty_bitmap_cancel(Monitor *mon, const QDict *qdict)qmp_log_dirty_bitmap_cancel (NULL ); } +void hmp_log_dirty_bitmap_set_period (Monitor *mon,const QDict *qdict) +{ +int64_t period =qdict_get_int (qdict,"period" ); +Error *err =NULL ; +qmp_log_dirty_bitmap_set_period (period, &err); +if (err) { +monitor_printf (mon,"log_dirty_bitmap_set_period: %s\n" , +error_get_pretty (err)); +error_free (err); + } +} +void hmp_dump_guest_memory (Monitor *mon,const QDict *qdict) {Error *err =NULL ;
hmp.h +1 -0
@@ -97,6 +97,7 @@ void hmp_object_del(Monitor *mon, const QDict *qdict);void hmp_info_memdev (Monitor *mon,const QDict *qdict);void hmp_log_dirty_bitmap (Monitor *mon,const QDict *qdict);void hmp_log_dirty_bitmap_cancel (Monitor *mon,const QDict *qdict); +void hmp_log_dirty_bitmap_set_period (Monitor *mon,const QDict *qdict);void object_add_completion (ReadLineState *rs,int nb_args,const char *str);void object_del_completion (ReadLineState *rs,int nb_args,const char *str);void device_add_completion (ReadLineState *rs,int nb_args,const char *str);
qapi-schema.json +12 -0
@@ -3552,3 +3552,15 @@ # Since 2.2 ## { 'command': 'log-dirty-bitmap-cancel' } + +## +# @log-dirty-bitmap-set-period +# +# sets the period of the dirty bitmap logging process +# @frequency: the updated period value (in milliseconds). +# The min and max values are 10 and 100000 respectively. +# +# Since 2.2 +## +{ 'command': 'log-dirty-bitmap-set-period', + 'data': { 'period': 'int' } }
qmp-commands.hx +24 -0
@@ -3834,3 +3834,27 @@ Example: -> { "execute": "log-dirty-bitmap-cancel" } <- { "return": {} } EQMP + + { + .name = "log-dirty-bitmap-set-period", + .args_type = "period:i", + .mhandler.cmd_new = qmp_marshal_input_log_dirty_bitmap_set_period, + }, + +SQMP +log-dirty-bitmap-set-period +--------------------------- + +Update the period for the remaining iterations. + +Arguments: + +- "period": The updated period (json-int) (in milliseconds). + The min and max values are 10 and 100000 respectively. + +Example: + +-> { "execute": "log-dirty-bitmap-set-period", "arguments": { "period": 1024 } } +<- { "return": {} } + +EQMP
savevm.c +14 -0
@@ -1546,6 +1546,20 @@ void qmp_log_dirty_bitmap_cancel(Error **errp) }while (b->state !=LOG_BITMAP_STATE_CANCELING ); } +void qmp_log_dirty_bitmap_set_period (int64_t period,Error **errp) +{ +BitmapLogState *b =log_bitmap_get_current_state (); +Error *local_err =NULL ; +if (value_in_range (period,MIN_PERIOD_VALUE , +MAX_PERIOD_VALUE ,"period" , &local_err)) { +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } + b->current_period = period; +} +void qmp_xen_save_devices_state (const char *filename,Error **errp) {QEMUFile *f;
Signature · 1 line
1.9.3
Re: [Qemu-devel] [PATCH v5 5/6] BitmapLog: set the period of the dump bitmap process
Quoted 2 lines · Sanidhya Kashyap
No functional change except the variable name -- frequency has been modified to period.
This sounds like patch revision information that belongs...
Quoted 3 lines
Signed-off-by Sanidhya Kashyap <[email protected]>---
...here.
Quoted 13 lines
hmp-commands.hx | 15 +++++++++++++++ hmp.c | 12 ++++++++++++ hmp.h | 1 + qapi-schema.json | 12 ++++++++++++ qmp-commands.hx | 24 ++++++++++++++++++++++++ savevm.c | 14 ++++++++++++++ 6 files changed, 78 insertions(+)hmp-commands.hx +0
-0
hmp-commands.hx STEXI · line 1824
Quoted 5 lines
hmp-commands.hx +2
-0@@ -1820,6 +1820,21 @@ STEXI Cancel the current bitmap dump process ETEXI + { + .name = "ldbsf|log_dirty_bitmap_set_period",
As in 2/6, don't try to use a short 'ldbsf' name.
Signature · 2 lines
Eric Blake eblake redhat com +1-919-301-3266 Libvirt virtualization library http://libvirt.org
PATCH v5 6/6 BitmapLog: python script for extracting bitmap from a binary file
In reply to Sanidhya KashyapThe file not only extracts the bitmap from the file but also draws the figure if required. Currently, figure is drawn for all the bitmaps. Later, I'll make the change to draw for different blocks.
The picture is drawn by generating a matrix of 0s and 1s from the bitmap. The dimensions are calculated on the basis of total bitmap pages which is represented as sqrt(total pages) X (sqrt(total pages) + 1). The white parts indicate non dirtied region while the black - dirtied region.
The python code requires some libraries such as numpy, pylab and math to generate the images.
--- scripts/extract-bitmap.py | 144 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100755 scripts/extract-bitmap.py
scripts/extract-bitmap.py new file +144 -0
@@ -0,0 +1,144 @@ +#!/usr/bin/python +# This python script helps in extracting the dirty bitmap present +# in the file after executing the log-dirty-bitmap command either +# from the qmp or hmp interface. This file only processes binary +# file obtained via command. +# +# Copyright (C) 2014 Sanidhya Kashyap <[email protected]> +# +# Authors: +# Sanidhya Kashyap +# +# +# This work is licensed under the terms of the GNU GPL, version 2 or later. + +import struct +import argparse +from functools import partial +from math import sqrt +from numpy import array +from pylab import figure,imshow,show,gray + +long_bytes = 8 +byte_size = 8 +int_bytes = 4 +complete_bitmap_list = [] +block_list = [] + +def get_unsigned_long_integer(value): + return struct.unpack('<Q', value)[0] + +def get_long_integer(value): + return struct.unpack('<q', value)[0] + +def get_integer(value): + return struct.unpack('<i', value)[0] + +def get_char(value): + return struct.unpack('<c', value)[0] + +def get_string(value, length): + name = struct.unpack('<'+str(length)+'s', value)[0] + for i in range(len(name)): + if name[i] == '\x00': + return name[:i] + +def dec2bin(decimal): + bin_value = bin(decimal)[2:] + if len(bin_value) < long_bytes * byte_size: + add_zeroes = long_bytes * byte_size - len(bin_value) + for i in range(add_zeroes): + bin_value += "0" + return str(bin_value) + +def get_bitmap_length(ram_bitmap_pages): + bitmap_length = ram_bitmap_pages / (long_bytes * byte_size) + if ram_bitmap_pages % (long_bytes * byte_size) != 0: + bitmap_length += 1 + return bitmap_length + +def dump_ram_block_info(infile): + total_blocks = get_integer(infile.read(int_bytes)) + for i in range(total_blocks): + block_name_length = get_integer(infile.read(int_bytes)) + block_name = get_string(infile.read(block_name_length), block_name_length) + block_offset = get_unsigned_long_integer(infile.read(long_bytes)) + block_length = get_unsigned_long_integer(infile.read(long_bytes)) + block_list.append(dict(name=block_name, offset=block_offset, length=block_length)) + +def generate_images(): + r = 0 + for list in complete_bitmap_list: + all_digits = "" + for element in list: + all_digits += dec2bin(element) + l = len(all_digits) + sqrtvalue = int(sqrt(l)) + for i in range(sqrtvalue * (sqrtvalue+1)-l): + all_digits+="0" + + v = [] + l = len(all_digits) + for i in range(sqrtvalue+1): + v1 = [] + for j in range(sqrtvalue): + v1.append(int(all_digits[i*sqrtvalue+j])) + v.append(v1) + + im_array = array(v) + figure(r) + imshow(im_array, cmap=gray()) + r += 1 + show() + +def dump_bitmap(infile, draw): + marker = 'M' + count = 0 + value = ' ' + current_ram_bitmap_pages = 0 + prev_ram_bitmap_pages = 0 + while True: + if len(value) == 0 or marker != 'M': + print "issue with the dump" + return + bitmap_page_raw_value = infile.read(long_bytes) + if not bitmap_page_raw_value: + break + current_ram_bitmap_pages = get_long_integer(bitmap_page_raw_value) + if current_ram_bitmap_pages != prev_ram_bitmap_pages: + prev_ram_bitmap_pages = current_ram_bitmap_pages + dump_ram_block_info(infile) + + bitmap_length = get_bitmap_length(current_ram_bitmap_pages) + bitmap_list = [] + bitmap_raw_value = infile.read(long_bytes * bitmap_length) + if not bitmap_raw_value: + break + count+=1 + for i in range(bitmap_length): + mark = i * long_bytes + bitmap_list.append((get_unsigned_long_integer(bitmap_raw_value[mark:mark+long_bytes]))) + complete_bitmap_list.append(bitmap_list) + value = infile.read(1) + marker = get_char(value) + if draw is True: + generate_images() + else: + print complete_bitmap_list + +def main(): + extracter = argparse.ArgumentParser(description='Extract dirty bitmap from binary file.') + extracter.add_argument('infile', help='Input file to extract the bitmap') + extracter.add_argument('-d', action='store_true', dest='draw', default=False, + help='Draw a black and white image of the processed dirty bitmap') + args = extracter.parse_args() + print 'The filename is {}'.format(args.infile) + + infile = open(format(args.infile), 'rb') + + dump_bitmap(infile, args.draw); + + infile.close() + +if __name__ == '__main__': + main()
Signature · 1 line
1.9.3
Re: [Qemu-devel] [PATCH v5 6/6] BitmapLog: python script for extracting bitmap from a binary file
Quoted 11 lines · * Sanidhya Kashyap ([email protected]) wrote:
The file not only extracts the bitmap from the file but also draws the figure if required. Currently, figure is drawn for all the bitmaps. Later, I'll make the change to draw for different blocks.
The picture is drawn by generating a matrix of 0s and 1s from the bitmap. The dimensions are calculated on the basis of total bitmap pages which is represented as sqrt(total pages) X (sqrt(total pages) + 1). The white parts indicate non dirtied region while the black - dirtied region.
The python code requires some libraries such as numpy, pylab and math to generate the images.
This is interesting; I've tried this out with a Fedora boot and recorded about 240 frames; the script has problems when trying to convert that big a recording to images;
1) It loads it all into memory - so it used about 2GB of RAM
2) It opens each image as a window - so I then had to close about 240 windows
I fiddled with it a bit and created .png files, and then stiched these together with imagemagick to create an animation. It's quite interesting watch the OS boot and then see me login and start google's stress apptest which suddenly changes the whole of memory.
I suggest making it dump each image to a file in a similar way and making it free the python image data after each frame, then there shouldn't be a size limit. (I did try and upload the image, but QEMU's wiki didn't like the size)
Dave
Quoted 161 lines
Signed-off-by Sanidhya Kashyap <[email protected]>--- scripts/extract-bitmap.py | 144 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100755 scripts/extract-bitmap.pyscripts/extract-bitmap.py new file +144
-0@@ -0,0 +1,144 @@ +#!/usr/bin/python +# This python script helps in extracting the dirty bitmap present +# in the file after executing the log-dirty-bitmap command either +# from the qmp or hmp interface. This file only processes binary +# file obtained via command. +# +# Copyright (C) 2014 Sanidhya Kashyap <[email protected]> +# +# Authors: +# Sanidhya Kashyap +# +# +# This work is licensed under the terms of the GNU GPL, version 2 or later. + +import struct +import argparse +from functools import partial +from math import sqrt +from numpy import array +from pylab import figure,imshow,show,gray + +long_bytes = 8 +byte_size = 8 +int_bytes = 4 +complete_bitmap_list = [] +block_list = [] + +def get_unsigned_long_integer(value): + return struct.unpack('<Q', value)[0] + +def get_long_integer(value): + return struct.unpack('<q', value)[0] + +def get_integer(value): + return struct.unpack('<i', value)[0] + +def get_char(value): + return struct.unpack('<c', value)[0] + +def get_string(value, length): + name = struct.unpack('<'+str(length)+'s', value)[0] + for i in range(len(name)): + if name[i] == '\x00': + return name[:i] + +def dec2bin(decimal): + bin_value = bin(decimal)[2:] + if len(bin_value) < long_bytes * byte_size: + add_zeroes = long_bytes * byte_size - len(bin_value) + for i in range(add_zeroes): + bin_value += "0" + return str(bin_value) + +def get_bitmap_length(ram_bitmap_pages): + bitmap_length = ram_bitmap_pages / (long_bytes * byte_size) + if ram_bitmap_pages % (long_bytes * byte_size) != 0: + bitmap_length += 1 + return bitmap_length + +def dump_ram_block_info(infile): + total_blocks = get_integer(infile.read(int_bytes)) + for i in range(total_blocks): + block_name_length = get_integer(infile.read(int_bytes)) + block_name = get_string(infile.read(block_name_length), block_name_length) + block_offset = get_unsigned_long_integer(infile.read(long_bytes)) + block_length = get_unsigned_long_integer(infile.read(long_bytes)) + block_list.append(dict(name=block_name, offset=block_offset, length=block_length)) + +def generate_images(): + r = 0 + for list in complete_bitmap_list: + all_digits = "" + for element in list: + all_digits += dec2bin(element) + l = len(all_digits) + sqrtvalue = int(sqrt(l)) + for i in range(sqrtvalue * (sqrtvalue+1)-l): + all_digits+="0" + + v = [] + l = len(all_digits) + for i in range(sqrtvalue+1): + v1 = [] + for j in range(sqrtvalue): + v1.append(int(all_digits[i*sqrtvalue+j])) + v.append(v1) + + im_array = array(v) + figure(r) + imshow(im_array, cmap=gray()) + r += 1 + show() + +def dump_bitmap(infile, draw): + marker = 'M' + count = 0 + value = ' ' + current_ram_bitmap_pages = 0 + prev_ram_bitmap_pages = 0 + while True: + if len(value) == 0 or marker != 'M': + print "issue with the dump" + return + bitmap_page_raw_value = infile.read(long_bytes) + if not bitmap_page_raw_value: + break + current_ram_bitmap_pages = get_long_integer(bitmap_page_raw_value) + if current_ram_bitmap_pages != prev_ram_bitmap_pages: + prev_ram_bitmap_pages = current_ram_bitmap_pages + dump_ram_block_info(infile) + + bitmap_length = get_bitmap_length(current_ram_bitmap_pages) + bitmap_list = [] + bitmap_raw_value = infile.read(long_bytes * bitmap_length) + if not bitmap_raw_value: + break + count+=1 + for i in range(bitmap_length): + mark = i * long_bytes + bitmap_list.append((get_unsigned_long_integer(bitmap_raw_value[mark:mark+long_bytes]))) + complete_bitmap_list.append(bitmap_list) + value = infile.read(1) + marker = get_char(value) + if draw is True: + generate_images() + else: + print complete_bitmap_list + +def main(): + extracter = argparse.ArgumentParser(description='Extract dirty bitmap from binary file.') + extracter.add_argument('infile', help='Input file to extract the bitmap') + extracter.add_argument('-d', action='store_true', dest='draw', default=False, + help='Draw a black and white image of the processed dirty bitmap') + args = extracter.parse_args() + print 'The filename is {}'.format(args.infile) + + infile = open(format(args.infile), 'rb') + + dump_bitmap(infile, args.draw); + + infile.close() + +if __name__ == '__main__': + main()Signature · 1 line
1.9.3
--
Dr. David Alan Gilbert / [email protected] / Manchester, UK
Re: [Qemu-devel] [PATCH v5 6/6] BitmapLog: python script for extracting bitmap from a binary file
Quoted 18 lines · * Dr. David Alan Gilbert ([email protected]) wrote:
Quoted 11 lines · * Sanidhya Kashyap ([email protected]) wrote:
The file not only extracts the bitmap from the file but also draws the figure if required. Currently, figure is drawn for all the bitmaps. Later, I'll make the change to draw for different blocks.
The picture is drawn by generating a matrix of 0s and 1s from the bitmap. The dimensions are calculated on the basis of total bitmap pages which is represented as sqrt(total pages) X (sqrt(total pages) + 1). The white parts indicate non dirtied region while the black - dirtied region.
The python code requires some libraries such as numpy, pylab and math to generate the images.
This is interesting; I've tried this out with a Fedora boot and recorded about 240 frames; the script has problems when trying to convert that big a recording to images;
1) It loads it all into memory - so it used about 2GB of RAM
2) It opens each image as a window - so I then had to close about 240 windows
Quoted 8 lines
I fiddled with it a bit and created .png files, and then stiched these together with imagemagick to create an animation. It's quite interesting watch the OS boot and then see me login and start google's stress apptest which suddenly changes the whole of memory.
I suggest making it dump each image to a file in a similar way and making it free the python image data after each frame, then there shouldn't be a size limit. (I did try and upload the image, but QEMU's wiki didn't like the size)
and here's the (scaled down for size) looping gif:
http://wiki.qemu-project.org/images/9/9a/Fedboot50.gif
Dave
Quoted 159 lines
Dave
Quoted 155 lines
Signed-off-by Sanidhya Kashyap <[email protected]>--- scripts/extract-bitmap.py | 144 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100755 scripts/extract-bitmap.pyscripts/extract-bitmap.py new file +142
-0@@ -0,0 +1,144 @@ +#!/usr/bin/python +# This python script helps in extracting the dirty bitmap present +# in the file after executing the log-dirty-bitmap command either +# from the qmp or hmp interface. This file only processes binary +# file obtained via command. +# +# Copyright (C) 2014 Sanidhya Kashyap <[email protected]> +# +# Authors: +# Sanidhya Kashyap +# +# +# This work is licensed under the terms of the GNU GPL, version 2 or later. + +import struct +import argparse +from functools import partial +from math import sqrt +from numpy import array +from pylab import figure,imshow,show,gray + +long_bytes = 8 +byte_size = 8 +int_bytes = 4 +complete_bitmap_list = [] +block_list = [] + +def get_unsigned_long_integer(value): + return struct.unpack('<Q', value)[0] + +def get_long_integer(value): + return struct.unpack('<q', value)[0] + +def get_integer(value): + return struct.unpack('<i', value)[0] + +def get_char(value): + return struct.unpack('<c', value)[0] + +def get_string(value, length): + name = struct.unpack('<'+str(length)+'s', value)[0] + for i in range(len(name)): + if name[i] == '\x00': + return name[:i] + +def dec2bin(decimal): + bin_value = bin(decimal)[2:] + if len(bin_value) < long_bytes * byte_size: + add_zeroes = long_bytes * byte_size - len(bin_value) + for i in range(add_zeroes): + bin_value += "0" + return str(bin_value) + +def get_bitmap_length(ram_bitmap_pages): + bitmap_length = ram_bitmap_pages / (long_bytes * byte_size) + if ram_bitmap_pages % (long_bytes * byte_size) != 0: + bitmap_length += 1 + return bitmap_length + +def dump_ram_block_info(infile): + total_blocks = get_integer(infile.read(int_bytes)) + for i in range(total_blocks): + block_name_length = get_integer(infile.read(int_bytes)) + block_name = get_string(infile.read(block_name_length), block_name_length) + block_offset = get_unsigned_long_integer(infile.read(long_bytes)) + block_length = get_unsigned_long_integer(infile.read(long_bytes)) + block_list.append(dict(name=block_name, offset=block_offset, length=block_length)) + +def generate_images(): + r = 0 + for list in complete_bitmap_list: + all_digits = "" + for element in list: + all_digits += dec2bin(element) + l = len(all_digits) + sqrtvalue = int(sqrt(l)) + for i in range(sqrtvalue * (sqrtvalue+1)-l): + all_digits+="0" + + v = [] + l = len(all_digits) + for i in range(sqrtvalue+1): + v1 = [] + for j in range(sqrtvalue): + v1.append(int(all_digits[i*sqrtvalue+j])) + v.append(v1) + + im_array = array(v) + figure(r) + imshow(im_array, cmap=gray()) + r += 1 + show() + +def dump_bitmap(infile, draw): + marker = 'M' + count = 0 + value = ' ' + current_ram_bitmap_pages = 0 + prev_ram_bitmap_pages = 0 + while True: + if len(value) == 0 or marker != 'M': + print "issue with the dump" + return + bitmap_page_raw_value = infile.read(long_bytes) + if not bitmap_page_raw_value: + break + current_ram_bitmap_pages = get_long_integer(bitmap_page_raw_value) + if current_ram_bitmap_pages != prev_ram_bitmap_pages: + prev_ram_bitmap_pages = current_ram_bitmap_pages + dump_ram_block_info(infile) + + bitmap_length = get_bitmap_length(current_ram_bitmap_pages) + bitmap_list = [] + bitmap_raw_value = infile.read(long_bytes * bitmap_length) + if not bitmap_raw_value: + break + count+=1 + for i in range(bitmap_length): + mark = i * long_bytes + bitmap_list.append((get_unsigned_long_integer(bitmap_raw_value[mark:mark+long_bytes]))) + complete_bitmap_list.append(bitmap_list) + value = infile.read(1) + marker = get_char(value) + if draw is True: + generate_images() + else: + print complete_bitmap_list + +def main(): + extracter = argparse.ArgumentParser(description='Extract dirty bitmap from binary file.') + extracter.add_argument('infile', help='Input file to extract the bitmap') + extracter.add_argument('-d', action='store_true', dest='draw', default=False, + help='Draw a black and white image of the processed dirty bitmap') + args = extracter.parse_args() + print 'The filename is {}'.format(args.infile) + + infile = open(format(args.infile), 'rb') + + dump_bitmap(infile, args.draw); + + infile.close() +
Quoted 8 lines
Quoted 6 lines
scripts/extract-bitmap.py new file +2
-0+if __name__ == '__main__': + main()Signature · 1 line
1.9.3--
Dr. David Alan Gilbert / [email protected] / Manchester, UK
--
Dr. David Alan Gilbert / [email protected] / Manchester, UK