Obtain dirty bitmap via VM logging
v6 cover Obtain dirty bitmap via VM logging
Hi,
The following patches introduce the support of the dirty bitmap logging and dumping to a specified file. This patch series addresses the previous issues raised by David and Eric.
v5 --> v6
* The shared function between migration and log bitmap has been again modified (David's advice).
* Again changed the naming conventions for shared state between migration and log dirty bitmap (David's and Eric's advice).
* Removed acronyms (Eric's advice).
* Fixed grammatical mistakes and macros (Eric's advice).
* Modified the python script to utilize less memory as well as dump the snapshot into a file instead of separate window (David's advice).
v4 --> v5
* Removed the runstates code (Eric's and David's advice).
* Now, using global variable to resolve the issue between migration and bitmap dump (David's advice).
* Merged the hmp and qmp interface's patch (Eric's advice).
* Modified the code to support the dynamic change in last_ram_offset (David's advice).
* Formatted the strings in the python code i.e. length followed by text (David's advice).
* Added some code to generate black and white figure based on the bitmap.
* Modified the shared function to be more generic (David's advice).
v3 --> v4
* Added new qmp interface for information extraction from the bitmap process
v2 --> v3
* Reformatted the code and removed some unnecessary parts.
* Printing block info along with length and offset.
* Changed the functions that were directly using RUN_STATE_RUNNING as state.
v1 --> v2:
* Added two new run states to avoid simultaneous execution of both migration and bitmap dump process.
* Removed FILE pointer usage.
* Dumping the data only in machine-readable format.
* Tried to rectify mistakes of the previous version.
Sanidhya Kashyap (6):
generic function between migration and bitmap dump
BitmapLog: bitmap dump code
BitmapLog: get the information about the parameters
BitmapLog: cancel mechanism for an already running dump bitmap process
BitmapLog: set the period of the dump bitmap process
BitmapLog: python script for extracting bitmap from a binary file
arch_init.c | 19 ++- hmp-commands.hx | 47 ++++++ hmp.c | 56 ++++++ hmp.h | 4 + include/exec/cpu-all.h | 5 +- include/exec/ram_addr.h | 4 + include/sysemu/sysemu.h | 5 + migration.c | 12 ++ monitor.c | 7 + qapi-schema.json | 86 ++++++++++ qmp-commands.hx | 101 +++++++++++ savevm.c | 423 ++++++++++++++++++++++++++++++++++++++++++++++ scripts/extract-bitmap.py | 213 +++++++++++++++++++++++ vl.c | 23 +++ 14 files changed, 996 insertions(+), 9 deletions(-) create mode 100755 scripts/extract-bitmap.py
Signature · 1 line
1.9.1
PATCH v6 1/6 generic function between migration and bitmap dump
Now, the counter variable is available to all the functions that will call qemu_bitmap_sync_range. I have tried to make the function along with the variables as generic as possible.
--- arch_init.c | 19 +++++++++++-------- include/exec/ram_addr.h | 4 ++++ 2 files changed, 15 insertions(+), 8 deletions(-)
arch_init.c +11 -8
@@ -436,20 +436,22 @@ ram_addr_t migration_bitmap_find_and_reset_dirty(MemoryRegion *mr,return (next - base) <<TARGET_PAGE_BITS ; } -static inline bool migration_bitmap_set_dirty (ram_addr_t addr) +static inline bool qemu_bitmap_set_dirty (ram_addr_t addr,unsigned long *bitmap, +uint64_t *counter) {bool ret;int nr = addr >>TARGET_PAGE_BITS ; - ret =test_and_set_bit (nr, migration_bitmap); + ret =test_and_set_bit (nr, bitmap);if (!ret) { - migration_dirty_pages++; + (*counter)++; }return ret; } -static void migration_bitmap_sync_range (ram_addr_t start,ram_addr_t length) +void qemu_bitmap_sync_range (ram_addr_t start,ram_addr_t length, +unsigned long *bitmap,uint64_t *counter) {ram_addr_t addr;unsigned long page =BIT_WORD (start >>TARGET_PAGE_BITS );
@@ -463,8 +465,8 @@ static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)for (k = page; k < page + nr; k++) {if (src[k]) {unsigned long new_dirty; - new_dirty = ~migration_bitmap[k]; - migration_bitmap[k] |= src[k]; + new_dirty = ~bitmap[k]; + bitmap[k] |= src[k]; new_dirty &= src[k]; migration_dirty_pages +=ctpopl (new_dirty); src[k] =0 ;
@@ -478,7 +480,7 @@ static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)cpu_physical_memory_reset_dirty (start + addr,TARGET_PAGE_SIZE ,DIRTY_MEMORY_MIGRATION ); -migration_bitmap_set_dirty (start + addr); +qemu_bitmap_set_dirty (start + addr, bitmap, counter); } } }
@@ -514,7 +516,8 @@ static void migration_bitmap_sync(void)address_space_sync_dirty_bitmap (&address_space_memory); QTAILQ_FOREACH(block, &ram_list.blocks, next) { -migration_bitmap_sync_range (block->mr->ram_addr, block->length); +qemu_bitmap_sync_range (block->mr->ram_addr, block->length, + migration_bitmap, &migration_dirty_pages); }trace_migration_bitmap_sync_end (migration_dirty_pages - num_dirty_pages_init);
include/exec/ram_addr.h +4 -0
@@ -162,5 +162,9 @@ static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,void cpu_physical_memory_reset_dirty (ram_addr_t start,ram_addr_t length,unsigned client); + +void qemu_bitmap_sync_range (ram_addr_t start,ram_addr_t length, +unsigned long *bitmap,uint64_t *counter); +#endif #endif
Signature · 1 line
1.9.1
PATCH v6 2/6 BitmapLog: bitmap dump code
In reply to Sanidhya Kashyap---
No functional change, except:
- naming convention i.e. QemuProcess has been changed to QemuDirtyBitmapUser.
- rectified mistakes in documentation in qapi-schema.json.
- removed acronyms
hmp-commands.hx | 16 ++ hmp.c | 18 +++ hmp.h | 1 + include/exec/cpu-all.h | 5 +- include/sysemu/sysemu.h | 5 + migration.c | 12 ++ qapi-schema.json | 37 +++++ qmp-commands.hx | 32 ++++ savevm.c | 378 ++++++++++++++++++++++++++++++++++++++++++++++++ vl.c | 23 +++ 10 files changed, 526 insertions(+), 1 deletion(-)
hmp-commands.hx +16 -0
@@ -1786,6 +1786,22 @@ STEXI show available trace events and their state ETEXI + { + .name = "log_dirty_bitmap", + .args_type = "filename:s,iterations:i?,period:i?", + .params = "filename iterations period", + .help = "dumps the memory's dirty bitmap to file\n\t\t\t" + "filename: name of the file in which the bitmap will be saved\n\t\t\t" + "iterations: number of times the memory will be logged\n\t\t\t" + "period: time difference in milliseconds between each iteration", + .mhandler.cmd = hmp_log_dirty_bitmap, + }, +STEXI +@item log_dirty_bitmap @var{filename} +@findex log_dirty_bitmap +dumps the writable working set of a VM's memory to a file +ETEXI + STEXI @end table ETEXI
hmp.c +18 -0
@@ -1318,6 +1318,24 @@ void hmp_device_del(Monitor *mon, const QDict *qdict)hmp_handle_error (mon, &err); } +void hmp_log_dirty_bitmap (Monitor *mon,const QDict *qdict) +{ +const char *filename =qdict_get_str (qdict,"filename" ); +bool has_iterations =qdict_haskey (qdict,"iterations" ); +int64_t iterations =qdict_get_try_int (qdict,"iterations" ,3 ); +bool has_period =qdict_haskey (qdict,"period" ); +int64_t period =qdict_get_try_int (qdict,"period" ,10 ); +Error *err =NULL ; + +qmp_log_dirty_bitmap (filename, has_iterations, iterations, + has_period, period, &err); +if (err) { +monitor_printf (mon,"log_dirty_bitmap: %s\n" ,error_get_pretty (err)); +error_free (err); +return ; + } +} +void hmp_dump_guest_memory (Monitor *mon,const QDict *qdict) {Error *err =NULL ;
hmp.h +1 -0
@@ -94,6 +94,7 @@ void hmp_cpu_add(Monitor *mon, const QDict *qdict);void hmp_object_add (Monitor *mon,const QDict *qdict);void hmp_object_del (Monitor *mon,const QDict *qdict);void hmp_info_memdev (Monitor *mon,const QDict *qdict); +void hmp_log_dirty_bitmap (Monitor *mon,const QDict *qdict);void object_add_completion (ReadLineState *rs,int nb_args,const char *str);void object_del_completion (ReadLineState *rs,int nb_args,const char *str);void device_add_completion (ReadLineState *rs,int nb_args,const char *str);
include/exec/cpu-all.h +4 -1
@@ -299,13 +299,16 @@ CPUArchState *cpu_copy(CPUArchState *env);/* memory API */ +/* global name which is used with both migration and bitmap dump */ +#define RAMBLOCK_NAME_LENGTH 256 + typedef struct RAMBlock { struct MemoryRegion *mr; uint8_t *host; ram_addr_t offset; ram_addr_t length; uint32_t flags; - char idstr[256]; + char idstr[RAMBLOCK_NAME_LENGTH]; /* Reads can take either the iothread or the ramlist lock. * Writes must take both locks. */
include/sysemu/sysemu.h +5 -0
@@ -227,4 +227,9 @@ extern QemuOptsList qemu_net_opts;extern QemuOptsList qemu_global_opts;extern QemuOptsList qemu_mon_opts; +/* migration vs dirty bitmap process */ +bool qemu_process_check (QemuDirtyBitmapUser user); +void qemu_process_set (QemuDirtyBitmapUser user); +const char *get_qemu_dirty_bitmap_user_as_string (void ); +#endif
migration.c +12 -0
@@ -117,6 +117,7 @@ static void process_incoming_migration_co(void *opaque) }else {runstate_set (RUN_STATE_PAUSED ); } +qemu_process_set (QEMU_DIRTY_BITMAP_USER_NONE ); }void process_incoming_migration (QEMUFile *f)
@@ -317,6 +318,7 @@ static void migrate_fd_cleanup(void *opaque) }notifier_list_notify (&migration_state_notifiers, s); +qemu_process_set (QEMU_DIRTY_BITMAP_USER_NONE ); }void migrate_fd_error (MigrationState *s)
@@ -326,6 +328,7 @@ void migrate_fd_error(MigrationState *s) s->state =MIG_STATE_ERROR ;trace_migrate_set_state (MIG_STATE_ERROR );notifier_list_notify (&migration_state_notifiers, s); +qemu_process_set (QEMU_DIRTY_BITMAP_USER_NONE ); }static void migrate_fd_cancel (MigrationState *s)
@@ -436,6 +439,15 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,return ; } +if (!qemu_process_check (QEMU_DIRTY_BITMAP_USER_NONE ) && + !qemu_process_check (QEMU_DIRTY_BITMAP_USER_MIGRATION )) { +error_setg (errp,"Migration not possible, since %s " +"is in progress." ,get_qemu_dirty_bitmap_user_as_string ()); +return ; + } + +qemu_process_set (QEMU_DIRTY_BITMAP_USER_MIGRATION ); + s =migrate_init (¶ms);if (strstart (uri,"tcp:" , &p)) {
qapi-schema.json +37 -0
@@ -3481,3 +3481,40 @@ # Since: 2.1 ## { 'command': 'rtc-reset-reinjection' } + +## +# QemuDirtyBitmapUser +# +# @none: no other process is being executed besides a simple VM execution. +# +# @migration: migration process is going on. +# +# @bitmap-dump: bitmap dump process is being executed. +# +# Since 2.2 +## +{ 'enum': 'QemuDirtyBitmapUser', + 'data': [ 'none', 'migration', 'bitmap-dump' ] } + +## +# @log-dirty-bitmap +# +# This command will dump the dirty bitmap to a file by logging the +# memory for a specified number of times with a defined time difference +# +# @filename: name of the file in which the bitmap will be saved. +# +# @iterations: #optional number of times the memory will be logged. The +# min and max values are 3 and 100000 respectively. 3 is the default value, +# if the iterations parameter is not stated. +# +# @period: #optional time difference in milliseconds between each iteration. +# The min and max values are 10 and 100000 respectively. 10 is the default +# value if the period parameter is not stated. +# +# Since 2.2 +## +{ 'command' : 'log-dirty-bitmap', + 'data' : { 'filename' : 'str', + '*iterations' : 'int', + '*period' : 'int' } }
qmp-commands.hx +32 -0
@@ -3752,5 +3752,37 @@ Example: -> { "execute": "rtc-reset-reinjection" } <- { "return": {} } +EQMP + { + .name = "log-dirty-bitmap", + .args_type = "filename:s,iterations:i?,period:i?", + .mhandler.cmd_new = qmp_marshal_input_log_dirty_bitmap, + }, + +SQMP +log-dirty-bitmap +---------------- + +start logging the memory of the VM for writable working set + +Arguments: + +- "filename": name of the file, in which the bitmap will be saved. + +- "iterations": #optional number of times, the memory will be logged. + The min and max values are 3 and 100000 respectively. 3 is the default value. + +- "period": #optional time difference in milliseconds between each iteration. + The min and max values are 10 and 100000 respectively. 10 is the default + value. + +Examples: +-> { "execute": "log-dirty-bitmap", + "arguments": { + "filename": "/tmp/fileXXX", + "iterations": 3, + "period": 10 } } + +<- { "return": {} } EQMP
savevm.c +378 -0
390 lines not on this page · raw message
vl.c +23 -0
@@ -206,6 +206,8 @@ bool qemu_uuid_set;static QEMUBootSetHandler *boot_set_handler;static void *boot_set_opaque; +int dirty_bitmap_user; +static NotifierList exit_notifiers =NOTIFIER_LIST_INITIALIZER (exit_notifiers);
@@ -768,7 +770,27 @@ void vm_start(void)qapi_event_send_resume (&error_abort); } +/* +* A global variable to decide which process will only +* execute migration or bitmap dump +*/ + +static QemuDirtyBitmapUser dbu =QEMU_DIRTY_BITMAP_USER_NONE ; + +bool qemu_process_check (QemuDirtyBitmapUser user) +{ +return user == dbu; +} + +void qemu_process_set (QemuDirtyBitmapUser user) +{ + dbu = user; +} +const char *get_qemu_dirty_bitmap_user_as_string (void ) +{ +return QemuDirtyBitmapUser_lookup[dbu]; +}/***********************************************************/ /* real time host monotonic timer */
@@ -4545,6 +4567,7 @@ int main(int argc, char **argv, char **envp) }if (incoming) { +qemu_process_set (QEMU_DIRTY_BITMAP_USER_MIGRATION );Error *local_err =NULL ;qemu_start_incoming_migration (incoming, &local_err);if (local_err) {
Signature · 1 line
1.9.1
PATCH v6 3/6 BitmapLog: get the information about the parameters
In reply to Sanidhya Kashyap---
Removed acronyms, no functional change.
hmp-commands.hx | 2 ++ hmp.c | 21 +++++++++++++++++++++ hmp.h | 1 + monitor.c | 7 +++++++ qapi-schema.json | 28 ++++++++++++++++++++++++++++ qmp-commands.hx | 25 +++++++++++++++++++++++++ savevm.c | 17 +++++++++++++++++ 7 files changed, 101 insertions(+)
hmp-commands.hx +2 -0
@@ -1778,6 +1778,8 @@ show qdev device model list show roms @item info tpm show the TPM device +@item info log_dirty_bitmap +show the current parameters values @end table ETEXI
hmp.c +21 -0
@@ -1732,3 +1732,24 @@ void hmp_info_memdev(Monitor *mon, const QDict *qdict)monitor_printf (mon,"\n" ); } + +void hmp_info_log_dirty_bitmap (Monitor *mon,const QDict *qdict) +{ +Error *err =NULL ; +BitmapLogStateInfo *info =qmp_query_log_dirty_bitmap (&err); + +if (info) { +monitor_printf (mon,"current iteration: %" PRId64"\n" , + info->current_iteration); +monitor_printf (mon,"total iterations: %" PRId64"\n" , + info->iterations); +monitor_printf (mon,"current period value: %" PRId64"\n" , + info->period); + } + +if (err) { +hmp_handle_error (mon, &err); + } + +qapi_free_BitmapLogStateInfo (info); +}
hmp.h +1 -0
@@ -38,6 +38,7 @@ void hmp_info_balloon(Monitor *mon, const QDict *qdict);void hmp_info_pci (Monitor *mon,const QDict *qdict);void hmp_info_block_jobs (Monitor *mon,const QDict *qdict);void hmp_info_tpm (Monitor *mon,const QDict *qdict); +void hmp_info_log_dirty_bitmap (Monitor *mon,const QDict *qdict);void hmp_quit (Monitor *mon,const QDict *qdict);void hmp_stop (Monitor *mon,const QDict *qdict);void hmp_system_reset (Monitor *mon,const QDict *qdict);
monitor.c +7 -0
@@ -2921,6 +2921,13 @@ static mon_cmd_t info_cmds[] = { .mhandler.cmd = hmp_info_memdev, }, { + .name = "log_dirty_bitmap", + .args_type = "", + .params = "", + .help = "show the current parameters values", + .mhandler.cmd = hmp_info_log_dirty_bitmap, + }, + { .name = NULL, }, };
qapi-schema.json +28 -0
@@ -3518,3 +3518,31 @@ 'data' : { 'filename' : 'str', '*iterations' : 'int', '*period' : 'int' } } +## +# @BitmapLogStateInfo +# +# Provides information for the bitmap logging process +# +# @current-iteration: stores current iteration value +# +# @iterations: total iterations value +# +# @period: the time difference in milliseconds between each iteration +# +# Since 2.2 +## +{ 'type': 'BitmapLogStateInfo', + 'data': { 'current-iteration' : 'int', + 'iterations' : 'int', + 'period' : 'int' } } + +## +# @query-log-dirty-bitmap +# +# Get the current values of the parameters involved in bitmap logging process +# +# This command returns the BitmapLogStateInfo +# +# Since 2.2 +## +{ 'command': 'query-log-dirty-bitmap', 'returns': 'BitmapLogStateInfo' }
qmp-commands.hx +25 -0
@@ -3786,3 +3786,28 @@ Examples: <- { "return": {} } EQMP + + { + .name = "query-log-dirty-bitmap", + .args_type = "", + .mhandler.cmd_new = qmp_marshal_input_query_log_dirty_bitmap, + }, + +SQMP +query-log-dirty-bitmap +---------------------- + +Get the parameters information + +- "current-iteration": stores current iteration value +- "iterations": total iterations value +- "period": the time difference in milliseconds between each iteration + +Example: + +-> { "execute": "query-log-dirty-bitmap" } +<- { "return": { + "current-iteration": 3, + "iterations": 10, + "period": 100 } } +EQMP
savevm.c +17 -0
@@ -1515,6 +1515,23 @@ void qmp_log_dirty_bitmap(const char *filename, bool has_iterations,return ; } +BitmapLogStateInfo *qmp_query_log_dirty_bitmap (Error **errp) +{ +BitmapLogState *b =log_bitmap_get_current_state (); +BitmapLogStateInfo *info =NULL ; + +if (b->state !=LOG_BITMAP_STATE_ACTIVE ) { +return info; + } + + info =g_malloc0 (sizeof (BitmapLogStateInfo)); + info->current_iteration = b->current_iteration; + info->iterations = b->iterations; + info->period = b->current_period; + +return info; +} +void qmp_xen_save_devices_state (const char *filename,Error **errp) {QEMUFile *f;
Signature · 1 line
1.9.1
PATCH v6 4/6 BitmapLog: cancel mechanism for an already running dump bitmap process
In reply to Sanidhya Kashyap---
No functional change, except acronyms have been removed.
Removed acronyms, no functional change.
hmp-commands.hx | 14 ++++++++++++++ hmp.c | 5 +++++ hmp.h | 1 + qapi-schema.json | 9 +++++++++ qmp-commands.hx | 20 ++++++++++++++++++++ savevm.c | 14 ++++++++++++++ 6 files changed, 63 insertions(+)
hmp-commands.hx +14 -0
@@ -1804,6 +1804,20 @@ STEXI dumps the writable working set of a VM's memory to a file ETEXI + { + .name = "log_dirty_bitmap_cancel", + .args_type = "", + .params = "", + .help = "cancel the current bitmap dump process", + .mhandler.cmd = hmp_log_dirty_bitmap_cancel, +}, + +STEXI +@item log_dirty_bitmap_cancel +@findex log_dirty_bitmap_cancel +Cancel the current bitmap dump process +ETEXI + STEXI @end table ETEXI
hmp.c +5 -0
@@ -1336,6 +1336,11 @@ void hmp_log_dirty_bitmap(Monitor *mon, const QDict *qdict) } } +void hmp_log_dirty_bitmap_cancel (Monitor *mon,const QDict *qdict) +{ +qmp_log_dirty_bitmap_cancel (NULL ); +} +void hmp_dump_guest_memory (Monitor *mon,const QDict *qdict) {Error *err =NULL ;
hmp.h +1 -0
@@ -96,6 +96,7 @@ void hmp_object_add(Monitor *mon, const QDict *qdict);void hmp_object_del (Monitor *mon,const QDict *qdict);void hmp_info_memdev (Monitor *mon,const QDict *qdict);void hmp_log_dirty_bitmap (Monitor *mon,const QDict *qdict); +void hmp_log_dirty_bitmap_cancel (Monitor *mon,const QDict *qdict);void object_add_completion (ReadLineState *rs,int nb_args,const char *str);void object_del_completion (ReadLineState *rs,int nb_args,const char *str);void device_add_completion (ReadLineState *rs,int nb_args,const char *str);
qapi-schema.json +9 -0
@@ -3546,3 +3546,12 @@ # Since 2.2 ## { 'command': 'query-log-dirty-bitmap', 'returns': 'BitmapLogStateInfo' } + +## +# @log-dirty-bitmap-cancel +# +# cancel the dirty bitmap logging process +# +# Since 2.2 +## +{ 'command': 'log-dirty-bitmap-cancel' }
qmp-commands.hx +20 -0
@@ -3811,3 +3811,23 @@ Example: "iterations": 10, "period": 100 } } EQMP + + { + .name = "log-dirty-bitmap-cancel", + .args_type = "", + .mhandler.cmd_new = qmp_marshal_input_log_dirty_bitmap_cancel, + }, + +SQMP +log-dirty-bitmap-cancel +----------------------- + +Cancel the current bitmap dump process. + +Arguments: None. + +Example: + +-> { "execute": "log-dirty-bitmap-cancel" } +<- { "return": {} } +EQMP
savevm.c +14 -0
@@ -1532,6 +1532,20 @@ BitmapLogStateInfo *qmp_query_log_dirty_bitmap(Error **errp)return info; } +void qmp_log_dirty_bitmap_cancel (Error **errp) +{ +BitmapLogState *b =log_bitmap_get_current_state (); +int old_state; +do { + old_state = b->state; +if (old_state !=LOG_BITMAP_STATE_ACTIVE ) { +break ; + } +log_bitmap_set_status (b, old_state, +LOG_BITMAP_STATE_CANCELING ); + }while (b->state !=LOG_BITMAP_STATE_CANCELING ); +} +void qmp_xen_save_devices_state (const char *filename,Error **errp) {QEMUFile *f;
Signature · 1 line
1.9.1
PATCH v6 5/6 BitmapLog: set the period of the dump bitmap process
In reply to Sanidhya Kashyap---
Removed acronyms, no functional change.
hmp-commands.hx | 15 +++++++++++++++ hmp.c | 12 ++++++++++++ hmp.h | 1 + qapi-schema.json | 12 ++++++++++++ qmp-commands.hx | 24 ++++++++++++++++++++++++ savevm.c | 14 ++++++++++++++ 6 files changed, 78 insertions(+)
hmp-commands.hx +15 -0
@@ -1818,6 +1818,21 @@ STEXI Cancel the current bitmap dump process ETEXI + { + .name = "log_dirty_bitmap_set_period", + .args_type = "period:i", + .params = "period", + .help = "set the period for bitmap dump process\n\t\t\t" + "period: the new period value to replace the existing", + .mhandler.cmd = hmp_log_dirty_bitmap_set_period, + }, + +STEXI +@item log_dirty_bitmap_set_period @var{period} +@findex log_dirty_bitmap_set_period +Set the period to @var{period} (int) for bitmap dump process. +ETEXI + STEXI @end table ETEXI
hmp.c +12 -0
@@ -1341,6 +1341,18 @@ void hmp_log_dirty_bitmap_cancel(Monitor *mon, const QDict *qdict)qmp_log_dirty_bitmap_cancel (NULL ); } +void hmp_log_dirty_bitmap_set_period (Monitor *mon,const QDict *qdict) +{ +int64_t period =qdict_get_int (qdict,"period" ); +Error *err =NULL ; +qmp_log_dirty_bitmap_set_period (period, &err); +if (err) { +monitor_printf (mon,"log_dirty_bitmap_set_period: %s\n" , +error_get_pretty (err)); +error_free (err); + } +} +void hmp_dump_guest_memory (Monitor *mon,const QDict *qdict) {Error *err =NULL ;
hmp.h +1 -0
@@ -97,6 +97,7 @@ void hmp_object_del(Monitor *mon, const QDict *qdict);void hmp_info_memdev (Monitor *mon,const QDict *qdict);void hmp_log_dirty_bitmap (Monitor *mon,const QDict *qdict);void hmp_log_dirty_bitmap_cancel (Monitor *mon,const QDict *qdict); +void hmp_log_dirty_bitmap_set_period (Monitor *mon,const QDict *qdict);void object_add_completion (ReadLineState *rs,int nb_args,const char *str);void object_del_completion (ReadLineState *rs,int nb_args,const char *str);void device_add_completion (ReadLineState *rs,int nb_args,const char *str);
qapi-schema.json +12 -0
@@ -3555,3 +3555,15 @@ # Since 2.2 ## { 'command': 'log-dirty-bitmap-cancel' } + +## +# @log-dirty-bitmap-set-period +# +# sets the period of the dirty bitmap logging process +# @frequency: the updated period value (in milliseconds). +# The min and max values are 10 and 100000 respectively. +# +# Since 2.2 +## +{ 'command': 'log-dirty-bitmap-set-period', + 'data': { 'period': 'int' } }
qmp-commands.hx +24 -0
@@ -3831,3 +3831,27 @@ Example: -> { "execute": "log-dirty-bitmap-cancel" } <- { "return": {} } EQMP + + { + .name = "log-dirty-bitmap-set-period", + .args_type = "period:i", + .mhandler.cmd_new = qmp_marshal_input_log_dirty_bitmap_set_period, + }, + +SQMP +log-dirty-bitmap-set-period +--------------------------- + +Update the period for the remaining iterations. + +Arguments: + +- "period": The updated period (json-int) (in milliseconds). + The min and max values are 10 and 100000 respectively. + +Example: + +-> { "execute": "log-dirty-bitmap-set-period", "arguments": { "period": 1024 } } +<- { "return": {} } + +EQMP
savevm.c +14 -0
@@ -1546,6 +1546,20 @@ void qmp_log_dirty_bitmap_cancel(Error **errp) }while (b->state !=LOG_BITMAP_STATE_CANCELING ); } +void qmp_log_dirty_bitmap_set_period (int64_t period,Error **errp) +{ +BitmapLogState *b =log_bitmap_get_current_state (); +Error *local_err =NULL ; +if (value_in_range (period,MIN_PERIOD_VALUE , +MAX_PERIOD_VALUE ,"period" , &local_err)) { +if (local_err) { +error_propagate (errp, local_err); +return ; + } + } + b->current_period = period; +} +void qmp_xen_save_devices_state (const char *filename,Error **errp) {QEMUFile *f;
Signature · 1 line
1.9.1
PATCH v6 6/6 BitmapLog: python script for extracting bitmap from a binary file
In reply to Sanidhya KashyapI have modified the script to support the dump of the images to the file. Earlier, everything was saved to the memory and later the processing was taking place. Now, I have tried to solve that issue with only using the required memory.
After discussion with David, I have tried to select a base 2 matrix dimension like 512 X 512 or 1024 X 512 etc for the dumping of the bitmap. But, I am still supporting the default square root based method of dimension selection.
---
scripts/extract-bitmap.py | 213 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100755 scripts/extract-bitmap.py
scripts/extract-bitmap.py new file +213 -0
@@ -0,0 +1,213 @@ +#!/usr/bin/python +# This python script helps in extracting the dirty bitmap present +# in the file after executing the log-dirty-bitmap command either +# from the qmp or hmp interface. This file only processes binary +# file obtained via command. +# +# Copyright (C) 2014 Sanidhya Kashyap <[email protected]> +# +# Authors: +# Sanidhya Kashyap +# +# +# This work is licensed under the terms of the GNU GPL, version 2 or later. + +import struct +import argparse +from functools import partial +from math import sqrt +from numpy import array +from pylab import figure, imshow, savefig, gray, xlim, ylim +from os import path, makedirs + +long_bytes = 8 +byte_size = 8 +int_bytes = 4 +block_list = [] +total_blocks = 0 + +def get_unsigned_long_integer(value): + return struct.unpack('<Q', value)[0] + +def get_long_integer(value): + return struct.unpack('<q', value)[0] + +def get_integer(value): + return struct.unpack('<i', value)[0] + +def get_char(value): + return struct.unpack('<c', value)[0] + +def get_string(value, length): + name = struct.unpack('<'+str(length)+'s', value)[0] + for i in range(len(name)): + if name[i] == '\x00': + return name[:i] + +def dec2bin(decimal): + bin_value = bin(decimal)[2:] + if len(bin_value) < long_bytes * byte_size: + add_zeroes = long_bytes * byte_size - len(bin_value) + for i in range(add_zeroes): + bin_value += "0" + return str(bin_value) + +def get_bitmap_length(ram_bitmap_pages): + bitmap_length = ram_bitmap_pages / (long_bytes * byte_size) + if ram_bitmap_pages % (long_bytes * byte_size) != 0: + bitmap_length += 1 + return bitmap_length + +def get_block_info(): + print "Select any one of the following:" + for i in range(len(block_list)): + print str(i) + " " + block_list[i]['name'] + return int(raw_input('Enter the number: ')) + +def get_matrix(all_digits, y, x): + v = [] + xlim(xmin = 0, xmax = x) + ylim(ymin = 0, ymax = y) + for i in range(y): + v1 = [] + for j in range(x): + v1.append(int(all_digits[i * x + j])) + v.append(v1) + return v + +def get_matrix_base2(all_digits): + l = len(all_digits) + sqrtvalue = int(sqrt(l)) + x = 2 ** (sqrtvalue.bit_length() - 1) + y = x * 2 + if (x * y - l < 0): + x = y + for i in range(x * y - l): + all_digits += "0" + + return get_matrix(all_digits, x, y); + +def get_matrix_sqrt(all_digits): + l = len(all_digits) + sqrtvalue = int(sqrt(l)) + for i in range(sqrtvalue * (sqrtvalue + 1) - l): + all_digits += "0" + + if sqrtvalue * sqrtvalue == l: + return get_matrix(all_digits, sqrtvalue, sqrtvalue) + else: + return get_matrix(all_digits, sqrtvalue, sqrtvalue + 1) + +def dump_ram_block_info(infile): + total_blocks = get_integer(infile.read(int_bytes)) + for i in range(total_blocks): + block_name_length = get_integer(infile.read(int_bytes)) + block_name = get_string(infile.read(block_name_length), block_name_length) + block_offset = get_unsigned_long_integer(infile.read(long_bytes)) + block_length = get_unsigned_long_integer(infile.read(long_bytes)) + block_list.append(dict(name=block_name, offset=block_offset, length=block_length)) + +def generate_image(all_digits, num, debug, dir, base): + v = [] + + if base is False: + v = get_matrix_sqrt(all_digits) + else: + v = get_matrix_base2(all_digits) + + im_array = array(v) + figure(num) + imshow(im_array, cmap=gray()) + + filename=dir + "/" + "out_" + str(num) + ".png" + del v + savefig(filename) + if debug is True: + print 'file ' + filename + ' dumped' + + +def dump_bitmap(args): + marker = 'M' + count = 0 + value = ' ' + block_offset = 0 + block_length = 0 + block_num = 0 + current_ram_bitmap_pages = 0 + prev_ram_bitmap_pages = 0 + infile = open(format(args.infile), 'rb') + debug = args.debug + dump_all = args.dump_all + dir = args.dir + base = args.base + + if not path.exists(dir): + makedirs(dir) + + while True: + if len(value) == 0 or marker != 'M': + print "issue with the dump" + return + bitmap_page_raw_value = infile.read(long_bytes) + if not bitmap_page_raw_value: + break + current_ram_bitmap_pages = get_long_integer(bitmap_page_raw_value) + if current_ram_bitmap_pages != prev_ram_bitmap_pages: + prev_ram_bitmap_pages = current_ram_bitmap_pages + dump_ram_block_info(infile) + # asking what should be printed + if dump_all is False: + if count == 0: + block_num = get_block_info() + if debug is True: + print block_list[block_num]['name'] + ' selected' + else: + x = block_num + y = len(block_list) / total_blocks + block_num = total_blocks * (y-1) + x + block_offset = block_list[block_num]['offset'] + block_length = block_list[block_num]['length'] + if debug is True: + print 'total ram bitmap pages: ' + str(current_ram_bitmap_pages) + print block_list[block_num]['name'] + ' offset: ' + str(block_offset) + print block_list[block_num]['name'] + ' length: ' + str(block_length) + + bitmap_length = get_bitmap_length(current_ram_bitmap_pages) + bitmap_raw_value = infile.read(long_bytes * bitmap_length) + if not bitmap_raw_value: + break + + count+=1 + all_digits="" + for i in range(bitmap_length): + mark = i * long_bytes + all_digits += dec2bin(get_unsigned_long_integer(bitmap_raw_value[mark : mark + long_bytes])) + + if dump_all is False: + generate_image(all_digits[block_offset : block_offset + block_length], count, debug, dir, base) + else: + generate_image(all_digits, count, debug, dir, base) + value = infile.read(1) + marker = get_char(value) + infile.close() + +def main(): + extracter = argparse.ArgumentParser(description='Extract dirty bitmap from binary file.') + extracter.add_argument('-f', '--file', dest='infile', + help='Input file to extract the bitmap', metavar='FILE') + extracter.add_argument('-a', '--all', action='store_true', dest='dump_all', default=False, + help='Use all memory blocks for the figure') + extracter.add_argument('-d', '--debug', action='store_true', dest='debug', default=False, + help='print the debug info') + extracter.add_argument('-dir', dest='dir', default='.', + help='directory to store the image files') + extracter.add_argument('-b', '--base2', dest='base', action='store_true', default=False, + help='dump the bitmap array in powers of 2') + + args = extracter.parse_args() + print 'The filename is {}'.format(args.infile) + + dump_bitmap(args) + +if __name__ == '__main__': + main()
Signature · 1 line
1.9.1