block: add logging facility for long standing IO requests
v2 cover block: add logging facility for long standing IO requests
There are severe delays with IO requests processing if QEMU is running in virtual machine or over software defined storage. Such delays potentially results in unpredictable guest behavior. For example, guests over IDE or SATA drive could remount filesystem read-only if write is performed longer than 10 seconds.
Such reports are very complex to process. Some good starting point for this seems quite reasonable. This patch provides one. It adds logging of such potentially dangerous long IO operations.
Changes from v1:
- fixed conversions using macros suggested by Stefan
- fixed option declaration
- enabled by default with patch 3
PATCH v2 1/3 block/block-backend: add converter from BlockAcctStats to BlockBackend
Right now BlockAcctStats is always reside on BlockBackend. This structure is not used in any other place. Thus we are able to create a converter from one pointer to another.
--- block/block-backend.c | 5 +++++ include/sysemu/block-backend.h | 1 + 2 files changed, 6 insertions(+)
block/block-backend.c +5 -0
@@ -2153,6 +2153,11 @@ BlockAcctStats *blk_get_stats(BlockBackend *blk)return &blk->stats; } +BlockBackend *blk_stats2blk (BlockAcctStats *s) +{ +return container_of (s, BlockBackend, stats); +} +void *blk_aio_get (const AIOCBInfo *aiocb_info,BlockBackend *blk,BlockCompletionFunc *cb,void *opaque) {
include/sysemu/block-backend.h +1 -0
@@ -227,6 +227,7 @@ void blk_add_insert_bs_notifier(BlockBackend *blk, Notifier *notify);void blk_io_plug (BlockBackend *blk);void blk_io_unplug (BlockBackend *blk);BlockAcctStats *blk_get_stats (BlockBackend *blk); +BlockBackend *blk_stats2blk (BlockAcctStats *stats);BlockBackendRootState *blk_get_root_state (BlockBackend *blk);void blk_update_root_state (BlockBackend *blk);bool blk_get_detect_zeroes_from_root_state (BlockBackend *blk);
Signature · 1 line
2.17.1
PATCH v2 2/3 block: add logging facility for long standing IO requests
In reply to Denis V. LunevThere are severe delays with IO requests processing if QEMU is running in virtual machine or over software defined storage. Such delays potentially results in unpredictable guest behavior. For example, guests over IDE or SATA drive could remount filesystem read-only if write is performed longer than 10 seconds.
Such reports are very complex to process. Some good starting point for this seems quite reasonable. This patch provides one. It adds logging of such potentially dangerous long IO operations.
--- block/accounting.c | 59 +++++++++++++++++++++++++++++++++++++- blockdev.c | 7 ++++- include/block/accounting.h | 5 +++- slirp | 2 +- 4 files changed, 69 insertions(+), 4 deletions(-)
block/accounting.c +58 -1
@@ -27,7 +27,9 @@#include "block/accounting.h" #include "block/block_int.h" #include "qemu/timer.h" +#include "qemu/log.h" #include "sysemu/qtest.h" +#include "sysemu/block-backend.h" static QEMUClockType clock_type =QEMU_CLOCK_REALTIME ;static const int qtest_latency_ns =NANOSECONDS_PER_SECOND /1000 ;
@@ -41,10 +43,11 @@ void block_acct_init(BlockAcctStats *stats) }void block_acct_setup (BlockAcctStats *stats,bool account_invalid, -bool account_failed) +bool account_failed,unsigned latency_log_threshold_ms) { stats->account_invalid = account_invalid; stats->account_failed = account_failed; + stats->latency_log_threshold_ms = latency_log_threshold_ms; }void block_acct_cleanup (BlockAcctStats *stats)
@@ -182,6 +185,58 @@ void block_latency_histograms_clear(BlockAcctStats *stats) } } +static const char *block_account_type (enum BlockAcctType type) +{ +switch (type) { +case BLOCK_ACCT_READ : +return "READ" ; +case BLOCK_ACCT_WRITE : +return "WRITE" ; +case BLOCK_ACCT_FLUSH : +return "DISCARD" ; +case BLOCK_ACCT_UNMAP : +return "TRUNCATE" ; +case BLOCK_ACCT_NONE : +return "NONE" ; +case BLOCK_MAX_IOTYPE : +break ; + } +return "UNKNOWN" ; +} + +static void block_acct_report_long (BlockAcctStats *stats, +BlockAcctCookie *cookie,int64_t latency_ns) +{ +unsigned latency_ms = latency_ns /SCALE_MS ; +int64_t start_time_host_s; +char buf[64 ]; +struct tm t; +BlockDriverState *bs; + +if (cookie->type ==BLOCK_ACCT_NONE ) { +return ; + } +if (stats->latency_log_threshold_ms ==0 ) { +return ; + } +if (latency_ms < stats->latency_log_threshold_ms) { +return ; + } + + start_time_host_s = cookie->start_time_ns /NANOSECONDS_PER_SECOND ; +strftime (buf,sizeof (buf),"%m-%d %H:%M:%S" , +localtime_r (&start_time_host_s, &t)); + + bs =blk_bs (blk_stats2blk (stats)); +qemu_log ("long %s[%ld] IO request: %d.03%d since %s.%03d bs: %s(%s, %s)\n" , +block_account_type (cookie->type), cookie->bytes, + (int )(latency_ms /1000 ), (int )(latency_ms %1000 ), buf, + (int )((cookie->start_time_ns /1000000 ) %1000 ), + bs ==NULL ?"unknown" :bdrv_get_node_name (bs), + bs ==NULL ?"unknown" :bdrv_get_format_name (bs), + bs ==NULL ?"unknown" : bs->filename); +} +static void block_account_one_io (BlockAcctStats *stats,BlockAcctCookie *cookie,bool failed) {
@@ -222,6 +277,8 @@ static void block_account_one_io(BlockAcctStats *stats, BlockAcctCookie *cookie,qemu_mutex_unlock (&stats->lock); +block_acct_report_long (stats, cookie, latency_ns); + cookie->type =BLOCK_ACCT_NONE ; }
blockdev.c +6 -1
@@ -622,7 +622,8 @@ static BlockBackend *blockdev_init(const char *file, QDict *bs_opts, bs->detect_zeroes = detect_zeroes; -block_acct_setup (blk_get_stats (blk), account_invalid, account_failed); +block_acct_setup (blk_get_stats (blk), account_invalid, account_failed, +qemu_opt_get_number (opts,"latency-log-threshold" ,0 ));if (!parse_stats_intervals (blk_get_stats (blk), interval_list, errp)) {blk_unref (blk);
@@ -3727,6 +3728,10 @@ QemuOptsList qemu_common_drive_opts = { .type = QEMU_OPT_BOOL, .help = "whether to account for failed I/O operations " "in the statistics", + },{ + .name = "latency-log-threshold", + .type = QEMU_OPT_NUMBER, + .help = "threshold for long I/O report (disabled if <=0), in ms", }, {/* end of list */ } },
include/block/accounting.h +4 -1
@@ -91,6 +91,9 @@ struct BlockAcctStats {bool account_invalid;bool account_failed;BlockLatencyHistogram latency_histogram[BLOCK_MAX_IOTYPE ]; + +/* Threshold for long I/O detection, ms */ +unsigned latency_log_threshold_ms; }; typedef struct BlockAcctCookie {
@@ -101,7 +104,7 @@ typedef struct BlockAcctCookie {void block_acct_init (BlockAcctStats *stats);void block_acct_setup (BlockAcctStats *stats,bool account_invalid, -bool account_failed); +bool account_failed,unsigned latency_log_threshold_ms);void block_acct_cleanup (BlockAcctStats *stats);void block_acct_add_interval (BlockAcctStats *stats,unsigned interval_length); BlockAcctTimedStats *block_acct_interval_next(BlockAcctStats *stats,
slirp +1 -1
@@ -1 +1 @@ -Subproject commit ce94eba2042d52a0ba3d9e252ebce86715e94275 +Subproject commit 2faae0f778f818fadc873308f983289df697eb93
Signature · 1 line
2.17.1
Quoted 22 lines · Denis V. Lunev
There are severe delays with IO requests processing if QEMU is running in virtual machine or over software defined storage. Such delays potentially results in unpredictable guest behavior. For example, guests over IDE or SATA drive could remount filesystem read-only if write is performed longer than 10 seconds.
Such reports are very complex to process. Some good starting point for this seems quite reasonable. This patch provides one. It adds logging of such potentially dangerous long IO operations.
Signed-off-by Denis V. Lunev <[email protected]>Reviewed-by Vladimir Sementsov-Ogievskiy <[email protected]>CC Stefan Hajnoczi <[email protected]>CC Kevin Wolf <[email protected]>CC Max Reitz <[email protected]>--- block/accounting.c | 59 +++++++++++++++++++++++++++++++++++++- blockdev.c | 7 ++++- include/block/accounting.h | 5 +++- slirp | 2 +- 4 files changed, 69 insertions(+), 4 deletions(-)
...
Quoted 15 lines
typedef struct BlockAcctCookie {
include/block/accounting.h +1
-1@@ -101,7 +104,7 @@ typedef struct BlockAcctCookie {void block_acct_init (BlockAcctStats *stats);void block_acct_setup (BlockAcctStats *stats,bool account_invalid, -bool account_failed); +bool account_failed,unsigned latency_log_threshold_ms);void block_acct_cleanup (BlockAcctStats *stats);void block_acct_add_interval (BlockAcctStats *stats,unsigned interval_length); BlockAcctTimedStats *block_acct_interval_next(BlockAcctStats *stats,slirp +0
-0
slirp line 1
Quoted 2 lines
slirp +1
-1@@ -1 +1 @@ -Subproject commit ce94eba2042d52a0ba3d9e252ebce86715e94275 +Subproject commit 2faae0f778f818fadc873308f983289df697eb93
SLiRP change unrelated I presume...
Quoted 28 lines · Philippe Mathieu-Daudé
Quoted 22 lines · Denis V. Lunev
There are severe delays with IO requests processing if QEMU is running in virtual machine or over software defined storage. Such delays potentially results in unpredictable guest behavior. For example, guests over IDE or SATA drive could remount filesystem read-only if write is performed longer than 10 seconds.
Such reports are very complex to process. Some good starting point for this seems quite reasonable. This patch provides one. It adds logging of such potentially dangerous long IO operations.
Signed-off-by Denis V. Lunev <[email protected]>Reviewed-by Vladimir Sementsov-Ogievskiy <[email protected]>CC Stefan Hajnoczi <[email protected]>CC Kevin Wolf <[email protected]>CC Max Reitz <[email protected]>--- block/accounting.c | 59 +++++++++++++++++++++++++++++++++++++- blockdev.c | 7 ++++- include/block/accounting.h | 5 +++- slirp | 2 +- 4 files changed, 69 insertions(+), 4 deletions(-)...
Quoted 3 lines
typedef struct BlockAcctCookie {
(quoted diff) +0
-0@@ -101,7 +104,7 @@ typedef struct BlockAcctCookie {
Quoted 16 lines
Quoted 12 lines
include/block/accounting.h +1
-1void block_acct_init (BlockAcctStats *stats);void block_acct_setup (BlockAcctStats *stats,bool account_invalid, -bool account_failed); +bool account_failed,unsigned latency_log_threshold_ms);void block_acct_cleanup (BlockAcctStats *stats);void block_acct_add_interval (BlockAcctStats *stats,unsigned interval_length); BlockAcctTimedStats *block_acct_interval_next(BlockAcctStats *stats,slirp +0
-0slirp line 1
Quoted 2 lines
slirp +1
-1@@ -1 +1 @@ -Subproject commit ce94eba2042d52a0ba3d9e252ebce86715e94275 +Subproject commit 2faae0f778f818fadc873308f983289df697eb93SLiRP change unrelated I presume...
yes :(
subprojects comes into play.. I have not event changed this. Just a result from pull :((((
PATCH v2 3/3 block: enable long IO requests report by default
In reply to Denis V. LunevLatency threshold is set to 10 seconds following guest request timeout on legacy storage controller.
--- blockdev.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
blockdev.c +2 -1
@@ -623,7 +623,8 @@ static BlockBackend *blockdev_init(const char *file, QDict *bs_opts, bs->detect_zeroes = detect_zeroes;block_acct_setup (blk_get_stats (blk), account_invalid, account_failed, -qemu_opt_get_number (opts,"latency-log-threshold" ,0 )); +qemu_opt_get_number (opts,"latency-log-threshold" , +10000 /* ms */ ));if (!parse_stats_intervals (blk_get_stats (blk), interval_list, errp)) {blk_unref (blk);
Signature · 1 line
2.17.1