Skip to contentSkip to conversation

qcow2: Expose bitmaps' size during measure

PATCH Superseded Eric Blake 2 participants 5 messages Apr 16, 2020 – Apr 17, 2020 Block layer

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Eric Blake

author
#0 lore raw

PATCH 1/1 qcow2: Expose bitmaps' size during measure

It's useful to know how much space can be occupied by qcow2 persistent bitmaps, even though such metadata is unrelated to the guest-visible data. Report this value as an additional field.

---

This is independent from my 'qemu-img convert --bitmaps' series, but highly related. As an example, if I create a 100M image, then 2 persistent bitmaps, all with default cluster/granularity sizing, I now see:

$ ./qemu-img measure -f qcow2 -O qcow2 build/img.top
required size: 52756480
fully allocated size: 105185280
bitmaps size: 327680

which argues that I should allocate 52756480 + 327680 bytes prior to attempting qemu-img convert --bitmaps to a pre-sized destination.

If we like the idea, I probably need to submit a 2/1 patch adding iotest coverage of the new measurement.

 qapi/block-core.json | 15 ++++++++++-----
 block/qcow2.c        | 25 +++++++++++++++++++++++++
 qemu-img.c           |  3 +++
 3 files changed, 38 insertions(+), 5 deletions(-)
qapi/block-core.json +10 -5
index 943df1926a91..b47c6d69ba27 100644
@@ -633,18 +633,23 @@
 # efficiently so file size may be smaller than virtual disk size.
 #
 # The values are upper bounds that are guaranteed to fit the new image file.
-# Subsequent modification, such as internal snapshot or bitmap creation, may
-# require additional space and is not covered here.
+# Subsequent modification, such as internal snapshot or further bitmap
+# creation, may require additional space and is not covered here.
 #
-# @required: Size required for a new image file, in bytes.
+# @required: Size required for a new image file, in bytes, when copying just
+#            guest-visible contents.
 #
 # @fully-allocated: Image file size, in bytes, once data has been written
-#                   to all sectors.
+#                   to all sectors, when copying just guest-visible contents.
+#
+# @bitmaps: Additional size required for bitmap metadata not directly used
+#           for guest contents, when that metadata can be copied in addition
+#           to guest contents. (since 5.1)
 #
 # Since: 2.10
 ##
 { 'struct': 'BlockMeasureInfo',
-  'data': {'required': 'int', 'fully-allocated': 'int'} }
+  'data': {'required': 'int', 'fully-allocated': 'int', '*bitmaps': 'int'} }
 
 ##
 # @query-block:
block/qcow2.c +25 -0
index b524b0c53f84..eba6c2511e60 100644
@@ -4657,6 +4657,7 @@ static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs,
     PreallocMode prealloc;
     bool has_backing_file;
     bool has_luks;
+    uint64_t bitmaps_size = 0; /* size occupied by bitmaps in in_bs */
 
     /* Parse image creation options */
     cluster_size = qcow2_opt_get_cluster_size_del(opts, &local_err);
@@ -4732,6 +4733,8 @@ static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs,
 
     /* Account for input image */
     if (in_bs) {
+        BdrvDirtyBitmap *bm;
+        size_t bitmap_overhead = 0;
         int64_t ssize = bdrv_getlength(in_bs);
         if (ssize < 0) {
             error_setg_errno(&local_err, -ssize,
@@ -4739,6 +4742,26 @@ static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs,
             goto err;
         }
 
+        FOR_EACH_DIRTY_BITMAP(in_bs, bm) {
+            if (bdrv_dirty_bitmap_get_persistence(bm)) {
+                uint64_t bmsize = bdrv_dirty_bitmap_size(bm);
+                uint32_t granularity = bdrv_dirty_bitmap_granularity(bm);
+                const char *name = bdrv_dirty_bitmap_name(bm);
+                uint64_t bmclusters = DIV_ROUND_UP(bmsize / granularity
+                                                   / CHAR_BIT, cluster_size);
+
+                /* Assume the entire bitmap is allocated */
+                bitmaps_size += bmclusters * cluster_size;
+                /* Also reserve space for the bitmap table entries */
+                bitmaps_size += ROUND_UP(bmclusters * sizeof(uint64_t),
+                                         cluster_size);
+                /* Guess at contribution to bitmap directory size */
+                bitmap_overhead += ROUND_UP(strlen(name) + 24,
+                                            sizeof(uint64_t));
+            }
+        }
+        bitmaps_size += ROUND_UP(bitmap_overhead, cluster_size);
+
         virtual_size = ROUND_UP(ssize, cluster_size);
 
         if (has_backing_file) {
@@ -4795,6 +4818,8 @@ static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs,
      * still counted.
      */
     info->required = info->fully_allocated - virtual_size + required;
+    info->has_bitmaps = !!bitmaps_size;
+    info->bitmaps = bitmaps_size;
     return info;
 
 err:
qemu-img.c +3 -0
index 6541357179c2..d900bde89911 100644
@@ -5084,6 +5084,9 @@ static int img_measure(int argc, char **argv)
     if (output_format == OFORMAT_HUMAN) {
         printf("required size: %" PRIu64 "\n", info->required);
         printf("fully allocated size: %" PRIu64 "\n", info->fully_allocated);
+        if (info->has_bitmaps) {
+            printf("bitmaps size: %" PRIu64 "\n", info->bitmaps);
+        }
     } else {
         dump_json_block_measure_info(info);
     }
Signature · 1 line
2.26.1
author
#1 lore raw
Quoted 8 lines · Eric Blake

It's useful to know how much space can be occupied by qcow2 persistent bitmaps, even though such metadata is unrelated to the guest-visible data. Report this value as an additional field.

---

Per https://bugzilla.redhat.com/show_bug.cgi?id=1779904#c0, I didn't quite round up in enough places:

block/qcow2.c static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs, · line 4751

Quoted 11 lines
block/qcow2.c +7 -0
@@ -4739,6 +4742,26 @@ static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs,
              goto err;
          }
 
+        FOR_EACH_DIRTY_BITMAP(in_bs, bm) {
+            if (bdrv_dirty_bitmap_get_persistence(bm)) {
+                uint64_t bmsize = bdrv_dirty_bitmap_size(bm);
+                uint32_t granularity = bdrv_dirty_bitmap_granularity(bm);
+                const char *name = bdrv_dirty_bitmap_name(bm);
+                uint64_t bmclusters = DIV_ROUND_UP(bmsize / granularity
+                                                   / CHAR_BIT, cluster_size);

All of these divisions need to round up. For example, in an image with 512-byte clusters and granularity, and a bitmap covering 512*512*8+512 bytes (2097664), we need 2 clusters, not 1, for the bitmap itself. Fortunately, it is an edge case, and we usually have enough slop in the final round up to cluster size that most users won't trip on this.

block/qcow2.c static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs, · line 4759

Quoted 8 lines
block/qcow2.c +8 -0
+
+                /* Assume the entire bitmap is allocated */
+                bitmaps_size += bmclusters * cluster_size;
+                /* Also reserve space for the bitmap table entries */
+                bitmaps_size += ROUND_UP(bmclusters * sizeof(uint64_t),
+                                         cluster_size);
+                /* Guess at contribution to bitmap directory size */
+                bitmap_overhead += ROUND_UP(strlen(name) + 24,

And I don't like this magic number, but sizeof(Qcow2BitmapDirEntry) from qcow2-bitmap.c is a private struct not accessible here.

Quoted 5 lines
block/qcow2.c +5 -0
+                                            sizeof(uint64_t));
+            }
+        }
+        bitmaps_size += ROUND_UP(bitmap_overhead, cluster_size);
+
Signature · 3 lines
Eric Blake, Principal Software Engineer
Red Hat, Inc.           +1-919-301-3226
Virtualization:  qemu.org | libvirt.org
#2 lore raw
In reply to Eric Blake
Patchew URL: https://patchew.org/QEMU/[email protected]/

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series failed the docker-quick@centos7 build test. Please find the testing commands and their output below. If you have Docker installed, you can probably reproduce it locally.

=== TEST SCRIPT BEGIN ===
#!/bin/bash
make docker-image-centos7 V=1 NETWORK=1
time make docker-test-quick@centos7 SHOW_ENV=1 J=14 NETWORK=1 === TEST SCRIPT END ===

Not run: 259
Failures: 190
Failed 1 of 117 iotests
make: *** [check-tests/check-block.sh] Error 1
make: *** Waiting for unfinished jobs....
TEST check-qtest-aarch64: tests/qtest/qos-test
Traceback (most recent call last):
---
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run', '--label', 'com.qemu.instance.uuid=fab7467314384d429b3d07d0ac891780', '-u', '1003', '--security-opt', 'seccomp=unconfined', '--rm', '-e', 'TARGET_LIST=', '-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e', 'J=14', '-e', 'DEBUG=', '-e', 'SHOW_ENV=1', '-e', 'CCACHE_DIR=/var/tmp/ccache', '-v', '/home/patchew2/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v', '/var/tmp/patchew-tester-tmp-o6sxcgfh/src/docker-src.2020-04-16-18.34.06.7222:/var/tmp/qemu:z,ro', 'qemu:centos7', '/var/tmp/qemu/run', 'test-quick']' returned non-zero exit status 2. filter=--filter=label=com.qemu.instance.uuid=fab7467314384d429b3d07d0ac891780
make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-o6sxcgfh/src'
make: *** [docker-run-test-quick@centos7] Error 2

real 15m8.073s
user 0m8.781s

The full log is available at
http://patchew.org/logs/[email protected]/testing.docker-quick@centos7/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to [email protected]

author
#3 lore raw
Quoted 16 lines · On 4/16/20 5:49 PM, [email protected] wrote:

Patchew URL: https://patchew.org/QEMU/[email protected]/

Hi,

This series failed the docker-quick@centos7 build test. Please find the testing commands and their output below. If you have Docker installed, you can probably reproduce it locally.

=== TEST SCRIPT BEGIN ===
#!/bin/bash
make docker-image-centos7 V=1 NETWORK=1
time make docker-test-quick@centos7 SHOW_ENV=1 J=14 NETWORK=1 === TEST SCRIPT END ===

Quoted 3 lines

Not run: 259
Failures: 190
Failed 1 of 117 iotests

Hmm - the email truncated the useful part of the failure. Anyways, reading from...

Quoted 2 lines

The full log is available at
http://patchew.org/logs/[email protected]/testing.docker-quick@centos7/?type=message.

I see:

tmp/qemu-test/build/tests/qemu-iotests/190.out.bad renamed from tmp/qemu-test/src/tests/qemu-iotests/190.out +1 -0
@@ -4,6 +4,7 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2199023255552
 required size: 2199023255552
 fully allocated size: 2199023255552
+bitmaps size: 4846791580151137091
 required size: 335806464
 

which looks suspiciously like an uninitialized variable leaking through when there are no bitmaps to be measured. I'll fix it in v2.

Signature · 3 lines
Eric Blake, Principal Software Engineer
Red Hat, Inc.           +1-919-301-3226
Virtualization:  qemu.org | libvirt.org
author
#4 lore raw
Quoted 17 lines · Eric Blake
Quoted 3 lines

The full log is available at
http://patchew.org/logs/[email protected]/testing.docker-quick@centos7/?type=message.

I see:

--- /tmp/qemu-test/src/tests/qemu-iotests/190.out    2020-04-16 21:15:51.000000000 +0000
+++ /tmp/qemu-test/build/tests/qemu-iotests/190.out.bad    2020-04-16 22:45:47.504493172 +0000

(quoted diff) +1 -0
@@ -4,6 +4,7 @@
  Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2199023255552
  required size: 2199023255552
  fully allocated size: 2199023255552
+bitmaps size: 4846791580151137091
  required size: 335806464
 
Quoted 1 line

which looks suspiciously like an uninitialized variable leaking through when there are no bitmaps to be measured.  I'll fix it in v2.

Here's what I'm squashing in:

block/crypto.c +0 -0
index d577f89659fa..4e0f3ec97f0e 100644
@@ -535,7 +535,7 @@ static BlockMeasureInfo *block_crypto_measure(QemuOpts *opts,
      * Unallocated blocks are still encrypted so allocation status 

makes no
* difference to the file size.
*/
- info = g_new(BlockMeasureInfo, 1);

+    info = g_new0(BlockMeasureInfo, 1);
     info->fully_allocated = luks_payload_size + size;
     info->required = luks_payload_size + size;
     return info;
block/qcow2.c +1 -1
index eba6c2511e60..8d7a9e87fba0 100644
@@ -4808,7 +4808,7 @@ static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs,
         required = virtual_size;
     }
 
-    info = g_new(BlockMeasureInfo, 1);
+    info = g_new0(BlockMeasureInfo, 1);
     info->fully_allocated =
         qcow2_calc_prealloc_size(virtual_size, cluster_size,
                                  ctz32(refcount_bits)) + 

luks_payload_size;

block/raw-format.c +1 -1
index 93b25e1b6b0b..4bb54f4ac6c5 100644
@@ -346,7 +346,7 @@ static BlockMeasureInfo *raw_measure(QemuOpts *opts, BlockDriverState *in_bs,
                             BDRV_SECTOR_SIZE);
     }
 
-    info = g_new(BlockMeasureInfo, 1);
+    info = g_new0(BlockMeasureInfo, 1);
     info->required = required;
 
     /* Unallocated sectors count towards the file size in raw images */
Signature · 3 lines
Eric Blake, Principal Software Engineer
Red Hat, Inc.           +1-919-301-3226
Virtualization:  qemu.org | libvirt.org