Skip to contentSkip to conversation

qcow2: Implement zstd cluster compression method

PATCH Superseded Denis Plotnikov 2 participants 12 messages Mar 30, 2020 – Mar 31, 2020 Block layer

v11 of 24 Superseded Latest is v24 →

lore
Share & tools
lore
#2 lore raw

PATCH v11 2/4 qcow2: rework the cluster compression routine

In reply to Denis Plotnikov

The patch enables processing the image compression type defined for the image and chooses an appropriate method for image clusters (de)compression.

---
 block/qcow2-threads.c | 71 ++++++++++++++++++++++++++++++++++++-------
 1 file changed, 60 insertions(+), 11 deletions(-)
block/qcow2-threads.c +60 -11
index a68126f291..7dbaf53489 100644
@@ -74,7 +74,9 @@ typedef struct Qcow2CompressData {
 } Qcow2CompressData;
 
 /*
- * qcow2_compress()
+ * qcow2_zlib_compress()
+ *
+ * Compress @src_size bytes of data using zlib compression method
  *
  * @dest - destination buffer, @dest_size bytes
  * @src - source buffer, @src_size bytes
@@ -83,8 +85,8 @@ typedef struct Qcow2CompressData {
  *          -ENOMEM destination buffer is not enough to store compressed data
  *          -EIO    on any other error
  */
-static ssize_t qcow2_compress(void *dest, size_t dest_size,
-                              const void *src, size_t src_size)
+static ssize_t qcow2_zlib_compress(void *dest, size_t dest_size,
+                                   const void *src, size_t src_size)
 {
     ssize_t ret;
     z_stream strm;
@@ -119,10 +121,10 @@ static ssize_t qcow2_compress(void *dest, size_t dest_size,
 }
 
 /*
- * qcow2_decompress()
+ * qcow2_zlib_decompress()
  *
  * Decompress some data (not more than @src_size bytes) to produce exactly
- * @dest_size bytes.
+ * @dest_size bytes using zlib compression method
  *
  * @dest - destination buffer, @dest_size bytes
  * @src - source buffer, @src_size bytes
@@ -130,8 +132,8 @@ static ssize_t qcow2_compress(void *dest, size_t dest_size,
  * Returns: 0 on success
  *          -EIO on fail
  */
-static ssize_t qcow2_decompress(void *dest, size_t dest_size,
-                                const void *src, size_t src_size)
+static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size,
+                                     const void *src, size_t src_size)
 {
     int ret;
     z_stream strm;
@@ -191,20 +193,67 @@ qcow2_co_do_compress(BlockDriverState *bs, void *dest, size_t dest_size,
     return arg.ret;
 }
 
+/*
+ * qcow2_co_compress()
+ *
+ * Compress @src_size bytes of data using the compression
+ * method defined by the image compression type
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: compressed size on success
+ *          a negative error code on failure
+ */
 ssize_t coroutine_fn
 qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,
                   const void *src, size_t src_size)
 {
-    return qcow2_co_do_compress(bs, dest, dest_size, src, src_size,
-                                qcow2_compress);
+    BDRVQcow2State *s = bs->opaque;
+    Qcow2CompressFunc fn;
+
+    switch (s->compression_type) {
+    case QCOW2_COMPRESSION_TYPE_ZLIB:
+        fn = qcow2_zlib_compress;
+        break;
+
+    default:
+        abort();
+    }
+
+    return qcow2_co_do_compress(bs, dest, dest_size, src, src_size, fn);
 }
 
+/*
+ * qcow2_co_decompress()
+ *
+ * Decompress some data (not more than @src_size bytes) to produce exactly
+ * @dest_size bytes using the compression method defined by the image
+ * compression type
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: 0 on success
+ *          a negative error code on failure
+ */
 ssize_t coroutine_fn
 qcow2_co_decompress(BlockDriverState *bs, void *dest, size_t dest_size,
                     const void *src, size_t src_size)
 {
-    return qcow2_co_do_compress(bs, dest, dest_size, src, src_size,
-                                qcow2_decompress);
+    BDRVQcow2State *s = bs->opaque;
+    Qcow2CompressFunc fn;
+
+    switch (s->compression_type) {
+    case QCOW2_COMPRESSION_TYPE_ZLIB:
+        fn = qcow2_zlib_decompress;
+        break;
+
+    default:
+        abort();
+    }
+
+    return qcow2_co_do_compress(bs, dest, dest_size, src, src_size, fn);
 }
 
 
Signature · 1 line
2.17.0
#3 lore raw

PATCH v11 3/4 qcow2: add zstd cluster compression

In reply to Denis Plotnikov

zstd significantly reduces cluster compression time. It provides better compression performance maintaining the same level of the compression ratio in comparison with zlib, which, at the moment, is the only compression method available.

The performance test results:
Test compresses and decompresses qemu qcow2 image with just installed rhel-7.6 guest.
Image cluster size: 64K. Image on disk size: 2.2G

The test was conducted with brd disk to reduce the influence of disk subsystem to the test results.
The results is given in seconds.

compress cmd:
time ./qemu-img convert -O qcow2 -c -o compression_type=[zlib|zstd]
src.img [zlib|zstd]_compressed.img
decompress cmd
time ./qemu-img convert -O qcow2
[zlib|zstd]_compressed.img uncompressed.img

compression decompression
zlib zstd zlib zstd
------------------------------------------------------------

real     65.5       16.3 (-75 %)    1.9          1.6 (-16 %)
user     65.0       15.8            5.3          2.5
sys       3.3        0.2            2.0          2.0

Both ZLIB and ZSTD gave the same compression ratio: 1.57 compressed image size in both cases: 1.4G

QAPI part:

---
 docs/interop/qcow2.txt |   1 +
 configure              |   2 +-
 qapi/block-core.json   |   3 +-
 block/qcow2-threads.c  | 138 +++++++++++++++++++++++++++++++++++++++++
 block/qcow2.c          |   7 +++
 5 files changed, 149 insertions(+), 2 deletions(-)
docs/interop/qcow2.txt +1 -0
index 5597e24474..795dbb21dd 100644
@@ -208,6 +208,7 @@ version 2.
 
                     Available compression type values:
                         0: zlib <https://www.zlib.net/>
+                        1: zstd <http://github.com/facebook/zstd>
 
 
 === Header padding ===
configure +1 -1
index da09c35895..57cac2abe1 100755
@@ -1861,7 +1861,7 @@ disabled with --disable-FEATURE, default is enabled if available:
   lzfse           support of lzfse compression library
                   (for reading lzfse-compressed dmg images)
   zstd            support for zstd compression library
-                  (for migration compression)
+                  (for migration compression and qcow2 cluster compression)
   seccomp         seccomp support
   coroutine-pool  coroutine freelist (better performance)
   glusterfs       GlusterFS backend
qapi/block-core.json +2 -1
index d669ec0965..44690ed266 100644
@@ -4293,11 +4293,12 @@
 # Compression type used in qcow2 image file
 #
 # @zlib: zlib compression, see <http://zlib.net/>
+# @zstd: zstd compression, see <http://github.com/facebook/zstd>
 #
 # Since: 5.0
 ##
 { 'enum': 'Qcow2CompressionType',
-  'data': [ 'zlib' ] }
+  'data': [ 'zlib', { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } ] }
 
 ##
 # @BlockdevCreateOptionsQcow2:
block/qcow2-threads.c +138 -0

162 lines not on this page · raw message

block/qcow2.c +7 -0
index 643698934d..6632daf50b 100644
@@ -1246,6 +1246,9 @@ static int validate_compression_type(BDRVQcow2State *s, Error **errp)
 {
     switch (s->compression_type) {
     case QCOW2_COMPRESSION_TYPE_ZLIB:
+#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+#endif
         break;
 
     default:
@@ -3461,6 +3464,10 @@ qcow2_co_create(BlockdevCreateOptions *create_options, Error **errp)
         }
 
         switch (qcow2_opts->compression_type) {
+#ifdef CONFIG_ZSTD
+        case QCOW2_COMPRESSION_TYPE_ZSTD:
+            break;
+#endif
         default:
             error_setg(errp, "Unknown compression type");
             goto out;
Signature · 1 line
2.17.0
#0 lore raw

v11 cover qcow2: Implement zstd cluster compression method

v11:
* 03: the loops don't need "do{}while" form anymore and
the they were buggy (missed "do" in the beginning)
replace them with usual "while(){}" loops [Vladimir]
v10:
* 03: fix zstd (de)compressed loops for multi-frame
cases [Vladimir]
v9:
* 01: fix error checking and reporting in qcow2_amend compression type part [Vladimir]
* 03: replace asserts with -EIO in qcow2_zstd_decompression [Vladimir, Alberto]
* 03: reword/amend/add comments, fix typos [Vladimir]

v8:
* 03: switch zstd API from simple to stream [Eric] No need to state a special cluster layout for zstd compressed clusters.
v7:
* use qapi_enum_parse instead of the open-coding [Eric]
* fix wording, typos and spelling [Eric]

v6:
* "block/qcow2-threads: fix qcow2_decompress" is removed from the series since it has been accepted by Max already
* add compile time checking for Qcow2Header to be a multiple of 8 [Max, Alberto]
* report error on qcow2 amending when the compression type is actually chnged [Max]
* remove the extra space and the extra new line [Max]
* re-arrange acks and signed-off-s [Vladimir]

v5:
* replace -ENOTSUP with abort in qcow2_co_decompress [Vladimir]
* set cluster size for all test cases in the beginning of the 287 test

v4:
* the series is rebased on top of 01 "block/qcow2-threads: fix qcow2_decompress"
* 01 is just a no-change resend to avoid extra dependencies. Still, it may be merged in separate

v3:
* remove redundant max compression type value check [Vladimir, Eric] (the switch below checks everything)
* prevent compression type changing on "qemu-img amend" [Vladimir]
* remove zstd config setting, since it has been added already by "migration" patches [Vladimir]
* change the compression type error message [Vladimir]
* fix alignment and 80-chars exceeding [Vladimir]

v2:
* rework compression type setting [Vladimir]
* squash iotest changes to the compression type introduction patch [Vladimir, Eric]
* fix zstd availability checking in zstd iotest [Vladimir]
* remove unnecessry casting [Eric]
* remove rudundant checks [Eric]
* fix compressed cluster layout in qcow2 spec [Vladimir]
* fix wording [Eric, Vladimir]
* fix compression type filtering in iotests [Eric]

v1:
the initial series

Denis Plotnikov (4):
qcow2: introduce compression type feature
qcow2: rework the cluster compression routine
qcow2: add zstd cluster compression
iotests: 287: add qcow2 compression type test

 docs/interop/qcow2.txt           |   1 +
 configure                        |   2 +-
 qapi/block-core.json             |  23 +++-
 block/qcow2.h                    |  20 ++-
 include/block/block_int.h        |   1 +
 block/qcow2-threads.c            | 209 +++++++++++++++++++++++++++++--
 block/qcow2.c                    | 120 ++++++++++++++++++
 tests/qemu-iotests/031.out       |  14 +--
 tests/qemu-iotests/036.out       |   4 +-
 tests/qemu-iotests/049.out       | 102 +++++++--------
 tests/qemu-iotests/060.out       |   1 +
 tests/qemu-iotests/061.out       |  34 ++---
 tests/qemu-iotests/065           |  28 +++--
 tests/qemu-iotests/080           |   2 +-
 tests/qemu-iotests/144.out       |   4 +-
 tests/qemu-iotests/182.out       |   2 +-
 tests/qemu-iotests/242.out       |   5 +
 tests/qemu-iotests/255.out       |   8 +-
 tests/qemu-iotests/287           | 128 +++++++++++++++++++
 tests/qemu-iotests/287.out       |  43 +++++++
 tests/qemu-iotests/common.filter |   3 +-
 tests/qemu-iotests/group         |   1 +
 22 files changed, 647 insertions(+), 108 deletions(-)
 create mode 100755 tests/qemu-iotests/287
 create mode 100644 tests/qemu-iotests/287.out
Signature · 1 line
2.17.0
#11 lore raw

PATCH v11 4/4 iotests: 287: add qcow2 compression type test

The test checks fulfilling qcow2 requiriements for the compression type feature and zstd compression type operability.

---
 tests/qemu-iotests/287     | 128 +++++++++++++++++++++++++++++++++++++
 tests/qemu-iotests/287.out |  43 +++++++++++++
 tests/qemu-iotests/group   |   1 +
 3 files changed, 172 insertions(+)
 create mode 100755 tests/qemu-iotests/287
 create mode 100644 tests/qemu-iotests/287.out
tests/qemu-iotests/287 new file +128 -0
new file mode 100755
index 0000000000..49d15b3d43
@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+#
+# Test case for an image using zstd compression
+#
+# Copyright (c) 2020 Virtuozzo International GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
[email protected]
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# standard environment
+. ./common.rc
+. ./common.filter
+
+# This tests qocw2-specific low-level functionality
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+# for all the cases
+CLUSTER_SIZE=65536
+
+# Check if we can run this test.
+
+IMGOPTS='compression_type=zstd' _make_test_img 64M | grep "Invalid parameter 'zstd'" 2>&1 1>/dev/null
+
+ZSTD_SUPPORTED=$?
+
+if (($ZSTD_SUPPORTED==0)); then
+    _notrun "ZSTD is disabled"
+fi
+
+# Test: when compression is zlib the incompatible bit is unset
+echo
+echo "=== Testing compression type incompatible bit setting for zlib ==="
+echo
+
+IMGOPTS='compression_type=zlib' _make_test_img 64M
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+
+# Test: when compression differs from zlib the incompatible bit is set
+echo
+echo "=== Testing compression type incompatible bit setting for zstd ==="
+echo
+
+IMGOPTS='compression_type=zstd' _make_test_img 64M
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+
+# Test: an image can't be openned if compression type is zlib and
+#       incompatible feature compression type is set
+echo
+echo "=== Testing zlib with incompatible bit set  ==="
+echo
+
+IMGOPTS='compression_type=zlib' _make_test_img 64M
+$PYTHON qcow2.py "$TEST_IMG" set-feature-bit incompatible 3
+# to make sure the bit was actually set
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+$QEMU_IMG info "$TEST_IMG" 2>1 1>/dev/null
+if (($?==0)); then
+    echo "Error: The image openned successfully. The image must not be openned"
+fi
+
+# Test: an image can't be openned if compression type is NOT zlib and
+#       incompatible feature compression type is UNSET
+echo
+echo "=== Testing zstd with incompatible bit unset  ==="
+echo
+
+IMGOPTS='compression_type=zstd' _make_test_img 64M
+$PYTHON qcow2.py "$TEST_IMG" set-header incompatible_features 0
+# to make sure the bit was actually unset
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+$QEMU_IMG info "$TEST_IMG" 2>1 1>/dev/null
+if (($?==0)); then
+    echo "Error: The image openned successfully. The image must not be openned"
+fi
+# Test: check compression type values
+echo
+echo "=== Testing compression type values  ==="
+echo
+# zlib=0
+IMGOPTS='compression_type=zlib' _make_test_img 64M
+od -j104 -N1 -An -vtu1 "$TEST_IMG"
+
+# zstd=1
+IMGOPTS='compression_type=zstd' _make_test_img 64M
+od -j104 -N1 -An -vtu1 "$TEST_IMG"
+
+# Test: using zstd compression, write to and read from an image
+echo
+echo "=== Testing reading and writing with zstd ==="
+echo
+
+IMGOPTS='compression_type=zstd' _make_test_img 64M
+$QEMU_IO -c "write -c -P 0xAC 65536 64k " "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0xAC 65536 65536 " "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -v 131070 8 " "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -v 65534 8" "$TEST_IMG" | _filter_qemu_io
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
tests/qemu-iotests/287.out new file +43 -0

43 lines not on this page · raw message

tests/qemu-iotests/group +1 -0
index 79c6dfc85d..dacbcfc12d 100644
@@ -294,5 +294,6 @@
 283 auto quick
 284 rw
 286 rw quick
+287 auto quick
 288 quick
 289 rw quick
Signature · 1 line
2.17.0
#1 lore raw

PATCH v11 1/4 qcow2: introduce compression type feature

In reply to Denis Plotnikov

The patch adds some preparation parts for incompatible compression type feature to qcow2 allowing the use different compression methods for image clusters (de)compressing.

It is implied that the compression type is set on the image creation and can be changed only later by image conversion, thus compression type defines the only compression algorithm used for the image, and thus, for all image clusters.

The goal of the feature is to add support of other compression methods to qcow2. For example, ZSTD which is more effective on compression than ZLIB.

The default compression is ZLIB. Images created with ZLIB compression type are backward compatible with older qemu versions.

Adding of the compression type breaks a number of tests because now the compression type is reported on image creation and there are some changes in the qcow2 header in size and offsets.

The tests are fixed in the following ways:

    * filter out compression_type for many tests
    * fix header size, feature table size and backing file offset
      affected tests: 031, 036, 061, 080
      header_size +=8: 1 byte compression type
                       7 bytes padding
      feature_table += 48: incompatible feature compression type
      backing_file_offset += 56 (8 + 48 -> header_change + feature_table_change)
    * add "compression type" for test output matching when it isn't filtered
      affected tests: 049, 060, 061, 065, 144, 182, 242, 255
---
 qapi/block-core.json             |  22 +++++-
 block/qcow2.h                    |  20 +++++-
 include/block/block_int.h        |   1 +
 block/qcow2.c                    | 113 +++++++++++++++++++++++++++++++
 tests/qemu-iotests/031.out       |  14 ++--
 tests/qemu-iotests/036.out       |   4 +-
 tests/qemu-iotests/049.out       | 102 ++++++++++++++--------------
 tests/qemu-iotests/060.out       |   1 +
 tests/qemu-iotests/061.out       |  34 ++++++----
 tests/qemu-iotests/065           |  28 +++++---
 tests/qemu-iotests/080           |   2 +-
 tests/qemu-iotests/144.out       |   4 +-
 tests/qemu-iotests/182.out       |   2 +-
 tests/qemu-iotests/242.out       |   5 ++
 tests/qemu-iotests/255.out       |   8 +--
 tests/qemu-iotests/common.filter |   3 +-
 16 files changed, 267 insertions(+), 96 deletions(-)
qapi/block-core.json +20 -2
index 943df1926a..d669ec0965 100644
@@ -78,6 +78,8 @@
 #
 # @bitmaps: A list of qcow2 bitmap details (since 4.0)
 #
+# @compression-type: the image cluster compression method (since 5.0)
+#
 # Since: 1.7
 ##
 { 'struct': 'ImageInfoSpecificQCow2',
@@ -89,7 +91,8 @@
       '*corrupt': 'bool',
       'refcount-bits': 'int',
       '*encrypt': 'ImageInfoSpecificQCow2Encryption',
-      '*bitmaps': ['Qcow2BitmapInfo']
+      '*bitmaps': ['Qcow2BitmapInfo'],
+      'compression-type': 'Qcow2CompressionType'
   } }
 
 ##
@@ -4284,6 +4287,18 @@
   'data': [ 'v2', 'v3' ] }
 
 
+##
+# @Qcow2CompressionType:
+#
+# Compression type used in qcow2 image file
+#
+# @zlib: zlib compression, see <http://zlib.net/>
+#
+# Since: 5.0
+##
+{ 'enum': 'Qcow2CompressionType',
+  'data': [ 'zlib' ] }
+
 ##
 # @BlockdevCreateOptionsQcow2:
 #
@@ -4307,6 +4322,8 @@
 #                 allowed values: off, falloc, full, metadata)
 # @lazy-refcounts: True if refcounts may be updated lazily (default: off)
 # @refcount-bits: Width of reference counts in bits (default: 16)
+# @compression-type: The image cluster compression method
+#                    (default: zlib, since 5.0)
 #
 # Since: 2.12
 ##
@@ -4322,7 +4339,8 @@
             '*cluster-size':    'size',
             '*preallocation':   'PreallocMode',
             '*lazy-refcounts':  'bool',
-            '*refcount-bits':   'int' } }
+            '*refcount-bits':   'int',
+            '*compression-type':'Qcow2CompressionType' } }
 
 ##
 # @BlockdevCreateOptionsQed:
block/qcow2.h +19 -1
index 0942126232..fca2fe33f9 100644
@@ -146,8 +146,16 @@ typedef struct QCowHeader {
 
     uint32_t refcount_order;
     uint32_t header_length;
+
+    /* Additional fields */
+    uint8_t compression_type;
+
+    /* header must be a multiple of 8 */
+    uint8_t padding[7];
 } QEMU_PACKED QCowHeader;
 
+QEMU_BUILD_BUG_ON(!QEMU_IS_ALIGNED(sizeof(QCowHeader), 8));
+
 typedef struct QEMU_PACKED QCowSnapshotHeader {
     /* header is 8 byte aligned */
     uint64_t l1_table_offset;
@@ -216,13 +224,16 @@ enum {
     QCOW2_INCOMPAT_DIRTY_BITNR      = 0,
     QCOW2_INCOMPAT_CORRUPT_BITNR    = 1,
     QCOW2_INCOMPAT_DATA_FILE_BITNR  = 2,
+    QCOW2_INCOMPAT_COMPRESSION_BITNR = 3,
     QCOW2_INCOMPAT_DIRTY            = 1 << QCOW2_INCOMPAT_DIRTY_BITNR,
     QCOW2_INCOMPAT_CORRUPT          = 1 << QCOW2_INCOMPAT_CORRUPT_BITNR,
     QCOW2_INCOMPAT_DATA_FILE        = 1 << QCOW2_INCOMPAT_DATA_FILE_BITNR,
+    QCOW2_INCOMPAT_COMPRESSION      = 1 << QCOW2_INCOMPAT_COMPRESSION_BITNR,
 
     QCOW2_INCOMPAT_MASK             = QCOW2_INCOMPAT_DIRTY
                                     | QCOW2_INCOMPAT_CORRUPT
-                                    | QCOW2_INCOMPAT_DATA_FILE,
+                                    | QCOW2_INCOMPAT_DATA_FILE
+                                    | QCOW2_INCOMPAT_COMPRESSION,
 };
 
 /* Compatible feature bits */
@@ -369,6 +380,13 @@ typedef struct BDRVQcow2State {
 
     bool metadata_preallocation_checked;
     bool metadata_preallocation;
+    /*
+     * Compression type used for the image. Default: 0 - ZLIB
+     * The image compression type is set on image creation.
+     * For now, the only way to change the compression type
+     * is to convert the image with the desired compression type set.
+     */
+    Qcow2CompressionType compression_type;
 } BDRVQcow2State;
 
 typedef struct Qcow2COWRegion {
include/block/block_int.h +1 -0
index ae9c4da4d0..143b2a2308 100644
@@ -57,6 +57,7 @@
 #define BLOCK_OPT_REFCOUNT_BITS     "refcount_bits"
 #define BLOCK_OPT_DATA_FILE         "data_file"
 #define BLOCK_OPT_DATA_FILE_RAW     "data_file_raw"
+#define BLOCK_OPT_COMPRESSION_TYPE  "compression_type"
 
 #define BLOCK_PROBE_BUF_SIZE        512
 
block/qcow2.c +113 -0

191 lines not on this page · raw message

tests/qemu-iotests/031.out +7 -7
index d535e407bc..ed51afe9ce 100644
@@ -113,11 +113,11 @@ incompatible_features     []
 compatible_features       []
 autoclear_features        []
 refcount_order            4
-header_length             104
+header_length             112
 
 Header extension:
 magic                     0x6803f857
-length                    192
+length                    240
 data                      <binary>
 
 Header extension:
@@ -146,11 +146,11 @@ incompatible_features     []
 compatible_features       []
 autoclear_features        []
 refcount_order            4
-header_length             104
+header_length             112
 
 Header extension:
 magic                     0x6803f857
-length                    192
+length                    240
 data                      <binary>
 
 Header extension:
@@ -164,7 +164,7 @@ No errors were found on the image.
 
 magic                     0x514649fb
 version                   3
-backing_file_offset       0x178
+backing_file_offset       0x1b0
 backing_file_size         0x17
 cluster_bits              16
 size                      67108864
@@ -179,7 +179,7 @@ incompatible_features     []
 compatible_features       []
 autoclear_features        []
 refcount_order            4
-header_length             104
+header_length             112
 
 Header extension:
 magic                     0xe2792aca
@@ -188,7 +188,7 @@ data                      'host_device'
 
 Header extension:
 magic                     0x6803f857
-length                    192
+length                    240
 data                      <binary>
 
 Header extension:
tests/qemu-iotests/036.out +2 -2
index 0b52b934e1..fb509f6357 100644
@@ -26,7 +26,7 @@ compatible_features       []
 autoclear_features        [63]
 Header extension:
 magic                     0x6803f857
-length                    192
+length                    240
 data                      <binary>
 
 
@@ -38,7 +38,7 @@ compatible_features       []
 autoclear_features        []
 Header extension:
 magic                     0x6803f857
-length                    192
+length                    240
 data                      <binary>
 
 *** done
tests/qemu-iotests/049.out +51 -51

225 lines not on this page · raw message

tests/qemu-iotests/060.out +1 -0
index d27692a33c..3e47f537e8 100644
@@ -17,6 +17,7 @@ virtual size: 64 MiB (67108864 bytes)
 cluster_size: 65536
 Format specific information:
     compat: 1.1
+    compression type: zlib
     lazy refcounts: false
     refcount bits: 16
     corrupt: true
tests/qemu-iotests/061.out +20 -14
index 8b3091a412..c913f02ad6 100644
@@ -22,11 +22,11 @@ incompatible_features     []
 compatible_features       [0]
 autoclear_features        []
 refcount_order            4
-header_length             104
+header_length             112
 
 Header extension:
 magic                     0x6803f857
-length                    192
+length                    240
 data                      <binary>
 
 magic                     0x514649fb
@@ -80,11 +80,11 @@ incompatible_features     []
 compatible_features       [0]
 autoclear_features        []
 refcount_order            4
-header_length             104
+header_length             112
 
 Header extension:
 magic                     0x6803f857
-length                    192
+length                    240
 data                      <binary>
 
 magic                     0x514649fb
@@ -136,11 +136,11 @@ incompatible_features     [0]
 compatible_features       [0]
 autoclear_features        []
 refcount_order            4
-header_length             104
+header_length             112
 
 Header extension:
 magic                     0x6803f857
-length                    192
+length                    240
 data                      <binary>
 
 ERROR cluster 5 refcount=0 reference=1
@@ -191,11 +191,11 @@ incompatible_features     []
 compatible_features       [42]
 autoclear_features        [42]
 refcount_order            4
-header_length             104
+header_length             112
 
 Header extension:
 magic                     0x6803f857
-length                    192
+length                    240
 data                      <binary>
 
 magic                     0x514649fb
@@ -260,11 +260,11 @@ incompatible_features     []
 compatible_features       [0]
 autoclear_features        []
 refcount_order            4
-header_length             104
+header_length             112
 
 Header extension:
 magic                     0x6803f857
-length                    192
+length                    240
 data                      <binary>
 
 read 65536/65536 bytes at offset 44040192
@@ -294,11 +294,11 @@ incompatible_features     [0]
 compatible_features       [0]
 autoclear_features        []
 refcount_order            4
-header_length             104
+header_length             112
 
 Header extension:
 magic                     0x6803f857
-length                    192
+length                    240
 data                      <binary>
 
 ERROR cluster 5 refcount=0 reference=1
@@ -323,11 +323,11 @@ incompatible_features     []
 compatible_features       []
 autoclear_features        []
 refcount_order            4
-header_length             104
+header_length             112
 
 Header extension:
 magic                     0x6803f857
-length                    192
+length                    240
 data                      <binary>
 
 read 131072/131072 bytes at offset 0
@@ -491,6 +491,7 @@ virtual size: 64 MiB (67108864 bytes)
 cluster_size: 65536
 Format specific information:
     compat: 1.1
+    compression type: zlib
     lazy refcounts: false
     refcount bits: 16
     data file: TEST_DIR/t.IMGFMT.data
@@ -511,6 +512,7 @@ virtual size: 64 MiB (67108864 bytes)
 cluster_size: 65536
 Format specific information:
     compat: 1.1
+    compression type: zlib
     lazy refcounts: false
     refcount bits: 16
     data file: foo
@@ -524,6 +526,7 @@ virtual size: 64 MiB (67108864 bytes)
 cluster_size: 65536
 Format specific information:
     compat: 1.1
+    compression type: zlib
     lazy refcounts: false
     refcount bits: 16
     data file raw: false
@@ -538,6 +541,7 @@ virtual size: 64 MiB (67108864 bytes)
 cluster_size: 65536
 Format specific information:
     compat: 1.1
+    compression type: zlib
     lazy refcounts: false
     refcount bits: 16
     data file: TEST_DIR/t.IMGFMT.data
@@ -550,6 +554,7 @@ virtual size: 64 MiB (67108864 bytes)
 cluster_size: 65536
 Format specific information:
     compat: 1.1
+    compression type: zlib
     lazy refcounts: false
     refcount bits: 16
     data file: TEST_DIR/t.IMGFMT.data
@@ -563,6 +568,7 @@ virtual size: 64 MiB (67108864 bytes)
 cluster_size: 65536
 Format specific information:
     compat: 1.1
+    compression type: zlib
     lazy refcounts: false
     refcount bits: 16
     data file: TEST_DIR/t.IMGFMT.data
tests/qemu-iotests/065 +18 -10
index 6426474271..18dc488c7a 100755
@@ -88,24 +88,30 @@ class TestQMP(TestImageInfoSpecific):
 class TestQCow2(TestQemuImgInfo):
     '''Testing a qcow2 version 2 image'''
     img_options = 'compat=0.10'
-    json_compare = { 'compat': '0.10', 'refcount-bits': 16 }
-    human_compare = [ 'compat: 0.10', 'refcount bits: 16' ]
+    json_compare = { 'compat': '0.10', 'refcount-bits': 16,
+                     'compression-type': 'zlib' }
+    human_compare = [ 'compat: 0.10', 'compression type: zlib',
+                      'refcount bits: 16' ]
 
 class TestQCow3NotLazy(TestQemuImgInfo):
     '''Testing a qcow2 version 3 image with lazy refcounts disabled'''
     img_options = 'compat=1.1,lazy_refcounts=off'
     json_compare = { 'compat': '1.1', 'lazy-refcounts': False,
-                     'refcount-bits': 16, 'corrupt': False }
-    human_compare = [ 'compat: 1.1', 'lazy refcounts: false',
-                      'refcount bits: 16', 'corrupt: false' ]
+                     'refcount-bits': 16, 'corrupt': False,
+                     'compression-type': 'zlib' }
+    human_compare = [ 'compat: 1.1', 'compression type: zlib',
+                      'lazy refcounts: false', 'refcount bits: 16',
+                      'corrupt: false' ]
 
 class TestQCow3Lazy(TestQemuImgInfo):
     '''Testing a qcow2 version 3 image with lazy refcounts enabled'''
     img_options = 'compat=1.1,lazy_refcounts=on'
     json_compare = { 'compat': '1.1', 'lazy-refcounts': True,
-                     'refcount-bits': 16, 'corrupt': False }
-    human_compare = [ 'compat: 1.1', 'lazy refcounts: true',
-                      'refcount bits: 16', 'corrupt: false' ]
+                     'refcount-bits': 16, 'corrupt': False,
+                     'compression-type': 'zlib' }
+    human_compare = [ 'compat: 1.1', 'compression type: zlib',
+                      'lazy refcounts: true', 'refcount bits: 16',
+                      'corrupt: false' ]
 
 class TestQCow3NotLazyQMP(TestQMP):
     '''Testing a qcow2 version 3 image with lazy refcounts disabled, opening
@@ -113,7 +119,8 @@ class TestQCow3NotLazyQMP(TestQMP):
     img_options = 'compat=1.1,lazy_refcounts=off'
     qemu_options = 'lazy-refcounts=on'
     compare = { 'compat': '1.1', 'lazy-refcounts': False,
-                'refcount-bits': 16, 'corrupt': False }
+                'refcount-bits': 16, 'corrupt': False,
+                'compression-type': 'zlib' }
 
 
 class TestQCow3LazyQMP(TestQMP):
@@ -122,7 +129,8 @@ class TestQCow3LazyQMP(TestQMP):
     img_options = 'compat=1.1,lazy_refcounts=on'
     qemu_options = 'lazy-refcounts=off'
     compare = { 'compat': '1.1', 'lazy-refcounts': True,
-                'refcount-bits': 16, 'corrupt': False }
+                'refcount-bits': 16, 'corrupt': False,
+                'compression-type': 'zlib' }
 
 TestImageInfoSpecific = None
 TestQemuImgInfo = None
tests/qemu-iotests/080 +1 -1
index a3d13c414e..7588c63b6c 100755
@@ -45,7 +45,7 @@ _supported_os Linux
 # - This is generally a test for compat=1.1 images
 _unsupported_imgopts 'refcount_bits=1[^0-9]' data_file 'compat=0.10'
 
-header_size=104
+header_size=112
 
 offset_backing_file_offset=8
 offset_backing_file_size=16
tests/qemu-iotests/144.out +2 -2
index c7aa2e4820..885a8874a5 100644
@@ -9,7 +9,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=536870912
 { 'execute': 'qmp_capabilities' }
 {"return": {}}
 { 'execute': 'blockdev-snapshot-sync', 'arguments': { 'device': 'virtio0', 'snapshot-file':'TEST_DIR/tmp.IMGFMT', 'format': 'IMGFMT' } }
-Formatting 'TEST_DIR/tmp.qcow2', fmt=qcow2 size=536870912 backing_file=TEST_DIR/t.qcow2 backing_fmt=qcow2 cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/tmp.qcow2', fmt=qcow2 size=536870912 backing_file=TEST_DIR/t.qcow2 backing_fmt=qcow2 cluster_size=65536 lazy_refcounts=off refcount_bits=16 compression_type=zlib
 {"return": {}}
 
 === Performing block-commit on active layer ===
@@ -31,6 +31,6 @@ Formatting 'TEST_DIR/tmp.qcow2', fmt=qcow2 size=536870912 backing_file=TEST_DIR/
 === Performing Live Snapshot 2 ===
 
 { 'execute': 'blockdev-snapshot-sync', 'arguments': { 'device': 'virtio0', 'snapshot-file':'TEST_DIR/tmp2.IMGFMT', 'format': 'IMGFMT' } }
-Formatting 'TEST_DIR/tmp2.qcow2', fmt=qcow2 size=536870912 backing_file=TEST_DIR/t.qcow2 backing_fmt=qcow2 cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/tmp2.qcow2', fmt=qcow2 size=536870912 backing_file=TEST_DIR/t.qcow2 backing_fmt=qcow2 cluster_size=65536 lazy_refcounts=off refcount_bits=16 compression_type=zlib
 {"return": {}}
 *** done
tests/qemu-iotests/182.out +1 -1
index a8eea166c3..ae43654d32 100644
@@ -13,7 +13,7 @@ Is another process using the image [TEST_DIR/t.qcow2]?
 {'execute': 'blockdev-add', 'arguments': { 'node-name': 'node0', 'driver': 'file', 'filename': 'TEST_DIR/t.IMGFMT', 'locking': 'on' } }
 {"return": {}}
 {'execute': 'blockdev-snapshot-sync', 'arguments': { 'node-name': 'node0', 'snapshot-file': 'TEST_DIR/t.IMGFMT.overlay', 'snapshot-node-name': 'node1' } }
-Formatting 'TEST_DIR/t.qcow2.overlay', fmt=qcow2 size=197120 backing_file=TEST_DIR/t.qcow2 backing_fmt=file cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/t.qcow2.overlay', fmt=qcow2 size=197120 backing_file=TEST_DIR/t.qcow2 backing_fmt=file cluster_size=65536 lazy_refcounts=off refcount_bits=16 compression_type=zlib
 {"return": {}}
 {'execute': 'blockdev-add', 'arguments': { 'node-name': 'node1', 'driver': 'file', 'filename': 'TEST_DIR/t.IMGFMT', 'locking': 'on' } }
 {"return": {}}
tests/qemu-iotests/242.out +5 -0
index 7ac8404d11..091b9126ce 100644
@@ -12,6 +12,7 @@ virtual size: 1 MiB (1048576 bytes)
 cluster_size: 65536
 Format specific information:
     compat: 1.1
+    compression type: zlib
     lazy refcounts: false
     refcount bits: 16
     corrupt: false
@@ -32,6 +33,7 @@ virtual size: 1 MiB (1048576 bytes)
 cluster_size: 65536
 Format specific information:
     compat: 1.1
+    compression type: zlib
     lazy refcounts: false
     bitmaps:
         [0]:
@@ -64,6 +66,7 @@ virtual size: 1 MiB (1048576 bytes)
 cluster_size: 65536
 Format specific information:
     compat: 1.1
+    compression type: zlib
     lazy refcounts: false
     bitmaps:
         [0]:
@@ -104,6 +107,7 @@ virtual size: 1 MiB (1048576 bytes)
 cluster_size: 65536
 Format specific information:
     compat: 1.1
+    compression type: zlib
     lazy refcounts: false
     bitmaps:
         [0]:
@@ -153,6 +157,7 @@ virtual size: 1 MiB (1048576 bytes)
 cluster_size: 65536
 Format specific information:
     compat: 1.1
+    compression type: zlib
     lazy refcounts: false
     bitmaps:
         [0]:
tests/qemu-iotests/255.out +4 -4
index 348909fdef..a3c99fd62e 100644
@@ -3,9 +3,9 @@ Finishing a commit job with background reads
 
 === Create backing chain and start VM ===
 
-Formatting 'TEST_DIR/PID-t.qcow2.mid', fmt=qcow2 size=134217728 cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/PID-t.qcow2.mid', fmt=qcow2 size=134217728 cluster_size=65536 lazy_refcounts=off refcount_bits=16 compression_type=zlib
 
-Formatting 'TEST_DIR/PID-t.qcow2', fmt=qcow2 size=134217728 cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/PID-t.qcow2', fmt=qcow2 size=134217728 cluster_size=65536 lazy_refcounts=off refcount_bits=16 compression_type=zlib
 
 === Start background read requests ===
 
@@ -23,9 +23,9 @@ Closing the VM while a job is being cancelled
 
 === Create images and start VM ===
 
-Formatting 'TEST_DIR/PID-src.qcow2', fmt=qcow2 size=134217728 cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/PID-src.qcow2', fmt=qcow2 size=134217728 cluster_size=65536 lazy_refcounts=off refcount_bits=16 compression_type=zlib
 
-Formatting 'TEST_DIR/PID-dst.qcow2', fmt=qcow2 size=134217728 cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/PID-dst.qcow2', fmt=qcow2 size=134217728 cluster_size=65536 lazy_refcounts=off refcount_bits=16 compression_type=zlib
 
 wrote 1048576/1048576 bytes at offset 0
 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
tests/qemu-iotests/common.filter +2 -1
index 3f8ee3e5f7..279e0bbb0d 100644
@@ -152,7 +152,8 @@ _filter_img_create()
         -e "s# refcount_bits=[0-9]\\+##g" \
         -e "s# key-secret=[a-zA-Z0-9]\\+##g" \
         -e "s# iter-time=[0-9]\\+##g" \
-        -e "s# force_size=\\(on\\|off\\)##g"
+        -e "s# force_size=\\(on\\|off\\)##g" \
+        -e "s# compression_type=[a-zA-Z0-9]\\+##g"
 }
 
 _filter_img_info()
Signature · 1 line
2.17.0
#4 lore raw

Re: [PATCH v11 3/4] qcow2: add zstd cluster compression

In reply to Denis Plotnikov
Quoted 133 lines · Denis Plotnikov

zstd significantly reduces cluster compression time. It provides better compression performance maintaining the same level of the compression ratio in comparison with zlib, which, at the moment, is the only compression method available.

The performance test results:
Test compresses and decompresses qemu qcow2 image with just installed rhel-7.6 guest.
Image cluster size: 64K. Image on disk size: 2.2G

The test was conducted with brd disk to reduce the influence of disk subsystem to the test results.
The results is given in seconds.

compress cmd:
time ./qemu-img convert -O qcow2 -c -o compression_type=[zlib|zstd]
src.img [zlib|zstd]_compressed.img
decompress cmd
time ./qemu-img convert -O qcow2
[zlib|zstd]_compressed.img uncompressed.img

compression decompression
zlib zstd zlib zstd
------------------------------------------------------------

real     65.5       16.3 (-75 %)    1.9          1.6 (-16 %)
user     65.0       15.8            5.3          2.5
sys       3.3        0.2            2.0          2.0

Both ZLIB and ZSTD gave the same compression ratio: 1.57 compressed image size in both cases: 1.4G

QAPI part:

---
  docs/interop/qcow2.txt |   1 +
  configure              |   2 +-
  qapi/block-core.json   |   3 +-
  block/qcow2-threads.c  | 138 +++++++++++++++++++++++++++++++++++++++++
  block/qcow2.c          |   7 +++
  5 files changed, 149 insertions(+), 2 deletions(-)
docs/interop/qcow2.txt +1 -0
index 5597e24474..795dbb21dd 100644
@@ -208,6 +208,7 @@ version 2.
 
                      Available compression type values:
                          0: zlib <https://www.zlib.net/>
+                        1: zstd <http://github.com/facebook/zstd>
 
 
  === Header padding ===
configure +1 -1
index da09c35895..57cac2abe1 100755
@@ -1861,7 +1861,7 @@ disabled with --disable-FEATURE, default is enabled if available:
    lzfse           support of lzfse compression library
                    (for reading lzfse-compressed dmg images)
    zstd            support for zstd compression library
-                  (for migration compression)
+                  (for migration compression and qcow2 cluster compression)
    seccomp         seccomp support
    coroutine-pool  coroutine freelist (better performance)
    glusterfs       GlusterFS backend
qapi/block-core.json +2 -1
index d669ec0965..44690ed266 100644
@@ -4293,11 +4293,12 @@
  # Compression type used in qcow2 image file
  #
  # @zlib: zlib compression, see <http://zlib.net/>
+# @zstd: zstd compression, see <http://github.com/facebook/zstd>
  #
  # Since: 5.0
  ##
  { 'enum': 'Qcow2CompressionType',
-  'data': [ 'zlib' ] }
+  'data': [ 'zlib', { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } ] }
 
  ##
  # @BlockdevCreateOptionsQcow2:
block/qcow2-threads.c +32 -0
index 7dbaf53489..b8ccd97009 100644
@@ -28,6 +28,11 @@
  #define ZLIB_CONST
  #include <zlib.h>
 
+#ifdef CONFIG_ZSTD
+#include <zstd.h>
+#include <zstd_errors.h>
+#endif
+
  #include "qcow2.h"
  #include "block/thread-pool.h"
  #include "crypto.h"
@@ -166,6 +171,129 @@ static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size,
      return ret;
  }
 
+#ifdef CONFIG_ZSTD
+
+/*
+ * qcow2_zstd_compress()
+ *
+ * Compress @src_size bytes of data using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: compressed size on success
+ *          -ENOMEM destination buffer is not enough to store compressed data
+ *          -EIO    on any other error
+ */
+static ssize_t qcow2_zstd_compress(void *dest, size_t dest_size,
+                                   const void *src, size_t src_size)
+{
+    size_t ret;
+    ZSTD_outBuffer output = { dest, dest_size, 0 };
+    ZSTD_inBuffer input = { src, src_size, 0 };
+    ZSTD_CCtx *cctx = ZSTD_createCCtx();
+
+    if (!cctx) {
+        return -EIO;
+    }
+    /*
+     * ZSTD spec: "You must continue calling ZSTD_compressStream2()

block/qcow2-threads.c static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size, · line 210

Quoted 10 lines
block/qcow2-threads.c +10 -0
+     * with ZSTD_e_end until it returns 0, at which point you are
+     * free to start a new frame".
+     * In the loop, we try to compress all the data into one zstd frame.
+     * ZSTD_compressStream2 can potentially finish a frame earlier
+     * than the full input data is consumed. That's why we are looping
+     * until all the input data is consumed.
+     */
+    while (input.pos < input.size) {
+        /*
+         * zstd simple interface requires the exact compressed size.

on decompression you mean

block/qcow2-threads.c static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size, · line 212

Quoted 2 lines
block/qcow2-threads.c +2 -0
+         * zstd stream interface reads the comressed size from
+         * the compressed stream frame.

compressed size is not stored in the stream. I think, that streamed interface just decompress until it have something to decompress and have space to write output.

block/qcow2-threads.c static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size, · line 214

Quoted 2 lines
block/qcow2-threads.c +2 -0
+         * Instruct zstd to compress the whole buffer and write
+         * the frame which includes the compressed size.

I think this is wrong. compression size is not written.

block/qcow2-threads.c static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size, · line 217

Quoted 3 lines
block/qcow2-threads.c +3 -0
+         * This allows as to use zstd streaming semantics and
+         * don't store the compressed size for the zstd decompression.
+         */

Comment is just outdated. Accordingly to our off-list discussion, I'd rewrite it like this:

We want to use streamed interface on decompression, as we will not know exact size of compression data. Use streamed interface for compression just for symmetry.

block/qcow2-threads.c static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size, · line 227

Quoted 10 lines
block/qcow2-threads.c +10 -0
+        ret = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
+        if (ZSTD_isError(ret)) {
+            ret = -EIO;
+            goto out;
+        }
+        /* Dest buffer isn't big enough to store compressed content */
+        if (output.pos + ret > output.size) {
+            ret = -ENOMEM;
+            goto out;
+        }

Here you use "@return provides a minimum amount of data remaining to be flushed from internal buffers
or an error code, which can be tested using ZSTD_isError()."

I think we can safely drop this check, and wait for error from next ZSTD_compressStream2.. But it should work anyway.

block/qcow2-threads.c static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size, · line 231

Quoted 4 lines
block/qcow2-threads.c +4 -0
+    };
+
+    /* make sure we can safely return compressed buffer size with ssize_t */
+    assert(output.pos <= SSIZE_MAX);

Hmm. I hope it's not possible for cluster. But taking the function in separate, it _is_ possible. So may be better to assert at function start that

  assert(dest_size <= SSIZE_MAX);

To stress, that this limitation is part of qcow2_zstd_compress() interface.

Quoted 48 lines
block/qcow2-threads.c +48 -0
+    ret = output.pos;
+
+out:
+    ZSTD_freeCCtx(cctx);
+    return ret;
+}
+
+/*
+ * qcow2_zstd_decompress()
+ *
+ * Decompress some data (not more than @src_size bytes) to produce exactly
+ * @dest_size bytes using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: 0 on success
+ *          -EIO on any error
+ */
+static ssize_t qcow2_zstd_decompress(void *dest, size_t dest_size,
+                                     const void *src, size_t src_size)
+{
+    size_t ret = 0;
+    ZSTD_outBuffer output = { dest, dest_size, 0 };
+    ZSTD_inBuffer input = { src, src_size, 0 };
+    ZSTD_DCtx *dctx = ZSTD_createDCtx();
+
+    if (!dctx) {
+        return -EIO;
+    }
+
+    /*
+     * The compressed stream from input buffer may consist from more
+     * than one zstd frames. So we iterate until we get a fully
+     * uncompressed cluster.
+     */
+    while (output.pos < output.size) {
+        ret = ZSTD_decompressStream(dctx, &output, &input);
+        /*
+         * if we don't fully populate the output but have read
+         * all the frames from the input, we end up with error
+         * here
+         */
+        if (ZSTD_isError(ret)) {
+            ret = -EIO;
+            break;
+        }
+        /*

block/qcow2-threads.c static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size, · line 289

Quoted 10 lines
block/qcow2-threads.c +10 -0
+         * Dest buffer size is the image cluster size.
+         * It should be big enough to store uncompressed content.
+         * There shouldn't be any cases when the decompressed content
+         * size is greater then the cluster size, except cluster
+         * damaging.
+         */
+        if (output.pos + ret > output.size) {
+            ret = -EIO;
+            break;
+        }

But here, you use
"
or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
the return value is a suggested next input size (just a hint for better latency) that will never request more than the remaining frame size.
"

I'm afraid that "just a hint" is not safe API to make a conclusion from. So, I'd prefer to drop this optimization and just wait for an error from next ZSTD_decompressStream.

And therefore, for symmetry drop similar optimization on compression part..

What do you think?

Quoted 60 lines
block/qcow2-threads.c +17 -0
+    }
+
+    ZSTD_freeDCtx(dctx);
+    return ret;
+}
+#endif
+
  static int qcow2_compress_pool_func(void *opaque)
  {
      Qcow2CompressData *data = opaque;
@@ -217,6 +345,11 @@ qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,
          fn = qcow2_zlib_compress;
          break;
 
+#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+        fn = qcow2_zstd_compress;
+        break;
+#endif
      default:
          abort();
      }
@@ -249,6 +382,11 @@ qcow2_co_decompress(BlockDriverState *bs, void *dest, size_t dest_size,
          fn = qcow2_zlib_decompress;
          break;
 
+#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+        fn = qcow2_zstd_decompress;
+        break;
+#endif
      default:
          abort();
      }
block/qcow2.c +7 -0
index 643698934d..6632daf50b 100644
@@ -1246,6 +1246,9 @@ static int validate_compression_type(BDRVQcow2State *s, Error **errp)
  {
      switch (s->compression_type) {
      case QCOW2_COMPRESSION_TYPE_ZLIB:
+#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+#endif
          break;
 
      default:
@@ -3461,6 +3464,10 @@ qcow2_co_create(BlockdevCreateOptions *create_options, Error **errp)
          }
 
          switch (qcow2_opts->compression_type) {
+#ifdef CONFIG_ZSTD
+        case QCOW2_COMPRESSION_TYPE_ZSTD:
+            break;
+#endif
          default:
              error_setg(errp, "Unknown compression type");
              goto out;
Signature · 2 lines
Best regards,
Vladimir
#5 lore raw

Re: [PATCH v11 3/4] qcow2: add zstd cluster compression

Quoted 150 lines · Vladimir Sementsov-Ogievskiy
Quoted 135 lines · Denis Plotnikov

zstd significantly reduces cluster compression time. It provides better compression performance maintaining the same level of the compression ratio in comparison with zlib, which, at the moment, is the only compression method available.

The performance test results:
Test compresses and decompresses qemu qcow2 image with just installed rhel-7.6 guest.
Image cluster size: 64K. Image on disk size: 2.2G

The test was conducted with brd disk to reduce the influence of disk subsystem to the test results.
The results is given in seconds.

compress cmd:
   time ./qemu-img convert -O qcow2 -c -o compression_type=[zlib|zstd] src.img [zlib|zstd]_compressed.img decompress cmd
   time ./qemu-img convert -O qcow2 [zlib|zstd]_compressed.img uncompressed.img

            compression               decompression zlib       zstd           zlib         zstd
------------------------------------------------------------
real     65.5       16.3 (-75 %)    1.9          1.6 (-16 %) user     65.0       15.8            5.3          2.5 sys       3.3        0.2            2.0          2.0

Both ZLIB and ZSTD gave the same compression ratio: 1.57 compressed image size in both cases: 1.4G

QAPI part:

---
  docs/interop/qcow2.txt |   1 +
  configure              |   2 +-
  qapi/block-core.json   |   3 +-
  block/qcow2-threads.c  | 138 +++++++++++++++++++++++++++++++++++++++++ block/qcow2.c          |   7 +++

  5 files changed, 149 insertions(+), 2 deletions(-)
docs/interop/qcow2.txt +0 -0
index 5597e24474..795dbb21dd 100644
@@ -208,6 +208,7 @@ version 2.

                        Available compression type values:
                          0: zlib <https://www.zlib.net/>
+                        1: zstd <http://github.com/facebook/zstd>
      === Header padding ===

configure +0 -0
index da09c35895..57cac2abe1 100755
@@ -1861,7 +1861,7 @@ disabled with --disable-FEATURE, default is 

enabled if available:
    lzfse           support of lzfse compression library (for reading lzfse-compressed dmg images) zstd            support for zstd compression library -                  (for migration compression)
+                  (for migration compression and qcow2 cluster compression)
    seccomp         seccomp support
    coroutine-pool  coroutine freelist (better performance) glusterfs       GlusterFS backend

qapi/block-core.json +2 -1
index d669ec0965..44690ed266 100644
@@ -4293,11 +4293,12 @@
   # Compression type used in qcow2 image file
   #
   # @zlib: zlib compression, see <http://zlib.net/>
+# @zstd: zstd compression, see <http://github.com/facebook/zstd>
   #
   # Since: 5.0
   ##
   { 'enum': 'Qcow2CompressionType',
-  'data': [ 'zlib' ] }
+  'data': [ 'zlib', { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } 
 ] }
     ##
   # @BlockdevCreateOptionsQcow2:
block/qcow2-threads.c +0 -0
index 7dbaf53489..b8ccd97009 100644
@@ -28,6 +28,11 @@
   #define ZLIB_CONST
   #include <zlib.h>

  +#ifdef CONFIG_ZSTD

block/qcow2-threads.c +4 -0
+#include <zstd.h>
+#include <zstd_errors.h>
+#endif
+

  #include "qcow2.h"
  #include "block/thread-pool.h"
  #include "crypto.h"

block/qcow2-threads.c +0 -0
@@ -166,6 +171,129 @@ static ssize_t qcow2_zlib_decompress(void 
 *dest, size_t dest_size,
       return ret;
   }

  +#ifdef CONFIG_ZSTD

block/qcow2-threads.c +11 -0
+
+/*
+ * qcow2_zstd_compress()
+ *
+ * Compress @src_size bytes of data using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: compressed size on success
+ *          -ENOMEM destination buffer is not enough to store 

compressed data
+ *          -EIO    on any other error
+ */

block/qcow2-threads.c +16 -0
+static ssize_t qcow2_zstd_compress(void *dest, size_t dest_size,
+                                   const void *src, size_t src_size)
+{
+    size_t ret;
+    ZSTD_outBuffer output = { dest, dest_size, 0 };
+    ZSTD_inBuffer input = { src, src_size, 0 };
+    ZSTD_CCtx *cctx = ZSTD_createCCtx();
+
+    if (!cctx) {
+        return -EIO;
+    }
+    /*
+     * ZSTD spec: "You must continue calling ZSTD_compressStream2()
+     * with ZSTD_e_end until it returns 0, at which point you are
+     * free to start a new frame".
+     * In the loop, we try to compress all the data into one zstd 
Quoted 8 lines

frame.
+     * ZSTD_compressStream2 can potentially finish a frame earlier +     * than the full input data is consumed. That's why we are looping +     * until all the input data is consumed.
+     */
+    while (input.pos < input.size) {
+        /*
+         * zstd simple interface requires the exact compressed size.

on decompression you mean

Quoted 2 lines

+         * zstd stream interface reads the comressed size from +         * the compressed stream frame.

Quoted 8 lines

compressed size is not stored in the stream. I think, that streamed interface just decompress until it have something to decompress and have space to write output.

Quoted 2 lines

+         * Instruct zstd to compress the whole buffer and write +         * the frame which includes the compressed size.

I think this is wrong. compression size is not written.

Ok

Quoted 10 lines
Quoted 4 lines

+         * This allows as to use zstd streaming semantics and +         * don't store the compressed size for the zstd decompression. +         */

Comment is just outdated. Accordingly to our off-list discussion, I'd rewrite it like this:

We want to use streamed interface on decompression, as we will not know exact size of compression data.

This one is better then mine.

Quoted 2 lines

Use streamed interface for compression
just for symmetry.

I think this one is exceeding. If we have stream family functions on both sides this won't rise any questions from the reader.

Quoted 13 lines
Quoted 12 lines

+        ret = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end); +        if (ZSTD_isError(ret)) {
+            ret = -EIO;
+            goto out;
+        }
+        /* Dest buffer isn't big enough to store compressed content */ +        if (output.pos + ret > output.size) {
+            ret = -ENOMEM;
+            goto out;
+        }

Quoted 4 lines

Here you use "@return provides a minimum amount of data remaining to be flushed from internal buffers or an error code, which can be tested using ZSTD_isError()."

I think we can safely drop this check

No, we can't.
we should check for if zstd managed to write the stream correctly. ZSTD_compressStream2 may consume all the input buffer but return ret > 0 meaning that it needs more space.

This is from my tests:

(gdb) p ret
$1 = 11
(gdb) p input
$2 = {src = 0x562305536000, size = 65536, pos = 65536} (gdb) p output
$3 = {dst = 0x562305546000, size = 65535, pos = 65535}

Alternatively, we can replace the check with something like

if (ret != 0) {
    ret = -ENOMEM;
}

after the loop, but I think both are equivalent, so I would stick with my one :)))

Quoted 8 lines

work anyway.

Quoted 5 lines

+    };
+
+    /* make sure we can safely return compressed buffer size with ssize_t */
+    assert(output.pos <= SSIZE_MAX);

Quoted 6 lines

Hmm. I hope it's not possible for cluster. But taking the function in separate, it _is_ possible. So may be better to assert at function start that

  assert(dest_size <= SSIZE_MAX);

To stress, that this limitation is part of qcow2_zstd_compress() interface.

Agree, this is better

Quoted 68 lines
Quoted 50 lines

+    ret = output.pos;

block/qcow2-threads.c +10 -0
+
+out:
+    ZSTD_freeCCtx(cctx);
+    return ret;
+}
+
+/*
+ * qcow2_zstd_decompress()
+ *
+ * Decompress some data (not more than @src_size bytes) to produce 

exactly
+ * @dest_size bytes using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: 0 on success
+ *          -EIO on any error
+ */

block/qcow2-threads.c +29 -0
+static ssize_t qcow2_zstd_decompress(void *dest, size_t dest_size,
+                                     const void *src, size_t src_size)
+{
+    size_t ret = 0;
+    ZSTD_outBuffer output = { dest, dest_size, 0 };
+    ZSTD_inBuffer input = { src, src_size, 0 };
+    ZSTD_DCtx *dctx = ZSTD_createDCtx();
+
+    if (!dctx) {
+        return -EIO;
+    }
+
+    /*
+     * The compressed stream from input buffer may consist from more
+     * than one zstd frames. So we iterate until we get a fully
+     * uncompressed cluster.
+     */
+    while (output.pos < output.size) {
+        ret = ZSTD_decompressStream(dctx, &output, &input);
+        /*
+         * if we don't fully populate the output but have read
+         * all the frames from the input, we end up with error
+         * here
+         */
+        if (ZSTD_isError(ret)) {
+            ret = -EIO;
+            break;
+        }
+        /*

block/qcow2-threads.c static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size, · line 289

Quoted 10 lines
block/qcow2-threads.c +10 -0
+         * Dest buffer size is the image cluster size.
+         * It should be big enough to store uncompressed content.
+         * There shouldn't be any cases when the decompressed content
+         * size is greater then the cluster size, except cluster
+         * damaging.
+         */
+        if (output.pos + ret > output.size) {
+            ret = -EIO;
+            break;
+        }

But here, you use
"
or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
                                the return value is a suggested next input size (just a hint for better latency) that will never request more than the remaining frame size.
"

Quoted 6 lines

I'm afraid that "just a hint" is not safe API to make a conclusion from. So, I'd prefer to drop this optimization and just wait for an error from next ZSTD_decompressStream.

And therefore, for symmetry drop similar optimization on compression part..

What do you think?

I'd add some kind of check that we have finished with ret==0 (after loop). It looks like this is the only case when we can be sure that everything went ok.

Quoted 64 lines
Quoted 62 lines

+    }

block/qcow2-threads.c +6 -0
+
+    ZSTD_freeDCtx(dctx);
+    return ret;
+}
+#endif
+

  static int qcow2_compress_pool_func(void *opaque) {
      Qcow2CompressData *data = opaque;

block/qcow2-threads.c +0 -0
@@ -217,6 +345,11 @@ qcow2_co_compress(BlockDriverState *bs, void 
 *dest, size_t dest_size,
           fn = qcow2_zlib_compress;
           break;

  +#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+        fn = qcow2_zstd_compress;
+        break;
+#endif
      default:
          abort();
      }

block/qcow2-threads.c +0 -0
@@ -249,6 +382,11 @@ qcow2_co_decompress(BlockDriverState *bs, void 
 *dest, size_t dest_size,
           fn = qcow2_zlib_decompress;
           break;

  +#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+        fn = qcow2_zstd_decompress;
+        break;
+#endif
      default:
          abort();
      }

block/qcow2.c +0 -0
index 643698934d..6632daf50b 100644
@@ -1246,6 +1246,9 @@ static int 
 validate_compression_type(BDRVQcow2State *s, Error **errp)
   {
       switch (s->compression_type) {

      case QCOW2_COMPRESSION_TYPE_ZLIB:

block/qcow2.c +3 -0
+#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+#endif

          break;
        default:

block/qcow2.c +4 -0
@@ -3461,6 +3464,10 @@ qcow2_co_create(BlockdevCreateOptions 
 *create_options, Error **errp)
           }
             switch (qcow2_opts->compression_type) {
+#ifdef CONFIG_ZSTD
+        case QCOW2_COMPRESSION_TYPE_ZSTD:
+            break;
+#endif

          default:
              error_setg(errp, "Unknown compression type"); goto out;

#6 lore raw

Re: [PATCH v11 3/4] qcow2: add zstd cluster compression

Quoted 184 lines · Denis Plotnikov
Quoted 144 lines · Vladimir Sementsov-Ogievskiy
Quoted 127 lines · Denis Plotnikov

zstd significantly reduces cluster compression time. It provides better compression performance maintaining the same level of the compression ratio in comparison with zlib, which, at the moment, is the only compression method available.

The performance test results:
Test compresses and decompresses qemu qcow2 image with just installed rhel-7.6 guest.
Image cluster size: 64K. Image on disk size: 2.2G

The test was conducted with brd disk to reduce the influence of disk subsystem to the test results.
The results is given in seconds.

compress cmd:
   time ./qemu-img convert -O qcow2 -c -o compression_type=[zlib|zstd] src.img [zlib|zstd]_compressed.img decompress cmd
   time ./qemu-img convert -O qcow2 [zlib|zstd]_compressed.img uncompressed.img

            compression               decompression zlib       zstd           zlib         zstd
------------------------------------------------------------
real     65.5       16.3 (-75 %)    1.9          1.6 (-16 %) user     65.0       15.8            5.3          2.5 sys       3.3        0.2            2.0          2.0

Both ZLIB and ZSTD gave the same compression ratio: 1.57 compressed image size in both cases: 1.4G

QAPI part:

---
  docs/interop/qcow2.txt |   1 +
  configure              |   2 +-
  qapi/block-core.json   |   3 +-
  block/qcow2-threads.c  | 138 +++++++++++++++++++++++++++++++++++++++++ block/qcow2.c          |   7 +++

  5 files changed, 149 insertions(+), 2 deletions(-)
docs/interop/qcow2.txt +0 -0
index 5597e24474..795dbb21dd 100644
@@ -208,6 +208,7 @@ version 2.

                        Available compression type values:
                          0: zlib <https://www.zlib.net/>
+                        1: zstd <http://github.com/facebook/zstd>
      === Header padding ===

configure +0 -0
index da09c35895..57cac2abe1 100755
@@ -1861,7 +1861,7 @@ disabled with --disable-FEATURE, default is enabled if available:

    lzfse           support of lzfse compression library
                    (for reading lzfse-compressed dmg images) zstd            support for zstd compression library
-                  (for migration compression)
+                  (for migration compression and qcow2 cluster compression) seccomp         seccomp support
    coroutine-pool  coroutine freelist (better performance) glusterfs       GlusterFS backend

qapi/block-core.json +2 -1
index d669ec0965..44690ed266 100644
@@ -4293,11 +4293,12 @@
   # Compression type used in qcow2 image file
   #
   # @zlib: zlib compression, see <http://zlib.net/>
+# @zstd: zstd compression, see <http://github.com/facebook/zstd>
   #
   # Since: 5.0
   ##
   { 'enum': 'Qcow2CompressionType',
-  'data': [ 'zlib' ] }
+  'data': [ 'zlib', { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } ] }
     ##
   # @BlockdevCreateOptionsQcow2:
block/qcow2-threads.c +0 -0
index 7dbaf53489..b8ccd97009 100644
@@ -28,6 +28,11 @@
   #define ZLIB_CONST
   #include <zlib.h>

  +#ifdef CONFIG_ZSTD

block/qcow2-threads.c +4 -0
+#include <zstd.h>
+#include <zstd_errors.h>
+#endif
+

  #include "qcow2.h"
  #include "block/thread-pool.h"
  #include "crypto.h"

block/qcow2-threads.c +0 -0
@@ -166,6 +171,129 @@ static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size,
       return ret;
   }

  +#ifdef CONFIG_ZSTD

block/qcow2-threads.c +26 -0
+
+/*
+ * qcow2_zstd_compress()
+ *
+ * Compress @src_size bytes of data using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: compressed size on success
+ *          -ENOMEM destination buffer is not enough to store compressed data
+ *          -EIO    on any other error
+ */
+static ssize_t qcow2_zstd_compress(void *dest, size_t dest_size,
+                                   const void *src, size_t src_size)
+{
+    size_t ret;
+    ZSTD_outBuffer output = { dest, dest_size, 0 };
+    ZSTD_inBuffer input = { src, src_size, 0 };
+    ZSTD_CCtx *cctx = ZSTD_createCCtx();
+
+    if (!cctx) {
+        return -EIO;
+    }
+    /*
+     * ZSTD spec: "You must continue calling ZSTD_compressStream2()

block/qcow2-threads.c static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size, · line 210

Quoted 10 lines
block/qcow2-threads.c +10 -0
+     * with ZSTD_e_end until it returns 0, at which point you are
+     * free to start a new frame".
+     * In the loop, we try to compress all the data into one zstd frame.
+     * ZSTD_compressStream2 can potentially finish a frame earlier
+     * than the full input data is consumed. That's why we are looping
+     * until all the input data is consumed.
+     */
+    while (input.pos < input.size) {
+        /*
+         * zstd simple interface requires the exact compressed size.

on decompression you mean

Quoted 2 lines

+         * zstd stream interface reads the comressed size from +         * the compressed stream frame.

Quoted 8 lines

compressed size is not stored in the stream. I think, that streamed interface just decompress until it have something to decompress and have space to write output.

Quoted 2 lines

+         * Instruct zstd to compress the whole buffer and write +         * the frame which includes the compressed size.

I think this is wrong. compression size is not written.

Ok

Quoted 10 lines
Quoted 4 lines

+         * This allows as to use zstd streaming semantics and +         * don't store the compressed size for the zstd decompression. +         */

Comment is just outdated. Accordingly to our off-list discussion, I'd rewrite it like this:

We want to use streamed interface on decompression, as we will not know exact size of compression data.

This one is better then mine.

Quoted 2 lines

Use streamed interface for compression
just for symmetry.

I think this one is exceeding. If we have stream family functions on both sides this won't rise any questions from the reader.

Quoted 13 lines
Quoted 12 lines

+        ret = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end); +        if (ZSTD_isError(ret)) {
+            ret = -EIO;
+            goto out;
+        }
+        /* Dest buffer isn't big enough to store compressed content */ +        if (output.pos + ret > output.size) {
+            ret = -ENOMEM;
+            goto out;
+        }

Quoted 6 lines
Quoted 4 lines

Here you use "@return provides a minimum amount of data remaining to be flushed from internal buffers or an error code, which can be tested using ZSTD_isError()."

I think we can safely drop this check

No, we can't.
we should check for if zstd managed to write the stream correctly. ZSTD_compressStream2 may consume all the input buffer but return ret > 0 meaning that it needs more space.

Hmm, interesting thing. But your check doesn't protect us from it:

Assume we have

output.size = input.size = 64K
output.pos = 64K
input.pos = 10K
ret = 10K

- which means that all input is consumed, but we have some cached data (at least 10K) to flush.

10K + 10K = 20K < 64K, so your check will no lead to an error, but we'll exit the loop..

So, actually, what we need it two things:

1. input.pos = input.size, which means that all input is consumed
2. ret = 0, which means that all cached data flushed

So we need something like

do {
ret = ZSTD_compressStream2(cctx, &output, &input, ZDTD_e_end)

   if (ZSTD_isError(ret)) {
       ret = -EIO.
       goto out;

}
} while (ret || input.pos < input.size);

Quoted 10 lines

This is from my tests:

(gdb) p ret
$1 = 11
(gdb) p input
$2 = {src = 0x562305536000, size = 65536, pos = 65536} (gdb) p output
$3 = {dst = 0x562305546000, size = 65535, pos = 65535}

Quoted 5 lines

Alternatively, we can replace the check with something like

if (ret != 0) {
    ret = -ENOMEM;
}

It's not correct either, it's not an error: it just means that we need to flush a bit more data.

Quoted 83 lines

after the loop, but I think both are equivalent, so I would stick with my one :)))

Quoted 7 lines

work anyway.

Quoted 4 lines

+    };
+
+    /* make sure we can safely return compressed buffer size with ssize_t */ +    assert(output.pos <= SSIZE_MAX);

Quoted 6 lines

Hmm. I hope it's not possible for cluster. But taking the function in separate, it _is_ possible. So may be better to assert at function start that

  assert(dest_size <= SSIZE_MAX);

To stress, that this limitation is part of qcow2_zstd_compress() interface.

Agree, this is better

Quoted 67 lines
Quoted 49 lines

+    ret = output.pos;

block/qcow2-threads.c +47 -0
+
+out:
+    ZSTD_freeCCtx(cctx);
+    return ret;
+}
+
+/*
+ * qcow2_zstd_decompress()
+ *
+ * Decompress some data (not more than @src_size bytes) to produce exactly
+ * @dest_size bytes using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: 0 on success
+ *          -EIO on any error
+ */
+static ssize_t qcow2_zstd_decompress(void *dest, size_t dest_size,
+                                     const void *src, size_t src_size)
+{
+    size_t ret = 0;
+    ZSTD_outBuffer output = { dest, dest_size, 0 };
+    ZSTD_inBuffer input = { src, src_size, 0 };
+    ZSTD_DCtx *dctx = ZSTD_createDCtx();
+
+    if (!dctx) {
+        return -EIO;
+    }
+
+    /*
+     * The compressed stream from input buffer may consist from more
+     * than one zstd frames. So we iterate until we get a fully
+     * uncompressed cluster.
+     */
+    while (output.pos < output.size) {
+        ret = ZSTD_decompressStream(dctx, &output, &input);
+        /*
+         * if we don't fully populate the output but have read
+         * all the frames from the input, we end up with error
+         * here
+         */
+        if (ZSTD_isError(ret)) {
+            ret = -EIO;
+            break;
+        }
+        /*

block/qcow2-threads.c static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size, · line 289

Quoted 10 lines
block/qcow2-threads.c +10 -0
+         * Dest buffer size is the image cluster size.
+         * It should be big enough to store uncompressed content.
+         * There shouldn't be any cases when the decompressed content
+         * size is greater then the cluster size, except cluster
+         * damaging.
+         */
+        if (output.pos + ret > output.size) {
+            ret = -EIO;
+            break;
+        }

But here, you use
"
or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
                                the return value is a suggested next input size (just a hint for better latency) that will never request more than the remaining frame size.
"

Quoted 7 lines
Quoted 6 lines

I'm afraid that "just a hint" is not safe API to make a conclusion from. So, I'd prefer to drop this optimization and just wait for an error from next ZSTD_decompressStream.

And therefore, for symmetry drop similar optimization on compression part..

What do you think?

I'd add some kind of check that we have finished with ret==0 (after loop). It looks like this is the only case when we can be sure that everything went ok.

I think, we should not check it. It is possible that compressed data of another cluster is starting below input end. Is it guaranteed that ZSTD_decompressStream will not start to decompress (or at least plan to do it) the next frame, which is unrelated to our cluster? I'm afraid it's not guaranteed, so we can get ret>0 when output.pos=output.size in correct situation. So I think it's safer not to add this final check for ret. Note that we are not protected from wrong data anyway.

Quoted 61 lines
Quoted 60 lines
Quoted 58 lines

+    }

block/qcow2-threads.c +6 -0
+
+    ZSTD_freeDCtx(dctx);
+    return ret;
+}
+#endif
+

  static int qcow2_compress_pool_func(void *opaque) {
      Qcow2CompressData *data = opaque;

block/qcow2-threads.c +0 -0
@@ -217,6 +345,11 @@ qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,
           fn = qcow2_zlib_compress;
           break;

  +#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+        fn = qcow2_zstd_compress;
+        break;
+#endif
      default:
          abort();
      }

block/qcow2-threads.c +0 -0
@@ -249,6 +382,11 @@ qcow2_co_decompress(BlockDriverState *bs, void *dest, size_t dest_size,
           fn = qcow2_zlib_decompress;
           break;

  +#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+        fn = qcow2_zstd_decompress;
+        break;
+#endif
      default:
          abort();
      }

block/qcow2.c +0 -0
index 643698934d..6632daf50b 100644
@@ -1246,6 +1246,9 @@ static int validate_compression_type(BDRVQcow2State *s, Error **errp)
   {
       switch (s->compression_type) {

      case QCOW2_COMPRESSION_TYPE_ZLIB:

block/qcow2.c +3 -0
+#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+#endif

          break;
        default:

block/qcow2.c +4 -0
@@ -3461,6 +3464,10 @@ qcow2_co_create(BlockdevCreateOptions *create_options, Error **errp)
           }
             switch (qcow2_opts->compression_type) {
+#ifdef CONFIG_ZSTD
+        case QCOW2_COMPRESSION_TYPE_ZSTD:
+            break;
+#endif

          default:
              error_setg(errp, "Unknown compression type"); goto out;

Signature · 2 lines
Best regards,
Vladimir
#7 lore raw

Re: [PATCH v11 3/4] qcow2: add zstd cluster compression

Quoted 213 lines · Vladimir Sementsov-Ogievskiy
Quoted 195 lines · Denis Plotnikov
Quoted 152 lines · Vladimir Sementsov-Ogievskiy
Quoted 137 lines · Denis Plotnikov

zstd significantly reduces cluster compression time. It provides better compression performance maintaining the same level of the compression ratio in comparison with zlib, which, at the moment, is the only compression method available.

The performance test results:
Test compresses and decompresses qemu qcow2 image with just installed rhel-7.6 guest.
Image cluster size: 64K. Image on disk size: 2.2G

The test was conducted with brd disk to reduce the influence of disk subsystem to the test results.
The results is given in seconds.

compress cmd:
   time ./qemu-img convert -O qcow2 -c -o compression_type=[zlib|zstd] src.img [zlib|zstd]_compressed.img decompress cmd
   time ./qemu-img convert -O qcow2 [zlib|zstd]_compressed.img uncompressed.img

            compression               decompression zlib       zstd           zlib         zstd
------------------------------------------------------------
real     65.5       16.3 (-75 %)    1.9          1.6 (-16 %) user     65.0       15.8            5.3          2.5 sys       3.3        0.2            2.0          2.0

Both ZLIB and ZSTD gave the same compression ratio: 1.57 compressed image size in both cases: 1.4G

QAPI part:

---
  docs/interop/qcow2.txt |   1 +
  configure              |   2 +-
  qapi/block-core.json   |   3 +-
  block/qcow2-threads.c  | 138
+++++++++++++++++++++++++++++++++++++++++
  block/qcow2.c          |   7 +++

  5 files changed, 149 insertions(+), 2 deletions(-)
docs/interop/qcow2.txt +0 -0
index 5597e24474..795dbb21dd 100644
@@ -208,6 +208,7 @@ version 2.

                        Available compression type values:
                          0: zlib <https://www.zlib.net/>
+                        1: zstd <http://github.com/facebook/zstd>
      === Header padding ===

configure +0 -0
index da09c35895..57cac2abe1 100755
@@ -1861,7 +1861,7 @@ disabled with --disable-FEATURE, default is 

enabled if available:
    lzfse           support of lzfse compression library (for reading lzfse-compressed dmg images) zstd            support for zstd compression library -                  (for migration compression)
+                  (for migration compression and qcow2 cluster compression)
    seccomp         seccomp support
    coroutine-pool  coroutine freelist (better performance) glusterfs       GlusterFS backend

qapi/block-core.json +2 -1
index d669ec0965..44690ed266 100644
@@ -4293,11 +4293,12 @@
   # Compression type used in qcow2 image file
   #
   # @zlib: zlib compression, see <http://zlib.net/>
+# @zstd: zstd compression, see <http://github.com/facebook/zstd>
   #
   # Since: 5.0
   ##
   { 'enum': 'Qcow2CompressionType',
-  'data': [ 'zlib' ] }
+  'data': [ 'zlib', { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' 
 } ] }
     ##
   # @BlockdevCreateOptionsQcow2:
block/qcow2-threads.c +0 -0
index 7dbaf53489..b8ccd97009 100644
@@ -28,6 +28,11 @@
   #define ZLIB_CONST
   #include <zlib.h>

  +#ifdef CONFIG_ZSTD

block/qcow2-threads.c +4 -0
+#include <zstd.h>
+#include <zstd_errors.h>
+#endif
+

  #include "qcow2.h"
  #include "block/thread-pool.h"
  #include "crypto.h"

block/qcow2-threads.c +0 -0
@@ -166,6 +171,129 @@ static ssize_t qcow2_zlib_decompress(void 
 *dest, size_t dest_size,
       return ret;
   }

  +#ifdef CONFIG_ZSTD

block/qcow2-threads.c +11 -0
+
+/*
+ * qcow2_zstd_compress()
+ *
+ * Compress @src_size bytes of data using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: compressed size on success
+ *          -ENOMEM destination buffer is not enough to store 

compressed data
+ *          -EIO    on any other error
+ */

block/qcow2-threads.c +16 -0
+static ssize_t qcow2_zstd_compress(void *dest, size_t dest_size,
+                                   const void *src, size_t src_size)
+{
+    size_t ret;
+    ZSTD_outBuffer output = { dest, dest_size, 0 };
+    ZSTD_inBuffer input = { src, src_size, 0 };
+    ZSTD_CCtx *cctx = ZSTD_createCCtx();
+
+    if (!cctx) {
+        return -EIO;
+    }
+    /*
+     * ZSTD spec: "You must continue calling ZSTD_compressStream2()
+     * with ZSTD_e_end until it returns 0, at which point you are
+     * free to start a new frame".
+     * In the loop, we try to compress all the data into one zstd 

frame.

Quoted 8 lines

+     * ZSTD_compressStream2 can potentially finish a frame earlier +     * than the full input data is consumed. That's why we are looping
+     * until all the input data is consumed.
+     */
+    while (input.pos < input.size) {
+        /*
+         * zstd simple interface requires the exact compressed size.

on decompression you mean

Quoted 2 lines

+         * zstd stream interface reads the comressed size from +         * the compressed stream frame.

Quoted 8 lines

compressed size is not stored in the stream. I think, that streamed interface just decompress until it have something to decompress and have space to write output.

Quoted 2 lines

+         * Instruct zstd to compress the whole buffer and write +         * the frame which includes the compressed size.

I think this is wrong. compression size is not written.

Ok

Quoted 11 lines
Quoted 5 lines

+         * This allows as to use zstd streaming semantics and +         * don't store the compressed size for the zstd decompression.
+         */

Comment is just outdated. Accordingly to our off-list discussion, I'd rewrite it like this:

We want to use streamed interface on decompression, as we will not know exact size of compression data.

This one is better then mine.

Quoted 2 lines

Use streamed interface for compression
just for symmetry.

I think this one is exceeding. If we have stream family functions on both sides this won't rise any questions from the reader.

Quoted 15 lines
Quoted 14 lines

+        ret = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
+        if (ZSTD_isError(ret)) {
+            ret = -EIO;
+            goto out;
+        }
+        /* Dest buffer isn't big enough to store compressed content */
+        if (output.pos + ret > output.size) {
+            ret = -ENOMEM;
+            goto out;
+        }

Quoted 7 lines
Quoted 4 lines

Here you use "@return provides a minimum amount of data remaining to be flushed from internal buffers or an error code, which can be tested using ZSTD_isError()."

I think we can safely drop this check

No, we can't.
we should check for if zstd managed to write the stream correctly. ZSTD_compressStream2 may consume all the input buffer but return ret

Quoted 1 line

0 meaning that it needs more space.

Hmm, interesting thing. But your check doesn't protect us from it:

Assume we have

output.size = input.size = 64K
output.pos = 64K
input.pos = 10K
ret = 10K

Quoted 4 lines

- which means that all input is consumed, but we have some cached data (at least 10K) to flush.

10K + 10K = 20K < 64K, so your check will no lead to an error, but we'll exit the loop..

No, it does protect
you use incorrect formula: _input_.pos + ret < output.size

but the code is

        if (output.pos + ret > output.size) {
            ret = -ENOMEM;
            goto out;
        }

so, 64K + 10K = 74K -> 74K > 64K (true) -> ret = -ENOMEM

Quoted 26 lines

So, actually, what we need it two things:

  1. input.pos = input.size, which means that all input is consumed
  2. ret = 0, which means that all cached data flushed

So we need something like

do {
   ret = ZSTD_compressStream2(cctx, &output, &input, ZDTD_e_end) if (ZSTD_isError(ret)) {
       ret = -EIO.
       goto out;
   }
} while (ret || input.pos < input.size);

Quoted 10 lines

This is from my tests:

(gdb) p ret
$1 = 11
(gdb) p input
$2 = {src = 0x562305536000, size = 65536, pos = 65536} (gdb) p output
$3 = {dst = 0x562305546000, size = 65535, pos = 65535}

Quoted 7 lines
Quoted 5 lines

Alternatively, we can replace the check with something like

if (ret != 0) {
     ret = -ENOMEM;
}

It's not correct either, it's not an error: it just means that we need to flush a bit more data.

.. but we don't have space to do it, so it looks like -ENOMEM

Quoted 90 lines
Quoted 90 lines

after the loop, but I think both are equivalent, so I would stick with my one :)))

Quoted 8 lines

work anyway.

Quoted 5 lines

+    };
+
+    /* make sure we can safely return compressed buffer size with ssize_t */
+    assert(output.pos <= SSIZE_MAX);

Quoted 6 lines

Hmm. I hope it's not possible for cluster. But taking the function in separate, it _is_ possible. So may be better to assert at function start that

  assert(dest_size <= SSIZE_MAX);

To stress, that this limitation is part of qcow2_zstd_compress() interface.

Agree, this is better

Quoted 72 lines
Quoted 51 lines

+    ret = output.pos;

block/qcow2-threads.c +10 -0
+
+out:
+    ZSTD_freeCCtx(cctx);
+    return ret;
+}
+
+/*
+ * qcow2_zstd_decompress()
+ *
+ * Decompress some data (not more than @src_size bytes) to produce 

exactly
+ * @dest_size bytes using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: 0 on success
+ *          -EIO on any error
+ */
+static ssize_t qcow2_zstd_decompress(void *dest, size_t dest_size, +                                     const void *src, size_t src_size)

block/qcow2-threads.c +27 -0
+{
+    size_t ret = 0;
+    ZSTD_outBuffer output = { dest, dest_size, 0 };
+    ZSTD_inBuffer input = { src, src_size, 0 };
+    ZSTD_DCtx *dctx = ZSTD_createDCtx();
+
+    if (!dctx) {
+        return -EIO;
+    }
+
+    /*
+     * The compressed stream from input buffer may consist from more
+     * than one zstd frames. So we iterate until we get a fully
+     * uncompressed cluster.
+     */
+    while (output.pos < output.size) {
+        ret = ZSTD_decompressStream(dctx, &output, &input);
+        /*
+         * if we don't fully populate the output but have read
+         * all the frames from the input, we end up with error
+         * here
+         */
+        if (ZSTD_isError(ret)) {
+            ret = -EIO;
+            break;
+        }
+        /*

block/qcow2-threads.c static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size, · line 289

Quoted 10 lines
block/qcow2-threads.c +10 -0
+         * Dest buffer size is the image cluster size.
+         * It should be big enough to store uncompressed content.
+         * There shouldn't be any cases when the decompressed content
+         * size is greater then the cluster size, except cluster
+         * damaging.
+         */
+        if (output.pos + ret > output.size) {
+            ret = -EIO;
+            break;
+        }

But here, you use
"
or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
                                the return value is a suggested next input size (just a hint for better latency) that will never request more than the remaining frame size.
"

I'm afraid that "just a hint" is not safe API to make a conclusion from. So, I'd prefer to drop this optimization and just wait for an error from next ZSTD_decompressStream.

Quoted 6 lines
Quoted 4 lines
Quoted 3 lines

And therefore, for symmetry drop similar optimization on compression part..

What do you think?

I'd add some kind of check that we have finished with ret==0 (after loop). It looks like this is the only case when we can be sure that everything went ok.

I think, we should not check it. It is possible that compressed data of another cluster is starting below input end. Is it guaranteed that ZSTD_decompressStream will not start to decompress (or at least plan to do it) the next frame, which is unrelated to our cluster? I'm afraid it's not guaranteed, so we can get ret>0 when output.pos=output.size in correct situation. So I think it's safer not to add this final check for ret. Note that we are not protected from wrong data anyway.

Looking at zlib_compress implementation, yes it seems to be, so. Ok, I'll drop the check.

Quoted 68 lines
Quoted 66 lines
Quoted 65 lines
Quoted 63 lines

+    }

block/qcow2-threads.c +6 -0
+
+    ZSTD_freeDCtx(dctx);
+    return ret;
+}
+#endif
+

  static int qcow2_compress_pool_func(void *opaque) {
      Qcow2CompressData *data = opaque;

block/qcow2-threads.c +0 -0
@@ -217,6 +345,11 @@ qcow2_co_compress(BlockDriverState *bs, void 
 *dest, size_t dest_size,
           fn = qcow2_zlib_compress;
           break;

  +#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+        fn = qcow2_zstd_compress;
+        break;
+#endif
      default:
          abort();
      }

block/qcow2-threads.c +0 -0
@@ -249,6 +382,11 @@ qcow2_co_decompress(BlockDriverState *bs, void 
 *dest, size_t dest_size,
           fn = qcow2_zlib_decompress;
           break;

  +#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+        fn = qcow2_zstd_decompress;
+        break;
+#endif
      default:
          abort();
      }

block/qcow2.c +0 -0
index 643698934d..6632daf50b 100644
@@ -1246,6 +1246,9 @@ static int 
 validate_compression_type(BDRVQcow2State *s, Error **errp)
   {
       switch (s->compression_type) {

      case QCOW2_COMPRESSION_TYPE_ZLIB:

block/qcow2.c +3 -0
+#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+#endif

          break;
        default:

block/qcow2.c +4 -0
@@ -3461,6 +3464,10 @@ qcow2_co_create(BlockdevCreateOptions 
 *create_options, Error **errp)
           }
             switch (qcow2_opts->compression_type) {
+#ifdef CONFIG_ZSTD
+        case QCOW2_COMPRESSION_TYPE_ZSTD:
+            break;
+#endif

          default:
              error_setg(errp, "Unknown compression type"); goto out;

#8 lore raw

Re: [PATCH v11 3/4] qcow2: add zstd cluster compression

Quoted 210 lines · Denis Plotnikov
Quoted 196 lines · Vladimir Sementsov-Ogievskiy
Quoted 184 lines · Denis Plotnikov
Quoted 144 lines · Vladimir Sementsov-Ogievskiy
Quoted 127 lines · Denis Plotnikov

zstd significantly reduces cluster compression time. It provides better compression performance maintaining the same level of the compression ratio in comparison with zlib, which, at the moment, is the only compression method available.

The performance test results:
Test compresses and decompresses qemu qcow2 image with just installed rhel-7.6 guest.
Image cluster size: 64K. Image on disk size: 2.2G

The test was conducted with brd disk to reduce the influence of disk subsystem to the test results.
The results is given in seconds.

compress cmd:
   time ./qemu-img convert -O qcow2 -c -o compression_type=[zlib|zstd] src.img [zlib|zstd]_compressed.img decompress cmd
   time ./qemu-img convert -O qcow2 [zlib|zstd]_compressed.img uncompressed.img

            compression               decompression zlib       zstd           zlib         zstd
------------------------------------------------------------
real     65.5       16.3 (-75 %)    1.9          1.6 (-16 %) user     65.0       15.8            5.3          2.5 sys       3.3        0.2            2.0          2.0

Both ZLIB and ZSTD gave the same compression ratio: 1.57 compressed image size in both cases: 1.4G

QAPI part:

---
  docs/interop/qcow2.txt |   1 +
  configure              |   2 +-
  qapi/block-core.json   |   3 +-
  block/qcow2-threads.c  | 138 +++++++++++++++++++++++++++++++++++++++++ block/qcow2.c          |   7 +++

  5 files changed, 149 insertions(+), 2 deletions(-)
docs/interop/qcow2.txt +0 -0
index 5597e24474..795dbb21dd 100644
@@ -208,6 +208,7 @@ version 2.

                        Available compression type values:
                          0: zlib <https://www.zlib.net/>
+                        1: zstd <http://github.com/facebook/zstd>
      === Header padding ===

configure +0 -0
index da09c35895..57cac2abe1 100755
@@ -1861,7 +1861,7 @@ disabled with --disable-FEATURE, default is enabled if available:

    lzfse           support of lzfse compression library
                    (for reading lzfse-compressed dmg images) zstd            support for zstd compression library
-                  (for migration compression)
+                  (for migration compression and qcow2 cluster compression) seccomp         seccomp support
    coroutine-pool  coroutine freelist (better performance) glusterfs       GlusterFS backend

qapi/block-core.json +2 -1
index d669ec0965..44690ed266 100644
@@ -4293,11 +4293,12 @@
   # Compression type used in qcow2 image file
   #
   # @zlib: zlib compression, see <http://zlib.net/>
+# @zstd: zstd compression, see <http://github.com/facebook/zstd>
   #
   # Since: 5.0
   ##
   { 'enum': 'Qcow2CompressionType',
-  'data': [ 'zlib' ] }
+  'data': [ 'zlib', { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } ] }
     ##
   # @BlockdevCreateOptionsQcow2:
block/qcow2-threads.c +0 -0
index 7dbaf53489..b8ccd97009 100644
@@ -28,6 +28,11 @@
   #define ZLIB_CONST
   #include <zlib.h>

  +#ifdef CONFIG_ZSTD

block/qcow2-threads.c +4 -0
+#include <zstd.h>
+#include <zstd_errors.h>
+#endif
+

  #include "qcow2.h"
  #include "block/thread-pool.h"
  #include "crypto.h"

block/qcow2-threads.c +0 -0
@@ -166,6 +171,129 @@ static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size,
       return ret;
   }

  +#ifdef CONFIG_ZSTD

block/qcow2-threads.c +26 -0
+
+/*
+ * qcow2_zstd_compress()
+ *
+ * Compress @src_size bytes of data using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: compressed size on success
+ *          -ENOMEM destination buffer is not enough to store compressed data
+ *          -EIO    on any other error
+ */
+static ssize_t qcow2_zstd_compress(void *dest, size_t dest_size,
+                                   const void *src, size_t src_size)
+{
+    size_t ret;
+    ZSTD_outBuffer output = { dest, dest_size, 0 };
+    ZSTD_inBuffer input = { src, src_size, 0 };
+    ZSTD_CCtx *cctx = ZSTD_createCCtx();
+
+    if (!cctx) {
+        return -EIO;
+    }
+    /*
+     * ZSTD spec: "You must continue calling ZSTD_compressStream2()

block/qcow2-threads.c static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size, · line 210

Quoted 10 lines
block/qcow2-threads.c +10 -0
+     * with ZSTD_e_end until it returns 0, at which point you are
+     * free to start a new frame".
+     * In the loop, we try to compress all the data into one zstd frame.
+     * ZSTD_compressStream2 can potentially finish a frame earlier
+     * than the full input data is consumed. That's why we are looping
+     * until all the input data is consumed.
+     */
+    while (input.pos < input.size) {
+        /*
+         * zstd simple interface requires the exact compressed size.

on decompression you mean

Quoted 2 lines

+         * zstd stream interface reads the comressed size from +         * the compressed stream frame.

Quoted 8 lines

compressed size is not stored in the stream. I think, that streamed interface just decompress until it have something to decompress and have space to write output.

Quoted 2 lines

+         * Instruct zstd to compress the whole buffer and write +         * the frame which includes the compressed size.

I think this is wrong. compression size is not written.

Ok

Quoted 10 lines
Quoted 4 lines

+         * This allows as to use zstd streaming semantics and +         * don't store the compressed size for the zstd decompression. +         */

Comment is just outdated. Accordingly to our off-list discussion, I'd rewrite it like this:

We want to use streamed interface on decompression, as we will not know exact size of compression data.

This one is better then mine.

Quoted 2 lines

Use streamed interface for compression
just for symmetry.

I think this one is exceeding. If we have stream family functions on both sides this won't rise any questions from the reader.

Quoted 13 lines
Quoted 12 lines

+        ret = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end); +        if (ZSTD_isError(ret)) {
+            ret = -EIO;
+            goto out;
+        }
+        /* Dest buffer isn't big enough to store compressed content */ +        if (output.pos + ret > output.size) {
+            ret = -ENOMEM;
+            goto out;
+        }

Quoted 6 lines
Quoted 4 lines

Here you use "@return provides a minimum amount of data remaining to be flushed from internal buffers or an error code, which can be tested using ZSTD_isError()."

I think we can safely drop this check

No, we can't.
we should check for if zstd managed to write the stream correctly. ZSTD_compressStream2 may consume all the input buffer but return ret > 0 meaning that it needs more space.

Hmm, interesting thing. But your check doesn't protect us from it:

Assume we have

Quoted 8 lines

output.size = input.size = 64K
output.pos = 64K
input.pos = 10K
ret = 10K

- which means that all input is consumed, but we have some cached data (at least 10K) to flush.

10K + 10K = 20K < 64K, so your check will no lead to an error, but we'll exit the loop..

No, it does protect
you use incorrect formula: _input_.pos + ret < output.size

Quoted 8 lines

but the code is

        if (output.pos + ret > output.size) {
            ret = -ENOMEM;
            goto out;
        }

so, 64K + 10K = 74K -> 74K > 64K (true) -> ret = -ENOMEM

Oops, than another example:

output.size = input.size = 64K
output.pos = 40K
input.pos = 64K
ret = 10K

your check doesn't catch it, and it's not an error. But we exit the loop (because input.pos == input.size) and don't write last 10K.

Quoted 27 lines
Quoted 27 lines

So, actually, what we need it two things:

  1. input.pos = input.size, which means that all input is consumed
  2. ret = 0, which means that all cached data flushed

So we need something like

do {
   ret = ZSTD_compressStream2(cctx, &output, &input, ZDTD_e_end) if (ZSTD_isError(ret)) {
       ret = -EIO.
       goto out;
   }
} while (ret || input.pos < input.size);

Quoted 10 lines

This is from my tests:

(gdb) p ret
$1 = 11
(gdb) p input
$2 = {src = 0x562305536000, size = 65536, pos = 65536} (gdb) p output
$3 = {dst = 0x562305546000, size = 65535, pos = 65535}

Quoted 8 lines
Quoted 7 lines
Quoted 5 lines

Alternatively, we can replace the check with something like

if (ret != 0) {
     ret = -ENOMEM;
}

It's not correct either, it's not an error: it just means that we need to flush a bit more data.

.. but we don't have space to do it, so it looks like -ENOMEM

Why don't we have? See example above.

Quoted 161 lines
Quoted 87 lines
Quoted 87 lines

after the loop, but I think both are equivalent, so I would stick with my one :)))

Quoted 7 lines

work anyway.

Quoted 4 lines

+    };
+
+    /* make sure we can safely return compressed buffer size with ssize_t */ +    assert(output.pos <= SSIZE_MAX);

Quoted 6 lines

Hmm. I hope it's not possible for cluster. But taking the function in separate, it _is_ possible. So may be better to assert at function start that

  assert(dest_size <= SSIZE_MAX);

To stress, that this limitation is part of qcow2_zstd_compress() interface.

Agree, this is better

Quoted 70 lines
Quoted 49 lines

+    ret = output.pos;

block/qcow2-threads.c +47 -0
+
+out:
+    ZSTD_freeCCtx(cctx);
+    return ret;
+}
+
+/*
+ * qcow2_zstd_decompress()
+ *
+ * Decompress some data (not more than @src_size bytes) to produce exactly
+ * @dest_size bytes using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: 0 on success
+ *          -EIO on any error
+ */
+static ssize_t qcow2_zstd_decompress(void *dest, size_t dest_size,
+                                     const void *src, size_t src_size)
+{
+    size_t ret = 0;
+    ZSTD_outBuffer output = { dest, dest_size, 0 };
+    ZSTD_inBuffer input = { src, src_size, 0 };
+    ZSTD_DCtx *dctx = ZSTD_createDCtx();
+
+    if (!dctx) {
+        return -EIO;
+    }
+
+    /*
+     * The compressed stream from input buffer may consist from more
+     * than one zstd frames. So we iterate until we get a fully
+     * uncompressed cluster.
+     */
+    while (output.pos < output.size) {
+        ret = ZSTD_decompressStream(dctx, &output, &input);
+        /*
+         * if we don't fully populate the output but have read
+         * all the frames from the input, we end up with error
+         * here
+         */
+        if (ZSTD_isError(ret)) {
+            ret = -EIO;
+            break;
+        }
+        /*

block/qcow2-threads.c static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size, · line 289

Quoted 10 lines
block/qcow2-threads.c +10 -0
+         * Dest buffer size is the image cluster size.
+         * It should be big enough to store uncompressed content.
+         * There shouldn't be any cases when the decompressed content
+         * size is greater then the cluster size, except cluster
+         * damaging.
+         */
+        if (output.pos + ret > output.size) {
+            ret = -EIO;
+            break;
+        }

But here, you use
"
or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
                                the return value is a suggested next input size (just a hint for better latency) that will never request more than the remaining frame size.
"

I'm afraid that "just a hint" is not safe API to make a conclusion from. So, I'd prefer to drop this optimization and just wait for an error from next ZSTD_decompressStream.

Quoted 6 lines
Quoted 4 lines
Quoted 3 lines

And therefore, for symmetry drop similar optimization on compression part..

What do you think?

I'd add some kind of check that we have finished with ret==0 (after loop). It looks like this is the only case when we can be sure that everything went ok.

I think, we should not check it. It is possible that compressed data of another cluster is starting below input end. Is it guaranteed that ZSTD_decompressStream will not start to decompress (or at least plan to do it) the next frame, which is unrelated to our cluster? I'm afraid it's not guaranteed, so we can get ret>0 when output.pos=output.size in correct situation. So I think it's safer not to add this final check for ret. Note that we are not protected from wrong data anyway.

Looking at zlib_compress implementation, yes it seems to be, so. Ok, I'll drop the check.

Quoted 64 lines
Quoted 62 lines
Quoted 61 lines
Quoted 59 lines

+    }

block/qcow2-threads.c +6 -0
+
+    ZSTD_freeDCtx(dctx);
+    return ret;
+}
+#endif
+

  static int qcow2_compress_pool_func(void *opaque) {
      Qcow2CompressData *data = opaque;

block/qcow2-threads.c +0 -0
@@ -217,6 +345,11 @@ qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,
           fn = qcow2_zlib_compress;
           break;

  +#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+        fn = qcow2_zstd_compress;
+        break;
+#endif
      default:
          abort();
      }

block/qcow2-threads.c +0 -0
@@ -249,6 +382,11 @@ qcow2_co_decompress(BlockDriverState *bs, void *dest, size_t dest_size,
           fn = qcow2_zlib_decompress;
           break;

  +#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+        fn = qcow2_zstd_decompress;
+        break;
+#endif
      default:
          abort();
      }

block/qcow2.c +0 -0
index 643698934d..6632daf50b 100644
@@ -1246,6 +1246,9 @@ static int validate_compression_type(BDRVQcow2State *s, Error **errp)
   {
       switch (s->compression_type) {

      case QCOW2_COMPRESSION_TYPE_ZLIB:

block/qcow2.c +3 -0
+#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+#endif

          break;
        default:

block/qcow2.c +4 -0
@@ -3461,6 +3464,10 @@ qcow2_co_create(BlockdevCreateOptions *create_options, Error **errp)
           }
             switch (qcow2_opts->compression_type) {
+#ifdef CONFIG_ZSTD
+        case QCOW2_COMPRESSION_TYPE_ZSTD:
+            break;
+#endif

          default:
              error_setg(errp, "Unknown compression type"); goto out;

Signature · 2 lines
Best regards,
Vladimir
#9 lore raw

Re: [PATCH v11 3/4] qcow2: add zstd cluster compression

Quoted 234 lines · Vladimir Sementsov-Ogievskiy
Quoted 224 lines · Denis Plotnikov
Quoted 214 lines · Vladimir Sementsov-Ogievskiy
Quoted 197 lines · Denis Plotnikov
Quoted 154 lines · Vladimir Sementsov-Ogievskiy
Quoted 139 lines · Denis Plotnikov

zstd significantly reduces cluster compression time. It provides better compression performance maintaining the same level of the compression ratio in comparison with zlib, which, at the moment, is the only compression method available.

The performance test results:
Test compresses and decompresses qemu qcow2 image with just installed rhel-7.6 guest.
Image cluster size: 64K. Image on disk size: 2.2G

The test was conducted with brd disk to reduce the influence of disk subsystem to the test results.
The results is given in seconds.

compress cmd:
   time ./qemu-img convert -O qcow2 -c -o compression_type=[zlib|zstd] src.img [zlib|zstd]_compressed.img decompress cmd
   time ./qemu-img convert -O qcow2 [zlib|zstd]_compressed.img uncompressed.img

            compression               decompression zlib       zstd           zlib         zstd
------------------------------------------------------------
real     65.5       16.3 (-75 %)    1.9          1.6 (-16 %) user     65.0       15.8            5.3          2.5 sys       3.3        0.2            2.0          2.0

Both ZLIB and ZSTD gave the same compression ratio: 1.57 compressed image size in both cases: 1.4G

QAPI part:

---
  docs/interop/qcow2.txt |   1 +
  configure              |   2 +-
  qapi/block-core.json   |   3 +-
  block/qcow2-threads.c  | 138
+++++++++++++++++++++++++++++++++++++++++
  block/qcow2.c          |   7 +++

  5 files changed, 149 insertions(+), 2 deletions(-)
docs/interop/qcow2.txt +0 -0
index 5597e24474..795dbb21dd 100644
@@ -208,6 +208,7 @@ version 2.

                        Available compression type values:
                          0: zlib <https://www.zlib.net/>
+                        1: zstd <http://github.com/facebook/zstd>
      === Header padding ===

configure +0 -0
index da09c35895..57cac2abe1 100755
@@ -1861,7 +1861,7 @@ disabled with --disable-FEATURE, default is 

enabled if available:
    lzfse           support of lzfse compression library (for reading lzfse-compressed dmg images) zstd            support for zstd compression library -                  (for migration compression)
+                  (for migration compression and qcow2 cluster compression)
    seccomp         seccomp support
    coroutine-pool  coroutine freelist (better performance) glusterfs       GlusterFS backend

qapi/block-core.json +2 -1
index d669ec0965..44690ed266 100644
@@ -4293,11 +4293,12 @@
   # Compression type used in qcow2 image file
   #
   # @zlib: zlib compression, see <http://zlib.net/>
+# @zstd: zstd compression, see <http://github.com/facebook/zstd>
   #
   # Since: 5.0
   ##
   { 'enum': 'Qcow2CompressionType',
-  'data': [ 'zlib' ] }
+  'data': [ 'zlib', { 'name': 'zstd', 'if': 
 'defined(CONFIG_ZSTD)' } ] }
     ##
   # @BlockdevCreateOptionsQcow2:
block/qcow2-threads.c +0 -0
index 7dbaf53489..b8ccd97009 100644
@@ -28,6 +28,11 @@
   #define ZLIB_CONST
   #include <zlib.h>

  +#ifdef CONFIG_ZSTD

block/qcow2-threads.c +4 -0
+#include <zstd.h>
+#include <zstd_errors.h>
+#endif
+

  #include "qcow2.h"
  #include "block/thread-pool.h"
  #include "crypto.h"

block/qcow2-threads.c +0 -0
@@ -166,6 +171,129 @@ static ssize_t qcow2_zlib_decompress(void 
 *dest, size_t dest_size,
       return ret;
   }

  +#ifdef CONFIG_ZSTD

block/qcow2-threads.c +11 -0
+
+/*
+ * qcow2_zstd_compress()
+ *
+ * Compress @src_size bytes of data using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: compressed size on success
+ *          -ENOMEM destination buffer is not enough to store 

compressed data
+ *          -EIO    on any other error
+ */
+static ssize_t qcow2_zstd_compress(void *dest, size_t dest_size, +                                   const void *src, size_t src_size)

block/qcow2-threads.c +14 -0
+{
+    size_t ret;
+    ZSTD_outBuffer output = { dest, dest_size, 0 };
+    ZSTD_inBuffer input = { src, src_size, 0 };
+    ZSTD_CCtx *cctx = ZSTD_createCCtx();
+
+    if (!cctx) {
+        return -EIO;
+    }
+    /*
+     * ZSTD spec: "You must continue calling ZSTD_compressStream2()
+     * with ZSTD_e_end until it returns 0, at which point you are
+     * free to start a new frame".
+     * In the loop, we try to compress all the data into one 

zstd frame.
+     * ZSTD_compressStream2 can potentially finish a frame earlier

Quoted 8 lines

+     * than the full input data is consumed. That's why we are looping
+     * until all the input data is consumed.
+     */
+    while (input.pos < input.size) {
+        /*
+         * zstd simple interface requires the exact compressed size.

on decompression you mean

Quoted 2 lines

+         * zstd stream interface reads the comressed size from +         * the compressed stream frame.

Quoted 8 lines

compressed size is not stored in the stream. I think, that streamed interface just decompress until it have something to decompress and have space to write output.

Quoted 2 lines

+         * Instruct zstd to compress the whole buffer and write +         * the frame which includes the compressed size.

I think this is wrong. compression size is not written.

Ok

Quoted 11 lines
Quoted 5 lines

+         * This allows as to use zstd streaming semantics and +         * don't store the compressed size for the zstd decompression.
+         */

Comment is just outdated. Accordingly to our off-list discussion, I'd rewrite it like this:

We want to use streamed interface on decompression, as we will not know exact size of compression data.

This one is better then mine.

Quoted 2 lines

Use streamed interface for compression
just for symmetry.

I think this one is exceeding. If we have stream family functions on both sides this won't rise any questions from the reader.

Quoted 15 lines
Quoted 14 lines

+        ret = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
+        if (ZSTD_isError(ret)) {
+            ret = -EIO;
+            goto out;
+        }
+        /* Dest buffer isn't big enough to store compressed content */
+        if (output.pos + ret > output.size) {
+            ret = -ENOMEM;
+            goto out;
+        }

Quoted 6 lines
Quoted 4 lines

Here you use "@return provides a minimum amount of data remaining to be flushed from internal buffers or an error code, which can be tested using ZSTD_isError()."

I think we can safely drop this check

No, we can't.
we should check for if zstd managed to write the stream correctly. ZSTD_compressStream2 may consume all the input buffer but return ret > 0 meaning that it needs more space.

Hmm, interesting thing. But your check doesn't protect us from it:

Assume we have

output.size = input.size = 64K
output.pos = 64K
input.pos = 10K
ret = 10K

Quoted 4 lines

- which means that all input is consumed, but we have some cached data (at least 10K) to flush.

10K + 10K = 20K < 64K, so your check will no lead to an error, but we'll exit the loop..

No, it does protect
you use incorrect formula: _input_.pos + ret < output.size

Quoted 8 lines

but the code is

         if (output.pos + ret > output.size) {
             ret = -ENOMEM;
             goto out;
         }

so, 64K + 10K = 74K -> 74K > 64K (true) -> ret = -ENOMEM

Quoted 8 lines

Oops, than another example:

output.size = input.size = 64K
output.pos = 40K
input.pos = 64K
ret = 10K

your check doesn't catch it, and it's not an error. But we exit the loop (because input.pos == input.size) and don't write last 10K.

The check is supposed to check for no memory cases. Obviously, your example is NOT no memory case, since 40+10 > 64 -- false.

ret > 0 shouldn't happen, since compres has consumed everything it could, but who knows, what ZSTD_compressStream2 decided to return.

So, ok, we need to do one more iteration untill ret == 0

like so

do {
   ret = ZSTD_compressStream2(cctx, &output, &input, ZDTD_e_end) if (ZSTD_isError(ret)) {
       ret = -EIO.
       goto out;
   }
    if (output.pos + ret > output.size) {
        ret = -ENOMEM;
        goto out;
    }

} while (ret || input.pos < input.size);

I'd like to insist that the no memory check is valid and useful.

Quoted 212 lines
Quoted 29 lines
Quoted 29 lines

So, actually, what we need it two things:

  1. input.pos = input.size, which means that all input is consumed
  2. ret = 0, which means that all cached data flushed

So we need something like

do {
   ret = ZSTD_compressStream2(cctx, &output, &input, ZDTD_e_end) if (ZSTD_isError(ret)) {
       ret = -EIO.
       goto out;
   }
} while (ret || input.pos < input.size);

Quoted 10 lines

This is from my tests:

(gdb) p ret
$1 = 11
(gdb) p input
$2 = {src = 0x562305536000, size = 65536, pos = 65536} (gdb) p output
$3 = {dst = 0x562305546000, size = 65535, pos = 65535}

Quoted 8 lines
Quoted 7 lines
Quoted 5 lines

Alternatively, we can replace the check with something like

if (ret != 0) {
     ret = -ENOMEM;
}

It's not correct either, it's not an error: it just means that we need to flush a bit more data.

.. but we don't have space to do it, so it looks like -ENOMEM

Why don't we have? See example above.

Quoted 170 lines
Quoted 92 lines
Quoted 92 lines

after the loop, but I think both are equivalent, so I would stick with my one :)))

Quoted 8 lines

work anyway.

Quoted 5 lines

+    };
+
+    /* make sure we can safely return compressed buffer size with ssize_t */
+    assert(output.pos <= SSIZE_MAX);

Quoted 6 lines

Hmm. I hope it's not possible for cluster. But taking the function in separate, it _is_ possible. So may be better to assert at function start that

  assert(dest_size <= SSIZE_MAX);

To stress, that this limitation is part of qcow2_zstd_compress() interface.

Agree, this is better

Quoted 74 lines
Quoted 55 lines

+    ret = output.pos;

block/qcow2-threads.c +10 -0
+
+out:
+    ZSTD_freeCCtx(cctx);
+    return ret;
+}
+
+/*
+ * qcow2_zstd_decompress()
+ *
+ * Decompress some data (not more than @src_size bytes) to 

produce exactly
+ * @dest_size bytes using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: 0 on success
+ *          -EIO on any error
+ */
+static ssize_t qcow2_zstd_decompress(void *dest, size_t dest_size, +                                     const void *src, size_t src_size)

block/qcow2-threads.c +12 -0
+{
+    size_t ret = 0;
+    ZSTD_outBuffer output = { dest, dest_size, 0 };
+    ZSTD_inBuffer input = { src, src_size, 0 };
+    ZSTD_DCtx *dctx = ZSTD_createDCtx();
+
+    if (!dctx) {
+        return -EIO;
+    }
+
+    /*
+     * The compressed stream from input buffer may consist from 

more
+     * than one zstd frames. So we iterate until we get a fully +     * uncompressed cluster.
+     */
+    while (output.pos < output.size) {
+        ret = ZSTD_decompressStream(dctx, &output, &input); +        /*
+         * if we don't fully populate the output but have read +         * all the frames from the input, we end up with error +         * here
+         */
+        if (ZSTD_isError(ret)) {
+            ret = -EIO;
+            break;
+        }
+        /*
+         * Dest buffer size is the image cluster size. +         * It should be big enough to store uncompressed content. +         * There shouldn't be any cases when the decompressed

Quoted 8 lines

content
+         * size is greater then the cluster size, except cluster +         * damaging.
+         */
+        if (output.pos + ret > output.size) {
+            ret = -EIO;
+            break;
+        }

But here, you use
"
or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
                                the return value is a suggested next input size (just a hint for better latency) that will never request more than the remaining frame size.
"

I'm afraid that "just a hint" is not safe API to make a conclusion from. So, I'd prefer to drop this optimization and just wait for an error from next ZSTD_decompressStream.

Quoted 6 lines
Quoted 4 lines
Quoted 3 lines

And therefore, for symmetry drop similar optimization on compression part..

What do you think?

I'd add some kind of check that we have finished with ret==0 (after loop). It looks like this is the only case when we can be sure that everything went ok.

I think, we should not check it. It is possible that compressed data of another cluster is starting below input end. Is it guaranteed that ZSTD_decompressStream will not start to decompress (or at least plan to do it) the next frame, which is unrelated to our cluster? I'm afraid it's not guaranteed, so we can get ret>0 when output.pos=output.size in correct situation. So I think it's safer not to add this final check for ret. Note that we are not protected from wrong data anyway.

Looking at zlib_compress implementation, yes it seems to be, so. Ok, I'll drop the check.

Quoted 68 lines
Quoted 66 lines
Quoted 65 lines
Quoted 63 lines

+    }

block/qcow2-threads.c +6 -0
+
+    ZSTD_freeDCtx(dctx);
+    return ret;
+}
+#endif
+

  static int qcow2_compress_pool_func(void *opaque) {
      Qcow2CompressData *data = opaque;

block/qcow2-threads.c +0 -0
@@ -217,6 +345,11 @@ qcow2_co_compress(BlockDriverState *bs, void 
 *dest, size_t dest_size,
           fn = qcow2_zlib_compress;
           break;

  +#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+        fn = qcow2_zstd_compress;
+        break;
+#endif
      default:
          abort();
      }

block/qcow2-threads.c +0 -0
@@ -249,6 +382,11 @@ qcow2_co_decompress(BlockDriverState *bs, 
 void *dest, size_t dest_size,
           fn = qcow2_zlib_decompress;
           break;

  +#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+        fn = qcow2_zstd_decompress;
+        break;
+#endif
      default:
          abort();
      }

block/qcow2.c +0 -0
index 643698934d..6632daf50b 100644
@@ -1246,6 +1246,9 @@ static int 
 validate_compression_type(BDRVQcow2State *s, Error **errp)
   {
       switch (s->compression_type) {

      case QCOW2_COMPRESSION_TYPE_ZLIB:

block/qcow2.c +3 -0
+#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+#endif

          break;
        default:

block/qcow2.c +4 -0
@@ -3461,6 +3464,10 @@ qcow2_co_create(BlockdevCreateOptions 
 *create_options, Error **errp)
           }
             switch (qcow2_opts->compression_type) {
+#ifdef CONFIG_ZSTD
+        case QCOW2_COMPRESSION_TYPE_ZSTD:
+            break;
+#endif

          default:
              error_setg(errp, "Unknown compression type"); goto out;

#10 lore raw

Re: [PATCH v11 3/4] qcow2: add zstd cluster compression

Quoted 250 lines · Denis Plotnikov
Quoted 220 lines · Vladimir Sementsov-Ogievskiy
Quoted 210 lines · Denis Plotnikov
Quoted 196 lines · Vladimir Sementsov-Ogievskiy
Quoted 184 lines · Denis Plotnikov
Quoted 144 lines · Vladimir Sementsov-Ogievskiy
Quoted 127 lines · Denis Plotnikov

zstd significantly reduces cluster compression time. It provides better compression performance maintaining the same level of the compression ratio in comparison with zlib, which, at the moment, is the only compression method available.

The performance test results:
Test compresses and decompresses qemu qcow2 image with just installed rhel-7.6 guest.
Image cluster size: 64K. Image on disk size: 2.2G

The test was conducted with brd disk to reduce the influence of disk subsystem to the test results.
The results is given in seconds.

compress cmd:
   time ./qemu-img convert -O qcow2 -c -o compression_type=[zlib|zstd] src.img [zlib|zstd]_compressed.img decompress cmd
   time ./qemu-img convert -O qcow2 [zlib|zstd]_compressed.img uncompressed.img

            compression               decompression zlib       zstd           zlib         zstd
------------------------------------------------------------
real     65.5       16.3 (-75 %)    1.9          1.6 (-16 %) user     65.0       15.8            5.3          2.5 sys       3.3        0.2            2.0          2.0

Both ZLIB and ZSTD gave the same compression ratio: 1.57 compressed image size in both cases: 1.4G

QAPI part:

---
  docs/interop/qcow2.txt |   1 +
  configure              |   2 +-
  qapi/block-core.json   |   3 +-
  block/qcow2-threads.c  | 138 +++++++++++++++++++++++++++++++++++++++++ block/qcow2.c          |   7 +++

  5 files changed, 149 insertions(+), 2 deletions(-)
docs/interop/qcow2.txt +0 -0
index 5597e24474..795dbb21dd 100644
@@ -208,6 +208,7 @@ version 2.

                        Available compression type values:
                          0: zlib <https://www.zlib.net/>
+                        1: zstd <http://github.com/facebook/zstd>
      === Header padding ===

configure +0 -0
index da09c35895..57cac2abe1 100755
@@ -1861,7 +1861,7 @@ disabled with --disable-FEATURE, default is enabled if available:

    lzfse           support of lzfse compression library
                    (for reading lzfse-compressed dmg images) zstd            support for zstd compression library
-                  (for migration compression)
+                  (for migration compression and qcow2 cluster compression) seccomp         seccomp support
    coroutine-pool  coroutine freelist (better performance) glusterfs       GlusterFS backend

qapi/block-core.json +2 -1
index d669ec0965..44690ed266 100644
@@ -4293,11 +4293,12 @@
   # Compression type used in qcow2 image file
   #
   # @zlib: zlib compression, see <http://zlib.net/>
+# @zstd: zstd compression, see <http://github.com/facebook/zstd>
   #
   # Since: 5.0
   ##
   { 'enum': 'Qcow2CompressionType',
-  'data': [ 'zlib' ] }
+  'data': [ 'zlib', { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } ] }
     ##
   # @BlockdevCreateOptionsQcow2:
block/qcow2-threads.c +0 -0
index 7dbaf53489..b8ccd97009 100644
@@ -28,6 +28,11 @@
   #define ZLIB_CONST
   #include <zlib.h>

  +#ifdef CONFIG_ZSTD

block/qcow2-threads.c +4 -0
+#include <zstd.h>
+#include <zstd_errors.h>
+#endif
+

  #include "qcow2.h"
  #include "block/thread-pool.h"
  #include "crypto.h"

block/qcow2-threads.c +0 -0
@@ -166,6 +171,129 @@ static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size,
       return ret;
   }

  +#ifdef CONFIG_ZSTD

block/qcow2-threads.c +26 -0
+
+/*
+ * qcow2_zstd_compress()
+ *
+ * Compress @src_size bytes of data using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: compressed size on success
+ *          -ENOMEM destination buffer is not enough to store compressed data
+ *          -EIO    on any other error
+ */
+static ssize_t qcow2_zstd_compress(void *dest, size_t dest_size,
+                                   const void *src, size_t src_size)
+{
+    size_t ret;
+    ZSTD_outBuffer output = { dest, dest_size, 0 };
+    ZSTD_inBuffer input = { src, src_size, 0 };
+    ZSTD_CCtx *cctx = ZSTD_createCCtx();
+
+    if (!cctx) {
+        return -EIO;
+    }
+    /*
+     * ZSTD spec: "You must continue calling ZSTD_compressStream2()

block/qcow2-threads.c

Quoted 10 lines
block/qcow2-threads.c +10 -0
+     * with ZSTD_e_end until it returns 0, at which point you are
+     * free to start a new frame".
+     * In the loop, we try to compress all the data into one zstd frame.
+     * ZSTD_compressStream2 can potentially finish a frame earlier
+     * than the full input data is consumed. That's why we are looping
+     * until all the input data is consumed.
+     */
+    while (input.pos < input.size) {
+        /*
+         * zstd simple interface requires the exact compressed size.

on decompression you mean

Quoted 2 lines

+         * zstd stream interface reads the comressed size from +         * the compressed stream frame.

Quoted 8 lines

compressed size is not stored in the stream. I think, that streamed interface just decompress until it have something to decompress and have space to write output.

Quoted 2 lines

+         * Instruct zstd to compress the whole buffer and write +         * the frame which includes the compressed size.

I think this is wrong. compression size is not written.

Ok

Quoted 10 lines
Quoted 4 lines

+         * This allows as to use zstd streaming semantics and +         * don't store the compressed size for the zstd decompression. +         */

Comment is just outdated. Accordingly to our off-list discussion, I'd rewrite it like this:

We want to use streamed interface on decompression, as we will not know exact size of compression data.

This one is better then mine.

Quoted 2 lines

Use streamed interface for compression
just for symmetry.

I think this one is exceeding. If we have stream family functions on both sides this won't rise any questions from the reader.

Quoted 13 lines
Quoted 12 lines

+        ret = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end); +        if (ZSTD_isError(ret)) {
+            ret = -EIO;
+            goto out;
+        }
+        /* Dest buffer isn't big enough to store compressed content */ +        if (output.pos + ret > output.size) {
+            ret = -ENOMEM;
+            goto out;
+        }

Quoted 6 lines
Quoted 4 lines

Here you use "@return provides a minimum amount of data remaining to be flushed from internal buffers or an error code, which can be tested using ZSTD_isError()."

I think we can safely drop this check

No, we can't.
we should check for if zstd managed to write the stream correctly. ZSTD_compressStream2 may consume all the input buffer but return ret > 0 meaning that it needs more space.

Hmm, interesting thing. But your check doesn't protect us from it:

Assume we have

Quoted 8 lines

output.size = input.size = 64K
output.pos = 64K
input.pos = 10K
ret = 10K

- which means that all input is consumed, but we have some cached data (at least 10K) to flush.

10K + 10K = 20K < 64K, so your check will no lead to an error, but we'll exit the loop..

No, it does protect
you use incorrect formula: _input_.pos + ret < output.size

Quoted 8 lines

but the code is

         if (output.pos + ret > output.size) {
             ret = -ENOMEM;
             goto out;
         }

so, 64K + 10K = 74K -> 74K > 64K (true) -> ret = -ENOMEM

Quoted 8 lines

Oops, than another example:

output.size = input.size = 64K
output.pos = 40K
input.pos = 64K
ret = 10K

your check doesn't catch it, and it's not an error. But we exit the loop (because input.pos == input.size) and don't write last 10K.

The check is supposed to check for no memory cases. Obviously, your example is NOT no memory case, since 40+10 > 64 -- false.

ret > 0 shouldn't happen, since compres has consumed everything it could, but who knows, what ZSTD_compressStream2 decided to return.

So, ok, we need to do one more iteration untill ret == 0

like so

do {
   ret = ZSTD_compressStream2(cctx, &output, &input, ZDTD_e_end) if (ZSTD_isError(ret)) {
       ret = -EIO.
       goto out;
   }
    if (output.pos + ret > output.size) {
        ret = -ENOMEM;
        goto out;
    }

Quoted 3 lines

} while (ret || input.pos < input.size);

I'd like to insist that the no memory check is valid and useful.

Agree, I understand now: we may go to infinite loop, where ZSTD will ask for more space.

Quoted 206 lines
Quoted 205 lines
Quoted 31 lines
Quoted 31 lines

So, actually, what we need it two things:

  1. input.pos = input.size, which means that all input is consumed
  2. ret = 0, which means that all cached data flushed

So we need something like

do {
   ret = ZSTD_compressStream2(cctx, &output, &input, ZDTD_e_end) if (ZSTD_isError(ret)) {
       ret = -EIO.
       goto out;
   }
} while (ret || input.pos < input.size);

Quoted 10 lines

This is from my tests:

(gdb) p ret
$1 = 11
(gdb) p input
$2 = {src = 0x562305536000, size = 65536, pos = 65536} (gdb) p output
$3 = {dst = 0x562305546000, size = 65535, pos = 65535}

Quoted 8 lines
Quoted 7 lines
Quoted 5 lines

Alternatively, we can replace the check with something like

if (ret != 0) {
     ret = -ENOMEM;
}

It's not correct either, it's not an error: it just means that we need to flush a bit more data.

.. but we don't have space to do it, so it looks like -ENOMEM

Why don't we have? See example above.

Quoted 161 lines
Quoted 87 lines
Quoted 87 lines

after the loop, but I think both are equivalent, so I would stick with my one :)))

Quoted 7 lines

work anyway.

Quoted 4 lines

+    };
+
+    /* make sure we can safely return compressed buffer size with ssize_t */ +    assert(output.pos <= SSIZE_MAX);

Quoted 6 lines

Hmm. I hope it's not possible for cluster. But taking the function in separate, it _is_ possible. So may be better to assert at function start that

  assert(dest_size <= SSIZE_MAX);

To stress, that this limitation is part of qcow2_zstd_compress() interface.

Agree, this is better

Quoted 70 lines
Quoted 49 lines

+    ret = output.pos;

block/qcow2-threads.c +47 -0
+
+out:
+    ZSTD_freeCCtx(cctx);
+    return ret;
+}
+
+/*
+ * qcow2_zstd_decompress()
+ *
+ * Decompress some data (not more than @src_size bytes) to produce exactly
+ * @dest_size bytes using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: 0 on success
+ *          -EIO on any error
+ */
+static ssize_t qcow2_zstd_decompress(void *dest, size_t dest_size,
+                                     const void *src, size_t src_size)
+{
+    size_t ret = 0;
+    ZSTD_outBuffer output = { dest, dest_size, 0 };
+    ZSTD_inBuffer input = { src, src_size, 0 };
+    ZSTD_DCtx *dctx = ZSTD_createDCtx();
+
+    if (!dctx) {
+        return -EIO;
+    }
+
+    /*
+     * The compressed stream from input buffer may consist from more
+     * than one zstd frames. So we iterate until we get a fully
+     * uncompressed cluster.
+     */
+    while (output.pos < output.size) {
+        ret = ZSTD_decompressStream(dctx, &output, &input);
+        /*
+         * if we don't fully populate the output but have read
+         * all the frames from the input, we end up with error
+         * here
+         */
+        if (ZSTD_isError(ret)) {
+            ret = -EIO;
+            break;
+        }
+        /*

block/qcow2-threads.c

Quoted 10 lines
block/qcow2-threads.c +10 -0
+         * Dest buffer size is the image cluster size.
+         * It should be big enough to store uncompressed content.
+         * There shouldn't be any cases when the decompressed content
+         * size is greater then the cluster size, except cluster
+         * damaging.
+         */
+        if (output.pos + ret > output.size) {
+            ret = -EIO;
+            break;
+        }

But here, you use
"
or any other value > 0, which means there is still some decoding or flushing to do to complete current frame :
                                the return value is a suggested next input size (just a hint for better latency) that will never request more than the remaining frame size.
"

I'm afraid that "just a hint" is not safe API to make a conclusion from. So, I'd prefer to drop this optimization and just wait for an error from next ZSTD_decompressStream.

Quoted 6 lines
Quoted 4 lines
Quoted 3 lines

And therefore, for symmetry drop similar optimization on compression part..

What do you think?

I'd add some kind of check that we have finished with ret==0 (after loop). It looks like this is the only case when we can be sure that everything went ok.

I think, we should not check it. It is possible that compressed data of another cluster is starting below input end. Is it guaranteed that ZSTD_decompressStream will not start to decompress (or at least plan to do it) the next frame, which is unrelated to our cluster? I'm afraid it's not guaranteed, so we can get ret>0 when output.pos=output.size in correct situation. So I think it's safer not to add this final check for ret. Note that we are not protected from wrong data anyway.

Looking at zlib_compress implementation, yes it seems to be, so. Ok, I'll drop the check.

Quoted 64 lines
Quoted 62 lines
Quoted 61 lines
Quoted 59 lines

+    }

block/qcow2-threads.c +6 -0
+
+    ZSTD_freeDCtx(dctx);
+    return ret;
+}
+#endif
+

  static int qcow2_compress_pool_func(void *opaque) {
      Qcow2CompressData *data = opaque;

block/qcow2-threads.c +0 -0
@@ -217,6 +345,11 @@ qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,
           fn = qcow2_zlib_compress;
           break;

  +#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+        fn = qcow2_zstd_compress;
+        break;
+#endif
      default:
          abort();
      }

block/qcow2-threads.c +0 -0
@@ -249,6 +382,11 @@ qcow2_co_decompress(BlockDriverState *bs, void *dest, size_t dest_size,
           fn = qcow2_zlib_decompress;
           break;

  +#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+        fn = qcow2_zstd_decompress;
+        break;
+#endif
      default:
          abort();
      }

block/qcow2.c +0 -0
index 643698934d..6632daf50b 100644
@@ -1246,6 +1246,9 @@ static int validate_compression_type(BDRVQcow2State *s, Error **errp)
   {
       switch (s->compression_type) {

      case QCOW2_COMPRESSION_TYPE_ZLIB:

block/qcow2.c +3 -0
+#ifdef CONFIG_ZSTD
+    case QCOW2_COMPRESSION_TYPE_ZSTD:
+#endif

          break;
        default:

block/qcow2.c +4 -0
@@ -3461,6 +3464,10 @@ qcow2_co_create(BlockdevCreateOptions *create_options, Error **errp)
           }
             switch (qcow2_opts->compression_type) {
+#ifdef CONFIG_ZSTD
+        case QCOW2_COMPRESSION_TYPE_ZSTD:
+            break;
+#endif

          default:
              error_setg(errp, "Unknown compression type"); goto out;

Signature · 2 lines
Best regards,
Vladimir