Skip to contentSkip to conversation

qemu-img: add preallocation=full

PATCH Superseded Hu Tao 3 participants 7 messages Apr 9, 2014 – May 6, 2014 Block layer

v8 of 15 Superseded Latest is v15 →

lore
Share & tools
lore

Hu Tao

author
#0 lore raw

v8 cover qemu-img: add preallocation=full

The purpose of this series is to use posix_fallocate() when creating img file to ensure there are disk space for it which is way fast than acturally writing to disk. But this only works in file system level. For cases like thin provisioning, an option full preallocation is added to write zeros to storage to ensure disk space.

Hu Tao (4):
qapi: introduce PreallocMode and a new PreallocMode full. raw, qcow2: don't convert file size to sector size
raw-posix: Add full image preallocation option
qcow2: Add full image preallocation option

 block/qcow2.c              | 95 ++++++++++++++++++++++++++++++++++++++++------
 block/raw-posix.c          | 63 ++++++++++++++++++++++++++----
 block/raw-win32.c          |  4 +-
 qapi-schema.json           | 14 +++++++
 tests/qemu-iotests/082.out | 54 +++++++++++++-------------
 5 files changed, 182 insertions(+), 48 deletions(-)
Signature · 1 line
1.8.5.2.229.g4448466
author
#1 lore raw

PATCH v8 1/4 qapi: introduce PreallocMode and a new PreallocMode full.

This patch prepares for the subsequent patches.

---
 block/qcow2.c    |  8 ++++----
 qapi-schema.json | 14 ++++++++++++++
 2 files changed, 18 insertions(+), 4 deletions(-)
block/qcow2.c +4 -4
index e903d97..3377007 100644
@@ -1592,7 +1592,7 @@ static int preallocate(BlockDriverState *bs)
 
 static int qcow2_create2(const char *filename, int64_t total_size,
                          const char *backing_file, const char *backing_format,
-                         int flags, size_t cluster_size, int prealloc,
+                         int flags, size_t cluster_size, PreallocMode prealloc,
                          QEMUOptionParameter *options, int version,
                          Error **errp)
 {
@@ -1769,7 +1769,7 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options,
     uint64_t sectors = 0;
     int flags = 0;
     size_t cluster_size = DEFAULT_CLUSTER_SIZE;
-    int prealloc = 0;
+    PreallocMode prealloc = PREALLOC_MODE_OFF;
     int version = 3;
     Error *local_err = NULL;
     int ret;
@@ -1790,9 +1790,9 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options,
             }
         } else if (!strcmp(options->name, BLOCK_OPT_PREALLOC)) {
             if (!options->value.s || !strcmp(options->value.s, "off")) {
-                prealloc = 0;
+                prealloc = PREALLOC_MODE_OFF;
             } else if (!strcmp(options->value.s, "metadata")) {
-                prealloc = 1;
+                prealloc = PREALLOC_MODE_METADATA;
             } else {
                 error_setg(errp, "Invalid preallocation mode: '%s'",
                            options->value.s);
qapi-schema.json +14 -0
index 391356f..9e6221a 100644
@@ -4689,3 +4689,17 @@
               'btn'     : 'InputBtnEvent',
               'rel'     : 'InputMoveEvent',
               'abs'     : 'InputMoveEvent' } }
+
+##
+# @PreallocMode
+#
+# Preallocation mode of QEMU image file
+#
+# @off: no preallocation
+# @metadata: preallocate only for metadata
+# @full: preallocate all data, including metadata
+#
+# Since 2.0
+##
+{ 'enum': 'PreallocMode',
+  'data': [ 'off', 'metadata', 'full' ] }
Signature · 1 line
1.8.5.2.229.g4448466
Reviewed-by on v8 1/4
#2 lore raw

Re: [Qemu-devel] [PATCH v8 1/4] qapi: introduce PreallocMode and a new PreallocMode full.

Quoted 16 lines · Hu Tao

This patch prepares for the subsequent patches.

---
 block/qcow2.c    |  8 ++++----
 qapi-schema.json | 14 ++++++++++++++
 2 files changed, 18 insertions(+), 4 deletions(-)

+++ b/qapi-schema.json

qapi-schema.json +1 -0
@@ -4689,3 +4689,17 @@
               'btn'     : 'InputBtnEvent',
               'rel'     : 'InputMoveEvent',
               'abs'     : 'InputMoveEvent' } }
+

qapi-schema.json line 4702

Quoted 10 lines
qapi-schema.json +10 -0
+##
+# @PreallocMode
+#
+# Preallocation mode of QEMU image file
+#
+# @off: no preallocation
+# @metadata: preallocate only for metadata
+# @full: preallocate all data, including metadata
+#
+# Since 2.0

You've missed 2.0. This should be 2.1. With that fixed,

Signature · 2 lines
Eric Blake   eblake redhat com    +1-919-301-3266
Libvirt virtualization library http://libvirt.org
author
#3 lore raw

PATCH v8 2/4 raw, qcow2: don't convert file size to sector size

In reply to Hu Tao

and avoid converting it back later.

---
 block/qcow2.c     | 8 ++++----
 block/raw-posix.c | 4 ++--
 block/raw-win32.c | 4 ++--
 3 files changed, 8 insertions(+), 8 deletions(-)
block/qcow2.c +4 -4
index 3377007..b2ff9c3 100644
@@ -1713,7 +1713,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
     }
 
     /* Okay, now that we have a valid image, let's give it the right size */
-    ret = bdrv_truncate(bs, total_size * BDRV_SECTOR_SIZE);
+    ret = bdrv_truncate(bs, total_size);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not resize image");
         goto out;
@@ -1766,7 +1766,7 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options,
 {
     const char *backing_file = NULL;
     const char *backing_fmt = NULL;
-    uint64_t sectors = 0;
+    uint64_t size = 0;
     int flags = 0;
     size_t cluster_size = DEFAULT_CLUSTER_SIZE;
     PreallocMode prealloc = PREALLOC_MODE_OFF;
@@ -1777,7 +1777,7 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options,
     /* Read out options */
     while (options && options->name) {
         if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
-            sectors = options->value.n / 512;
+            size = options->value.n & BDRV_SECTOR_MASK;
         } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FILE)) {
             backing_file = options->value.s;
         } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FMT)) {
@@ -1828,7 +1828,7 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options,
         return -EINVAL;
     }
 
-    ret = qcow2_create2(filename, sectors, backing_file, backing_fmt, flags,
+    ret = qcow2_create2(filename, size, backing_file, backing_fmt, flags,
                         cluster_size, prealloc, options, version, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
block/raw-posix.c +2 -2
index 1688e16..a363b71 100644
@@ -1246,7 +1246,7 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,
     /* Read out options */
     while (options && options->name) {
         if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
-            total_size = options->value.n / BDRV_SECTOR_SIZE;
+            total_size = options->value.n & BDRV_SECTOR_MASK;
         }
         options++;
     }
@@ -1257,7 +1257,7 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,
         result = -errno;
         error_setg_errno(errp, -result, "Could not create file");
     } else {
-        if (ftruncate(fd, total_size * BDRV_SECTOR_SIZE) != 0) {
+        if (ftruncate(fd, total_size) != 0) {
             result = -errno;
             error_setg_errno(errp, -result, "Could not resize file");
         }
block/raw-win32.c +2 -2
index 48cb2c2..9fe60f3 100644
@@ -486,7 +486,7 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,
     /* Read out options */
     while (options && options->name) {
         if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
-            total_size = options->value.n / 512;
+            total_size = options->value.n & BDRV_SECTOR_MASK;
         }
         options++;
     }
@@ -498,7 +498,7 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,
         return -EIO;
     }
     set_sparse(fd);
-    ftruncate(fd, total_size * 512);
+    ftruncate(fd, total_size);
     qemu_close(fd);
     return 0;
 }
Signature · 1 line
1.8.5.2.229.g4448466
author
#4 lore raw

PATCH v8 3/4 raw-posix: Add full image preallocation option

In reply to Hu Tao

This patch adds a new option preallocation for raw format, and implements full preallocation.

---
 block/raw-posix.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 54 insertions(+), 7 deletions(-)
block/raw-posix.c +54 -7
index a363b71..44a1b4a 100644
@@ -1240,6 +1240,7 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,
     int fd;
     int result = 0;
     int64_t total_size = 0;
+    PreallocMode prealloc = PREALLOC_MODE_OFF;
 
     strstart(filename, "file:", &filename);
 
@@ -1247,6 +1248,18 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,
     while (options && options->name) {
         if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
             total_size = options->value.n & BDRV_SECTOR_MASK;
+        } else if (!strcmp(options->name, BLOCK_OPT_PREALLOC)) {
+            if (!options->value.s || !strcmp(options->value.s, "off")) {
+                prealloc = PREALLOC_MODE_OFF;
+            } else if (!strcmp(options->value.s, "metadata")) {
+                prealloc = PREALLOC_MODE_METADATA;
+            } else if (!strcmp(options->value.s, "full")) {
+                prealloc = PREALLOC_MODE_FULL;
+            } else {
+                error_setg(errp, "Invalid preallocation mode: '%s'",
+                           options->value.s);
+                return -EINVAL;
+            }
         }
         options++;
     }
@@ -1256,16 +1269,45 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,
     if (fd < 0) {
         result = -errno;
         error_setg_errno(errp, -result, "Could not create file");
-    } else {
-        if (ftruncate(fd, total_size) != 0) {
-            result = -errno;
-            error_setg_errno(errp, -result, "Could not resize file");
+        goto out;
+    }
+    if (ftruncate(fd, total_size) != 0) {
+        result = -errno;
+        error_setg_errno(errp, -result, "Could not resize file");
+        goto out_close;
+    }
+    if (prealloc == PREALLOC_MODE_METADATA) {
+        /* posix_fallocate() doesn't set errno. */
+        result = -posix_fallocate(fd, 0, total_size);
+        if (result != 0) {
+            error_setg_errno(errp, -result,
+                             "Could not preallocate data for the new file");
         }
-        if (qemu_close(fd) != 0) {
-            result = -errno;
-            error_setg_errno(errp, -result, "Could not close the new file");
+    } else if (prealloc == PREALLOC_MODE_FULL) {
+        char *buf = g_malloc0(65536);
+        int64_t num = 0, left = total_size;
+
+        while (left > 0) {
+            num = MIN(left, 65536);
+            result = write(fd, buf, num);
+            if (result < 0) {
+                result = -errno;
+                error_setg_errno(errp, -result,
+                                 "Could not write to the new file");
+                g_free(buf);
+                goto out_close;
+            }
+            left -= num;
         }
+        fsync(fd);
+        g_free(buf);
+    }
+out_close:
+    if (qemu_close(fd) != 0) {
+        result = -errno;
+        error_setg_errno(errp, -result, "Could not close the new file");
     }
+out:
     return result;
 }
 
@@ -1416,6 +1458,11 @@ static QEMUOptionParameter raw_create_options[] = {
         .type = OPT_SIZE,
         .help = "Virtual disk size"
     },
+    {
+        .name = BLOCK_OPT_PREALLOC,
+        .type = OPT_STRING,
+        .help = "Preallocation mode (allowed values: off, metadata, full)"
+    },
     { NULL }
 };
 
Signature · 1 line
1.8.5.2.229.g4448466
author
#5 lore raw

PATCH v8 4/4 qcow2: Add full image preallocation option

In reply to Hu Tao

This adds a preallocation=full mode to qcow2 image creation, which creates a non-sparse image file.

---
 block/qcow2.c              | 79 ++++++++++++++++++++++++++++++++++++++++++++--
 tests/qemu-iotests/082.out | 54 +++++++++++++++----------------
 2 files changed, 103 insertions(+), 30 deletions(-)
block/qcow2.c +76 -3
index b2ff9c3..1b0034d 100644
@@ -1596,6 +1596,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
                          QEMUOptionParameter *options, int version,
                          Error **errp)
 {
+    QEMUOptionParameter *alloc_options = NULL;
     /* Calculate cluster_bits */
     int cluster_bits;
     cluster_bits = ffs(cluster_size) - 1;
@@ -1625,10 +1626,78 @@ static int qcow2_create2(const char *filename, int64_t total_size,
     Error *local_err = NULL;
     int ret;
 
+    if (prealloc == PREALLOC_MODE_FULL || prealloc == PREALLOC_MODE_METADATA) {
+        int64_t meta_size = 0;
+        unsigned nreftablee, nrefblocke, nl1e, nl2e;
+        BlockDriver *drv;
+
+        total_size = align_offset(total_size, cluster_size);
+
+        drv = bdrv_find_protocol(filename, true);
+        if (drv == NULL) {
+            error_setg(errp, "Could not find protocol for file '%s'", filename);
+            return -ENOENT;
+        }
+
+        alloc_options = append_option_parameters(alloc_options,
+                                                 drv->create_options);
+        alloc_options = append_option_parameters(alloc_options, options);
+
+        /* header: 1 cluster */
+        meta_size += cluster_size;
+
+        /* total size of L2 tables */
+        nl2e = total_size / cluster_size;
+        nl2e = align_offset(nl2e, cluster_size / sizeof(uint64_t));
+        meta_size += nl2e * sizeof(uint64_t);
+
+        /* total size of L1 tables */
+        nl1e = nl2e * sizeof(uint64_t) / cluster_size;
+        nl1e = align_offset(nl1e, cluster_size / sizeof(uint64_t));
+        meta_size += nl1e * sizeof(uint64_t);
+
+        /* total size of refcount blocks
+         *
+         * note: every host cluster is reference-counted, including metadata
+         * (even refcount blocks are recursively included).
+         * Let:
+         *   a = total_size (this is the guest disk size)
+         *   m = meta size not including refcount blocks and refcount tables
+         *   c = cluster size
+         *   y1 = number of refcount blocks entries
+         *   y2 = meta size including everything
+         * then,
+         *   y1 = (y2 + a)/c
+         *   y2 = y1 * sizeof(u16) + y1 * sizeof(u16) * sizeof(u64) / c + m
+         * we can get y1:
+         *   y1 = (a + m) / (c - sizeof(u16) - sizeof(u16) * sizeof(u64) / c)
+         */
+        nrefblocke = (total_size + meta_size + cluster_size) /
+            (cluster_size - sizeof(uint16_t) -
+             1.0 * sizeof(uint16_t) * sizeof(uint64_t) / cluster_size);
+        nrefblocke = align_offset(nrefblocke, cluster_size / sizeof(uint16_t));
+        meta_size += nrefblocke * sizeof(uint16_t);
+
+        /* total size of refcount tables */
+        nreftablee = nrefblocke * sizeof(uint16_t) / cluster_size;
+        nreftablee = align_offset(nreftablee, cluster_size / sizeof(uint64_t));
+        meta_size += nreftablee * sizeof(uint64_t);
+
+        set_option_parameter_int(alloc_options, BLOCK_OPT_SIZE,
+                                 total_size + meta_size);
+        if (prealloc == PREALLOC_MODE_FULL) {
+            set_option_parameter(alloc_options, BLOCK_OPT_PREALLOC, "full");
+        } else if (prealloc == PREALLOC_MODE_METADATA) {
+            set_option_parameter(alloc_options, BLOCK_OPT_PREALLOC, "metadata");
+        }
+
+        options = alloc_options;
+    }
+
     ret = bdrv_create_file(filename, options, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
-        return ret;
+        goto out_options;
     }
 
     bs = NULL;
@@ -1636,7 +1705,7 @@ static int qcow2_create2(const char *filename, int64_t total_size,
                     NULL, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
-        return ret;
+        goto out_options;
     }
 
     /* Write the header */
@@ -1758,6 +1827,8 @@ out:
     if (bs) {
         bdrv_unref(bs);
     }
+out_options:
+    free_option_parameters(alloc_options);
     return ret;
 }
 
@@ -1793,6 +1864,8 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options,
                 prealloc = PREALLOC_MODE_OFF;
             } else if (!strcmp(options->value.s, "metadata")) {
                 prealloc = PREALLOC_MODE_METADATA;
+            } else if (!strcmp(options->value.s, "full")) {
+                prealloc = PREALLOC_MODE_FULL;
             } else {
                 error_setg(errp, "Invalid preallocation mode: '%s'",
                            options->value.s);
@@ -2358,7 +2431,7 @@ static QEMUOptionParameter qcow2_create_options[] = {
     {
         .name = BLOCK_OPT_PREALLOC,
         .type = OPT_STRING,
-        .help = "Preallocation mode (allowed values: off, metadata)"
+        .help = "Preallocation mode (allowed values: off, metadata, full)"
     },
     {
         .name = BLOCK_OPT_LAZY_REFCOUNTS,
tests/qemu-iotests/082.out +27 -27

216 lines not on this page · raw message

Signature · 1 line
1.8.5.2.229.g4448466
#6 lore raw
In reply to Hu Tao
Quoted 12 lines · Hu Tao

The purpose of this series is to use posix_fallocate() when creating img file to ensure there are disk space for it which is way fast than acturally writing to disk. But this only works in file system level. For cases like thin provisioning, an option full preallocation is added to write zeros to storage to ensure disk space.

Hu Tao (4):
qapi: introduce PreallocMode and a new PreallocMode full. raw, qcow2: don't convert file size to sector size
raw-posix: Add full image preallocation option
qcow2: Add full image preallocation option

Quoted 6 lines
 block/qcow2.c              | 95 ++++++++++++++++++++++++++++++++++++++++------
 block/raw-posix.c          | 63 ++++++++++++++++++++++++++----
 block/raw-win32.c          |  4 +-
 qapi-schema.json           | 14 +++++++
 tests/qemu-iotests/082.out | 54 +++++++++++++-------------
 5 files changed, 182 insertions(+), 48 deletions(-)

Kevin: You had review comments on the last revision. Are you happy with v8?

Stefan