qemu-img: add preallocation=full
v6 cover qemu-img: add preallocation=full
This series implements full image preallocation to create a non-sparse image file at creation time, both for raw and qcow2 format. The purpose is to avoid performance deterioration of the guest cause by sparse image.
v6:
- add `Since 2.0' to PreallocMode
- apply total_size change to raw-win32.c as well
Hu Tao (4):
qapi: introduce PreallocMode and a new PreallocMode full. raw, qcow2: don't convert file size to sector size
raw-posix: Add full image preallocation option
qcow2: Add full image preallocation option
block/qcow2.c | 91 ++++++++++++++++++++++++++++++++++++++++++++++++------- block/raw-posix.c | 45 +++++++++++++++++++++------ block/raw-win32.c | 4 +-- qapi-schema.json | 14 +++++++++ 4 files changed, 132 insertions(+), 22 deletions(-)
Signature · 1 line
1.8.0
PATCH v6 1/4 qapi: introduce PreallocMode and a new PreallocMode full.
This patch prepares for the subsequent patches.
--- block/qcow2.c | 8 ++++---- qapi-schema.json | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-)
block/qcow2.c +4 -4
@@ -1454,7 +1454,7 @@ static int preallocate(BlockDriverState *bs)static int qcow2_create2 (const char *filename,int64_t total_size,const char *backing_file,const char *backing_format, -int flags,size_t cluster_size,int prealloc, +int flags,size_t cluster_size,PreallocMode prealloc,QEMUOptionParameter *options,int version,Error **errp) {
@@ -1624,7 +1624,7 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options,uint64_t sectors =0 ;int flags =0 ;size_t cluster_size =DEFAULT_CLUSTER_SIZE ; -int prealloc =0 ; +PreallocMode prealloc =PREALLOC_MODE_OFF ;int version =3 ;Error *local_err =NULL ;int ret;
@@ -1645,9 +1645,9 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options, } }else if (!strcmp (options->name,BLOCK_OPT_PREALLOC )) {if (!options->value.s || !strcmp (options->value.s,"off" )) { - prealloc =0 ; + prealloc =PREALLOC_MODE_OFF ; }else if (!strcmp (options->value.s,"metadata" )) { - prealloc =1 ; + prealloc =PREALLOC_MODE_METADATA ; }else {error_setg (errp,"Invalid preallocation mode: '%s'" , options->value.s);
qapi-schema.json +14 -0
@@ -4420,3 +4420,17 @@ # Since: 1.7 ## { 'command': 'blockdev-add', 'data': { 'options': 'BlockdevOptions' } } + +## +# @PreallocMode +# +# Preallocation mode of QEMU image file +# +# @off: no preallocation +# @metadata: preallocate only for metadata +# @full: preallocate all data, including metadata +# +# Since 2.0 +## +{ 'enum': 'PreallocMode', + 'data': [ 'off', 'metadata', 'full' ] }
Signature · 1 line
1.8.0
PATCH v6 2/4 raw, qcow2: don't convert file size to sector size
In reply to Hu Taoand avoid converting it back later.
--- block/qcow2.c | 8 ++++---- block/raw-posix.c | 4 ++-- block/raw-win32.c | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-)
block/qcow2.c +4 -4
@@ -1571,7 +1571,7 @@ static int qcow2_create2(const char *filename, int64_t total_size, }/* Okay, now that we have a valid image, let's give it the right size */ - ret =bdrv_truncate (bs, total_size *BDRV_SECTOR_SIZE ); + ret =bdrv_truncate (bs, total_size);if (ret <0 ) {error_setg_errno (errp, -ret,"Could not resize image" ); goto out;
@@ -1621,7 +1621,7 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options, {const char *backing_file =NULL ;const char *backing_fmt =NULL ; -uint64_t sectors =0 ; +uint64_t size =0 ;int flags =0 ;size_t cluster_size =DEFAULT_CLUSTER_SIZE ;PreallocMode prealloc =PREALLOC_MODE_OFF ;
@@ -1632,7 +1632,7 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options,/* Read out options */ while (options && options->name) {if (!strcmp (options->name,BLOCK_OPT_SIZE )) { - sectors = options->value.n /512 ; + size = options->value.n &BDRV_SECTOR_MASK ; }else if (!strcmp (options->name,BLOCK_OPT_BACKING_FILE )) { backing_file = options->value.s; }else if (!strcmp (options->name,BLOCK_OPT_BACKING_FMT )) {
@@ -1683,7 +1683,7 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options,return -EINVAL ; } - ret =qcow2_create2 (filename, sectors, backing_file, backing_fmt, flags, + ret =qcow2_create2 (filename, size, backing_file, backing_fmt, flags, cluster_size, prealloc, options, version, &local_err);if (error_is_set (&local_err)) {error_propagate (errp, local_err);
block/raw-posix.c +2 -2
@@ -1233,7 +1233,7 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,/* Read out options */ while (options && options->name) {if (!strcmp (options->name,BLOCK_OPT_SIZE )) { - total_size = options->value.n /BDRV_SECTOR_SIZE ; + total_size = options->value.n &BDRV_SECTOR_MASK ; } options++; }
@@ -1244,7 +1244,7 @@ static int raw_create(const char *filename, QEMUOptionParameter *options, result = -errno;error_setg_errno (errp, -result,"Could not create file" ); }else { -if (ftruncate (fd, total_size *BDRV_SECTOR_SIZE ) !=0 ) { +if (ftruncate (fd, total_size) !=0 ) { result = -errno;error_setg_errno (errp, -result,"Could not resize file" ); }
block/raw-win32.c +2 -2
@@ -473,7 +473,7 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,/* Read out options */ while (options && options->name) {if (!strcmp (options->name,BLOCK_OPT_SIZE )) { - total_size = options->value.n /512 ; + total_size = options->value.n &BDRV_SECTOR_MASK ; } options++; }
@@ -485,7 +485,7 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,return -EIO ; }set_sparse (fd); -ftruncate (fd, total_size *512 ); +ftruncate (fd, total_size);qemu_close (fd);return 0 ; }
Signature · 1 line
1.8.0
[Qemu-devel] qemu-img cannot convert 280G VHD file to qcow2 rightly
Vhd file uslinux01-1.vhd is uploaded to KVM node, the original VHD's size is 290G
But when I convert to QCOW2, I find the disk size is 127G, and then I also found the uslinux01-1.vhd disk size is 127G by qemu-img info.
Qemu version is 1.7.0 stable
[root@raknode01 disk01]# qemu-img convert -f vpc uslinux01-2.vhd -O qcow2 kvm570_data
[root@raknode01 disk01]# qemu-img info kvm570_os
image: kvm570_os
file format: qcow2
virtual size: 10.0G (10737377280 bytes)
disk size: 3.9Gcluster_size: 65536
Format specific information:
compat: 1.1
lazy refcounts: false
[root@raknode01 disk01]# qemu-img info kvm570_data
image: kvm570_data
file format: qcow2
virtual size: 127G (136899993600 bytes)
disk size: 113Mcluster_size: 65536
Format specific information:
compat: 1.1
lazy refcounts: false
[root@raknode01 disk01]# qemu-img info uslinux01-2.vhd
image: uslinux01-2.vhd
file format: vpc
virtual size: 127G (136899993600 bytes)
disk size: 5.0G
Re: [Qemu-devel] qemu-img cannot convert 280G VHD file to qcow2 rightly
Quoted 6 lines · redtone
Vhd file uslinux01-1.vhd is uploaded to KVM node, the original VHD's size is 290G
But when I convert to QCOW2, I find the disk size is 127G, and then I also found the uslinux01-1.vhd disk size is 127G by qemu-img info.Qemu version is 1.7.0 stable
This bug report is unrelated to this patch series. In the future, please start new email threads for unrelated discussions.
Quoted 2 lines
[root@raknode01 disk01]# qemu-img convert -f vpc uslinux01-2.vhd -O qcow2
kvm570_data
[...]
Quoted 6 lines
[root@raknode01 disk01]# qemu-img info
kvm570_data
image:kvm570_data
file format: qcow2
virtual size: 127G (136899993600 bytes) disk size: 113Mcluster_size: 65536
Quoted 8 lines
Format specific information:
compat: 1.1
lazy refcounts: false
[root@raknode01 disk01]# qemu-img info uslinux01-2.vhd
image: uslinux01-2.vhd
file format: vpc
virtual size: 127G (136899993600 bytes)
disk size: 5.0G
This output doesn't show anything unexpected. You converted a VHD file to qcow2. The guest sees a 127 GB virtual disk for both the VHD and the qcow2 file.
You didn't show the part where the VHD is 290 GB.
Can you post step-by-step instructions for reproducing the problem?
Thanks,
Stefan
PATCH v6 3/4 raw-posix: Add full image preallocation option
In reply to Hu TaoThis patch adds a new option preallocation for raw format, and implements full preallocation.
--- block/raw-posix.c | 43 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-)
block/raw-posix.c +35 -8
@@ -1229,11 +1229,22 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,int fd;int result =0 ;int64_t total_size =0 ; +PreallocMode prealloc =PREALLOC_MODE_OFF ;/* Read out options */ while (options && options->name) {if (!strcmp (options->name,BLOCK_OPT_SIZE )) { total_size = options->value.n &BDRV_SECTOR_MASK ; + }else if (!strcmp (options->name,BLOCK_OPT_PREALLOC )) { +if (!options->value.s || !strcmp (options->value.s,"off" )) { + prealloc =PREALLOC_MODE_OFF ; + }else if (!strcmp (options->value.s,"full" )) { + prealloc =PREALLOC_MODE_FULL ; + }else { +error_setg (errp,"Invalid preallocation mode: '%s'" , + options->value.s); +return -EINVAL ; + } } options++; }
@@ -1243,16 +1254,27 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,if (fd <0 ) { result = -errno;error_setg_errno (errp, -result,"Could not create file" ); - }else { -if (ftruncate (fd, total_size) !=0 ) { - result = -errno; -error_setg_errno (errp, -result,"Could not resize file" ); - } -if (qemu_close (fd) !=0 ) { - result = -errno; -error_setg_errno (errp, -result,"Could not close the new file" ); + goto out; + } +if (ftruncate (fd, total_size) !=0 ) { + result = -errno; +error_setg_errno (errp, -result,"Could not resize file" ); + goto out_close; + } +if (prealloc ==PREALLOC_MODE_FULL ) { +/* posix_fallocate() doesn't set errno. */ + result = -posix_fallocate (fd,0 , total_size); +if (result !=0 ) { +error_setg_errno (errp, -result, +"Could not preallocate data for the new file" ); } } +out_close: +if (qemu_close (fd) !=0 ) { + result = -errno; +error_setg_errno (errp, -result,"Could not close the new file" ); + } +out:return result; }
@@ -1403,6 +1425,11 @@ static QEMUOptionParameter raw_create_options[] = { .type = OPT_SIZE, .help ="Virtual disk size" }, + { + .name = BLOCK_OPT_PREALLOC, + .type = OPT_STRING, + .help ="Preallocation mode (allowed values: off, full)" + }, { NULL } };
Signature · 1 line
1.8.0
PATCH v6 4/4 qcow2: Add full image preallocation option
In reply to Hu TaoThis adds a preallocation=full mode to qcow2 image creation, which creates a non-sparse image file.
--- block/qcow2.c | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 3 deletions(-)
block/qcow2.c +72 -3
@@ -1458,6 +1458,7 @@ static int qcow2_create2(const char *filename, int64_t total_size, QEMUOptionParameter *options, int version, Error **errp) { + QEMUOptionParameter *alloc_options = NULL;/* Calculate cluster_bits */ int cluster_bits; cluster_bits =ffs (cluster_size) -1 ;
@@ -1487,16 +1488,80 @@ static int qcow2_create2(const char *filename, int64_t total_size,Error *local_err =NULL ;int ret; +if (prealloc ==PREALLOC_MODE_FULL ) { +int64_t meta_size =0 ; +unsigned nreftablee, nrefblocke, nl1e, nl2e; +BlockDriver *drv; + + total_size =align_offset (total_size, cluster_size); + + drv =bdrv_find_protocol (filename, true); +if (drv ==NULL ) { +error_setg (errp,"Could not find protocol for file '%s'" , filename); +return -ENOENT ; + } + + alloc_options =append_option_parameters (alloc_options, + drv->create_options); + alloc_options =append_option_parameters (alloc_options, options); + +/* header: 1 cluster */ + meta_size += cluster_size; + +/* total size of L2 tables */ + nl2e = total_size / cluster_size; + nl2e =align_offset (nl2e, cluster_size /sizeof (uint64_t )); + meta_size += nl2e *sizeof (uint64_t ); + +/* total size of L1 tables */ + nl1e = nl2e *sizeof (uint64_t ) / cluster_size; + nl1e =align_offset (nl1e, cluster_size /sizeof (uint64_t )); + meta_size += nl1e *sizeof (uint64_t ); + +/* total size of refcount blocks +* +* note: every host cluster is reference-counted, including metadata +* (even refcount blocks are recursively included). +* Let: +* a = total_size (this is the guest disk size) +* m = meta size not including refcount blocks and refcount tables +* c = cluster size +* y1 = number of refcount blocks entries +* y2 = meta size including everything +* then, +* y1 = (y2 + a)/c +* y2 = y1 * sizeof(u16) + y1 * sizeof(u16) * sizeof(u64) / c + m +* we can get y1: +* y1 = (a + m) / (c - sizeof(u16) - sizeof(u16) * sizeof(u64) / c) +*/ + nrefblocke = (total_size + meta_size + cluster_size) / + (cluster_size -sizeof (uint16_t ) - +1.0 *sizeof (uint16_t ) *sizeof (uint64_t ) / cluster_size); + nrefblocke =align_offset (nrefblocke, cluster_size /sizeof (uint16_t )); + meta_size += nrefblocke *sizeof (uint16_t ); + +/* total size of refcount tables */ + nreftablee = nrefblocke *sizeof (uint16_t ) / cluster_size; + nreftablee =align_offset (nreftablee, cluster_size /sizeof (uint64_t )); + meta_size += nreftablee *sizeof (uint64_t ); + +set_option_parameter_int (alloc_options,BLOCK_OPT_SIZE , + total_size + meta_size); +set_option_parameter (alloc_options,BLOCK_OPT_PREALLOC ,"full" ); + + options = alloc_options; + } + ret =bdrv_create_file (filename, options, &local_err);if (ret <0 ) {error_propagate (errp, local_err); -return ret; + goto out_options; } ret =bdrv_file_open (&bs, filename,NULL ,NULL ,BDRV_O_RDWR , &local_err);if (ret <0 ) {error_propagate (errp, local_err); -return ret; + goto out_options; }/* Write the header */
@@ -1613,6 +1678,8 @@ static int qcow2_create2(const char *filename, int64_t total_size, ret =0 ; out:bdrv_unref (bs); +out_options: +free_option_parameters (alloc_options);return ret; }
@@ -1648,6 +1715,8 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options, prealloc =PREALLOC_MODE_OFF ; }else if (!strcmp (options->value.s,"metadata" )) { prealloc =PREALLOC_MODE_METADATA ; + }else if (!strcmp (options->value.s,"full" )) { + prealloc =PREALLOC_MODE_FULL ; }else {error_setg (errp,"Invalid preallocation mode: '%s'" , options->value.s);
@@ -2213,7 +2282,7 @@ static QEMUOptionParameter qcow2_create_options[] = { { .name = BLOCK_OPT_PREALLOC, .type = OPT_STRING, - .help ="Preallocation mode (allowed values: off, metadata)" + .help ="Preallocation mode (allowed values: off, metadata, full)" }, { .name = BLOCK_OPT_LAZY_REFCOUNTS,
Signature · 1 line
1.8.0
Quoted 14 lines · Hu Tao
This series implements full image preallocation to create a non-sparse image file at creation time, both for raw and qcow2 format. The purpose is to avoid performance deterioration of the guest cause by sparse image.
v6:
- add `Since 2.0' to PreallocMode
- applytotal_sizechange toraw-win32.cas wellHu Tao (4):
qapi: introduce PreallocMode and a new PreallocMode full. raw, qcow2: don't convert file size to sector size
raw-posix: Add full image preallocation option
qcow2: Add full image preallocation option
Quoted 5 lines
block/qcow2.c | 91 ++++++++++++++++++++++++++++++++++++++++++++++++------- block/raw-posix.c | 45 +++++++++++++++++++++------ block/raw-win32.c | 4 +-- qapi-schema.json | 14 +++++++++ 4 files changed, 132 insertions(+), 22 deletions(-)
Quoted 15 lines · Stefan Hajnoczi
Quoted 14 lines · Hu Tao
This series implements full image preallocation to create a non-sparse image file at creation time, both for raw and qcow2 format. The purpose is to avoid performance deterioration of the guest cause by sparse image.
v6:
- add `Since 2.0' to PreallocMode
- applytotal_sizechange toraw-win32.cas wellHu Tao (4):
qapi: introduce PreallocMode and a new PreallocMode full. raw, qcow2: don't convert file size to sector size
raw-posix: Add full image preallocation option
qcow2: Add full image preallocation option
Quoted 7 lines
Quoted 5 lines
block/qcow2.c | 91 ++++++++++++++++++++++++++++++++++++++++++++++++------- block/raw-posix.c | 45 +++++++++++++++++++++------ block/raw-win32.c | 4 +-- qapi-schema.json | 14 +++++++++ 4 files changed, 132 insertions(+), 22 deletions(-)Acked-by Stefan Hajnoczi <[email protected]>
Stefan, Kevin,
Is this series in time for 2.0?
Hu
Quoted 22 lines · Hu Tao
Quoted 21 lines · Stefan Hajnoczi
Quoted 19 lines · Hu Tao
This series implements full image preallocation to create a non-sparse image file at creation time, both for raw and qcow2 format. The purpose is to avoid performance deterioration of the guest cause by sparse image.
v6:
- add `Since 2.0' to PreallocMode
- applytotal_sizechange toraw-win32.cas wellHu Tao (4):
qapi: introduce PreallocMode and a new PreallocMode full. raw, qcow2: don't convert file size to sector size
raw-posix: Add full image preallocation option
qcow2: Add full image preallocation optionblock/qcow2.c | 91 ++++++++++++++++++++++++++++++++++++++++++++++++------- block/raw-posix.c | 45 +++++++++++++++++++++------ block/raw-win32.c | 4 +-- qapi-schema.json | 14 +++++++++ 4 files changed, 132 insertions(+), 22 deletions(-)
Quoted 5 lines
Quoted 1 line
Acked-by Stefan Hajnoczi <[email protected]>Stefan, Kevin,
Is this series in time for 2.0?
Yes, it was on the mailing list before soft freeze so it is a candidate for 2.0. In some cases maintainers may decide to hold back if they feel a patch series imposes a high risk of breaking things - but I think this series should be okay.
Stefan