qemu-img: add preallocation=full
v8 cover qemu-img: add preallocation=full
The purpose of this series is to use posix_fallocate() when creating img file to ensure there are disk space for it which is way fast than acturally writing to disk. But this only works in file system level. For cases like thin provisioning, an option full preallocation is added to write zeros to storage to ensure disk space.
Hu Tao (4):
qapi: introduce PreallocMode and a new PreallocMode full. raw, qcow2: don't convert file size to sector size
raw-posix: Add full image preallocation option
qcow2: Add full image preallocation option
block/qcow2.c | 95 ++++++++++++++++++++++++++++++++++++++++------ block/raw-posix.c | 63 ++++++++++++++++++++++++++---- block/raw-win32.c | 4 +- qapi-schema.json | 14 +++++++ tests/qemu-iotests/082.out | 54 +++++++++++++------------- 5 files changed, 182 insertions(+), 48 deletions(-)
Signature · 1 line
1.8.5.2.229.g4448466
PATCH v8 1/4 qapi: introduce PreallocMode and a new PreallocMode full.
This patch prepares for the subsequent patches.
--- block/qcow2.c | 8 ++++---- qapi-schema.json | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-)
block/qcow2.c +4 -4
@@ -1592,7 +1592,7 @@ static int preallocate(BlockDriverState *bs)static int qcow2_create2 (const char *filename,int64_t total_size,const char *backing_file,const char *backing_format, -int flags,size_t cluster_size,int prealloc, +int flags,size_t cluster_size,PreallocMode prealloc,QEMUOptionParameter *options,int version,Error **errp) {
@@ -1769,7 +1769,7 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options,uint64_t sectors =0 ;int flags =0 ;size_t cluster_size =DEFAULT_CLUSTER_SIZE ; -int prealloc =0 ; +PreallocMode prealloc =PREALLOC_MODE_OFF ;int version =3 ;Error *local_err =NULL ;int ret;
@@ -1790,9 +1790,9 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options, } }else if (!strcmp (options->name,BLOCK_OPT_PREALLOC )) {if (!options->value.s || !strcmp (options->value.s,"off" )) { - prealloc =0 ; + prealloc =PREALLOC_MODE_OFF ; }else if (!strcmp (options->value.s,"metadata" )) { - prealloc =1 ; + prealloc =PREALLOC_MODE_METADATA ; }else {error_setg (errp,"Invalid preallocation mode: '%s'" , options->value.s);
qapi-schema.json +14 -0
@@ -4689,3 +4689,17 @@ 'btn' : 'InputBtnEvent', 'rel' : 'InputMoveEvent', 'abs' : 'InputMoveEvent' } } + +## +# @PreallocMode +# +# Preallocation mode of QEMU image file +# +# @off: no preallocation +# @metadata: preallocate only for metadata +# @full: preallocate all data, including metadata +# +# Since 2.0 +## +{ 'enum': 'PreallocMode', + 'data': [ 'off', 'metadata', 'full' ] }
Signature · 1 line
1.8.5.2.229.g4448466
Re: [Qemu-devel] [PATCH v8 1/4] qapi: introduce PreallocMode and a new PreallocMode full.
Quoted 16 lines · Hu Tao
This patch prepares for the subsequent patches.
Reviewed-by Fam Zheng <[email protected]>Signed-off-by Hu Tao <[email protected]>--- block/qcow2.c | 8 ++++---- qapi-schema.json | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-)+++
b/qapi-schema.jsonqapi-schema.json +1
-0@@ -4689,3 +4689,17 @@ 'btn' : 'InputBtnEvent', 'rel' : 'InputMoveEvent', 'abs' : 'InputMoveEvent' } } +
qapi-schema.json line 4702
Quoted 10 lines
qapi-schema.json +10
-0+## +# @PreallocMode +# +# Preallocation mode of QEMU image file +# +# @off: no preallocation +# @metadata: preallocate only for metadata +# @full: preallocate all data, including metadata +# +# Since 2.0
You've missed 2.0. This should be 2.1. With that fixed,
Signature · 2 lines
Eric Blake eblake redhat com +1-919-301-3266 Libvirt virtualization library http://libvirt.org
PATCH v8 2/4 raw, qcow2: don't convert file size to sector size
In reply to Hu Taoand avoid converting it back later.
--- block/qcow2.c | 8 ++++---- block/raw-posix.c | 4 ++-- block/raw-win32.c | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-)
block/qcow2.c +4 -4
@@ -1713,7 +1713,7 @@ static int qcow2_create2(const char *filename, int64_t total_size, }/* Okay, now that we have a valid image, let's give it the right size */ - ret =bdrv_truncate (bs, total_size *BDRV_SECTOR_SIZE ); + ret =bdrv_truncate (bs, total_size);if (ret <0 ) {error_setg_errno (errp, -ret,"Could not resize image" ); goto out;
@@ -1766,7 +1766,7 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options, {const char *backing_file =NULL ;const char *backing_fmt =NULL ; -uint64_t sectors =0 ; +uint64_t size =0 ;int flags =0 ;size_t cluster_size =DEFAULT_CLUSTER_SIZE ;PreallocMode prealloc =PREALLOC_MODE_OFF ;
@@ -1777,7 +1777,7 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options,/* Read out options */ while (options && options->name) {if (!strcmp (options->name,BLOCK_OPT_SIZE )) { - sectors = options->value.n /512 ; + size = options->value.n &BDRV_SECTOR_MASK ; }else if (!strcmp (options->name,BLOCK_OPT_BACKING_FILE )) { backing_file = options->value.s; }else if (!strcmp (options->name,BLOCK_OPT_BACKING_FMT )) {
@@ -1828,7 +1828,7 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options,return -EINVAL ; } - ret =qcow2_create2 (filename, sectors, backing_file, backing_fmt, flags, + ret =qcow2_create2 (filename, size, backing_file, backing_fmt, flags, cluster_size, prealloc, options, version, &local_err);if (local_err) {error_propagate (errp, local_err);
block/raw-posix.c +2 -2
@@ -1246,7 +1246,7 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,/* Read out options */ while (options && options->name) {if (!strcmp (options->name,BLOCK_OPT_SIZE )) { - total_size = options->value.n /BDRV_SECTOR_SIZE ; + total_size = options->value.n &BDRV_SECTOR_MASK ; } options++; }
@@ -1257,7 +1257,7 @@ static int raw_create(const char *filename, QEMUOptionParameter *options, result = -errno;error_setg_errno (errp, -result,"Could not create file" ); }else { -if (ftruncate (fd, total_size *BDRV_SECTOR_SIZE ) !=0 ) { +if (ftruncate (fd, total_size) !=0 ) { result = -errno;error_setg_errno (errp, -result,"Could not resize file" ); }
block/raw-win32.c +2 -2
@@ -486,7 +486,7 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,/* Read out options */ while (options && options->name) {if (!strcmp (options->name,BLOCK_OPT_SIZE )) { - total_size = options->value.n /512 ; + total_size = options->value.n &BDRV_SECTOR_MASK ; } options++; }
@@ -498,7 +498,7 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,return -EIO ; }set_sparse (fd); -ftruncate (fd, total_size *512 ); +ftruncate (fd, total_size);qemu_close (fd);return 0 ; }
Signature · 1 line
1.8.5.2.229.g4448466
PATCH v8 3/4 raw-posix: Add full image preallocation option
In reply to Hu TaoThis patch adds a new option preallocation for raw format, and implements full preallocation.
--- block/raw-posix.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 7 deletions(-)
block/raw-posix.c +54 -7
@@ -1240,6 +1240,7 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,int fd;int result =0 ;int64_t total_size =0 ; +PreallocMode prealloc =PREALLOC_MODE_OFF ;strstart (filename,"file:" , &filename);
@@ -1247,6 +1248,18 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,while (options && options->name) {if (!strcmp (options->name,BLOCK_OPT_SIZE )) { total_size = options->value.n &BDRV_SECTOR_MASK ; + }else if (!strcmp (options->name,BLOCK_OPT_PREALLOC )) { +if (!options->value.s || !strcmp (options->value.s,"off" )) { + prealloc =PREALLOC_MODE_OFF ; + }else if (!strcmp (options->value.s,"metadata" )) { + prealloc =PREALLOC_MODE_METADATA ; + }else if (!strcmp (options->value.s,"full" )) { + prealloc =PREALLOC_MODE_FULL ; + }else { +error_setg (errp,"Invalid preallocation mode: '%s'" , + options->value.s); +return -EINVAL ; + } } options++; }
@@ -1256,16 +1269,45 @@ static int raw_create(const char *filename, QEMUOptionParameter *options,if (fd <0 ) { result = -errno;error_setg_errno (errp, -result,"Could not create file" ); - }else { -if (ftruncate (fd, total_size) !=0 ) { - result = -errno; -error_setg_errno (errp, -result,"Could not resize file" ); + goto out; + } +if (ftruncate (fd, total_size) !=0 ) { + result = -errno; +error_setg_errno (errp, -result,"Could not resize file" ); + goto out_close; + } +if (prealloc ==PREALLOC_MODE_METADATA ) { +/* posix_fallocate() doesn't set errno. */ + result = -posix_fallocate (fd,0 , total_size); +if (result !=0 ) { +error_setg_errno (errp, -result, +"Could not preallocate data for the new file" ); } -if (qemu_close (fd) !=0 ) { - result = -errno; -error_setg_errno (errp, -result,"Could not close the new file" ); + }else if (prealloc ==PREALLOC_MODE_FULL ) { +char *buf =g_malloc0 (65536 ); +int64_t num =0 , left = total_size; + +while (left >0 ) { + num =MIN (left,65536 ); + result =write (fd, buf, num); +if (result <0 ) { + result = -errno; +error_setg_errno (errp, -result, +"Could not write to the new file" ); +g_free (buf); + goto out_close; + } + left -= num; } +fsync (fd); +g_free (buf); + } +out_close: +if (qemu_close (fd) !=0 ) { + result = -errno; +error_setg_errno (errp, -result,"Could not close the new file" ); } +out:return result; }
@@ -1416,6 +1458,11 @@ static QEMUOptionParameter raw_create_options[] = { .type = OPT_SIZE, .help ="Virtual disk size" }, + { + .name = BLOCK_OPT_PREALLOC, + .type = OPT_STRING, + .help ="Preallocation mode (allowed values: off, metadata, full)" + }, { NULL } };
Signature · 1 line
1.8.5.2.229.g4448466
PATCH v8 4/4 qcow2: Add full image preallocation option
In reply to Hu TaoThis adds a preallocation=full mode to qcow2 image creation, which creates a non-sparse image file.
--- block/qcow2.c | 79 ++++++++++++++++++++++++++++++++++++++++++++-- tests/qemu-iotests/082.out | 54 +++++++++++++++---------------- 2 files changed, 103 insertions(+), 30 deletions(-)
block/qcow2.c +76 -3
@@ -1596,6 +1596,7 @@ static int qcow2_create2(const char *filename, int64_t total_size, QEMUOptionParameter *options, int version, Error **errp) { + QEMUOptionParameter *alloc_options = NULL;/* Calculate cluster_bits */ int cluster_bits; cluster_bits =ffs (cluster_size) -1 ;
@@ -1625,10 +1626,78 @@ static int qcow2_create2(const char *filename, int64_t total_size,Error *local_err =NULL ;int ret; +if (prealloc ==PREALLOC_MODE_FULL || prealloc ==PREALLOC_MODE_METADATA ) { +int64_t meta_size =0 ; +unsigned nreftablee, nrefblocke, nl1e, nl2e; +BlockDriver *drv; + + total_size =align_offset (total_size, cluster_size); + + drv =bdrv_find_protocol (filename, true); +if (drv ==NULL ) { +error_setg (errp,"Could not find protocol for file '%s'" , filename); +return -ENOENT ; + } + + alloc_options =append_option_parameters (alloc_options, + drv->create_options); + alloc_options =append_option_parameters (alloc_options, options); + +/* header: 1 cluster */ + meta_size += cluster_size; + +/* total size of L2 tables */ + nl2e = total_size / cluster_size; + nl2e =align_offset (nl2e, cluster_size /sizeof (uint64_t )); + meta_size += nl2e *sizeof (uint64_t ); + +/* total size of L1 tables */ + nl1e = nl2e *sizeof (uint64_t ) / cluster_size; + nl1e =align_offset (nl1e, cluster_size /sizeof (uint64_t )); + meta_size += nl1e *sizeof (uint64_t ); + +/* total size of refcount blocks +* +* note: every host cluster is reference-counted, including metadata +* (even refcount blocks are recursively included). +* Let: +* a = total_size (this is the guest disk size) +* m = meta size not including refcount blocks and refcount tables +* c = cluster size +* y1 = number of refcount blocks entries +* y2 = meta size including everything +* then, +* y1 = (y2 + a)/c +* y2 = y1 * sizeof(u16) + y1 * sizeof(u16) * sizeof(u64) / c + m +* we can get y1: +* y1 = (a + m) / (c - sizeof(u16) - sizeof(u16) * sizeof(u64) / c) +*/ + nrefblocke = (total_size + meta_size + cluster_size) / + (cluster_size -sizeof (uint16_t ) - +1.0 *sizeof (uint16_t ) *sizeof (uint64_t ) / cluster_size); + nrefblocke =align_offset (nrefblocke, cluster_size /sizeof (uint16_t )); + meta_size += nrefblocke *sizeof (uint16_t ); + +/* total size of refcount tables */ + nreftablee = nrefblocke *sizeof (uint16_t ) / cluster_size; + nreftablee =align_offset (nreftablee, cluster_size /sizeof (uint64_t )); + meta_size += nreftablee *sizeof (uint64_t ); + +set_option_parameter_int (alloc_options,BLOCK_OPT_SIZE , + total_size + meta_size); +if (prealloc ==PREALLOC_MODE_FULL ) { +set_option_parameter (alloc_options,BLOCK_OPT_PREALLOC ,"full" ); + }else if (prealloc ==PREALLOC_MODE_METADATA ) { +set_option_parameter (alloc_options,BLOCK_OPT_PREALLOC ,"metadata" ); + } + + options = alloc_options; + } + ret =bdrv_create_file (filename, options, &local_err);if (ret <0 ) {error_propagate (errp, local_err); -return ret; + goto out_options; } bs =NULL ;
@@ -1636,7 +1705,7 @@ static int qcow2_create2(const char *filename, int64_t total_size, NULL, &local_err);if (ret <0 ) {error_propagate (errp, local_err); -return ret; + goto out_options; }/* Write the header */
@@ -1758,6 +1827,8 @@ out:if (bs) {bdrv_unref (bs); } +out_options: +free_option_parameters (alloc_options);return ret; }
@@ -1793,6 +1864,8 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options, prealloc =PREALLOC_MODE_OFF ; }else if (!strcmp (options->value.s,"metadata" )) { prealloc =PREALLOC_MODE_METADATA ; + }else if (!strcmp (options->value.s,"full" )) { + prealloc =PREALLOC_MODE_FULL ; }else {error_setg (errp,"Invalid preallocation mode: '%s'" , options->value.s);
@@ -2358,7 +2431,7 @@ static QEMUOptionParameter qcow2_create_options[] = { { .name = BLOCK_OPT_PREALLOC, .type = OPT_STRING, - .help ="Preallocation mode (allowed values: off, metadata)" + .help ="Preallocation mode (allowed values: off, metadata, full)" }, { .name = BLOCK_OPT_LAZY_REFCOUNTS,
tests/qemu-iotests/082.out +27 -27
216 lines not on this page · raw message
Signature · 1 line
1.8.5.2.229.g4448466
Quoted 12 lines · Hu Tao
The purpose of this series is to use
posix_fallocate()when creating img file to ensure there are disk space for it which is way fast than acturally writing to disk. But this only works in file system level. For cases like thin provisioning, an option full preallocation is added to write zeros to storage to ensure disk space.Hu Tao (4):
qapi: introduce PreallocMode and a new PreallocMode full. raw, qcow2: don't convert file size to sector size
raw-posix: Add full image preallocation option
qcow2: Add full image preallocation option
Quoted 6 lines
block/qcow2.c | 95 ++++++++++++++++++++++++++++++++++++++++------ block/raw-posix.c | 63 ++++++++++++++++++++++++++---- block/raw-win32.c | 4 +- qapi-schema.json | 14 +++++++ tests/qemu-iotests/082.out | 54 +++++++++++++------------- 5 files changed, 182 insertions(+), 48 deletions(-)
Kevin: You had review comments on the last revision. Are you happy with v8?
Stefan