qcow2: Add preallocation=full option
cover qcow2: Add preallocation=full option
This series impelments preallocation=full, using posix_fallocate() based on Kevin's original patch at:
http://lists.gnu.org/archive/html/qemu-devel/2011-01/msg03017.html
Hu Tao (4):
block: add BlockDriver.bdrv_zero_init.
block/raw-posix: implement bdrv_zero_init
qcow2: implement bdrv_zero_init
qcow2: Add full image preallocation option
block.c | 13 +++++++++++++ block/qcow2.c | 35 +++++++++++++++++++++++++++++------ block/raw-posix.c | 13 +++++++++++++ include/block/block.h | 1 + include/block/block_int.h | 2 ++ 5 files changed, 58 insertions(+), 6 deletions(-)
Signature · 1 line
1.8.3.1
RFC PATCH 1/4 block: add BlockDriver.bdrv_zero_init.
This field is used to zero-initialize block device. It can be used to preallocate space for the block device.
--- block.c | 13 +++++++++++++ include/block/block.h | 1 + include/block/block_int.h | 2 ++ 3 files changed, 16 insertions(+)
block.c +13 -0
@@ -3095,6 +3095,19 @@ int bdrv_has_zero_init(BlockDriverState *bs)return 0 ; } +int bdrv_zero_init (BlockDriverState *bs,int64_t offset,int64_t length) +{ +if (bs->backing_hd) { +return -1 ; + } + +if (bs->drv->bdrv_zero_init) { +return bs->drv->bdrv_zero_init (bs, offset, length); + } + +return -1 ; +} + typedef struct BdrvCoGetBlockStatusData { BlockDriverState *bs; BlockDriverState *base;
include/block/block.h +1 -0
@@ -315,6 +315,7 @@ void bdrv_drain_all(void);int bdrv_discard (BlockDriverState *bs,int64_t sector_num,int nb_sectors);int bdrv_co_discard (BlockDriverState *bs,int64_t sector_num,int nb_sectors);int bdrv_has_zero_init_1 (BlockDriverState *bs); +int bdrv_zero_init (BlockDriverState *bs,int64_t offset,int64_t length);int bdrv_has_zero_init (BlockDriverState *bs);int64_t bdrv_get_block_status (BlockDriverState *bs,int64_t sector_num,int nb_sectors,int *pnum);
include/block/block_int.h +2 -0
@@ -227,6 +227,8 @@ struct BlockDriver {*/ int (*bdrv_has_zero_init)(BlockDriverState *bs); +int (*bdrv_zero_init)(BlockDriverState *bs,int64_t offset,int64_t length); +QLIST_ENTRY (BlockDriver) list; };
Signature · 1 line
1.8.3.1
RFC PATCH 2/4 block/raw-posix: implement bdrv_zero_init
In reply to Hu TaoImplement bdrv_zero_init using posix_fallocate.
--- block/raw-posix.c | 13 +++++++++++++ 1 file changed, 13 insertions(+)
block/raw-posix.c +13 -0
@@ -1190,6 +1190,18 @@ static int64_t coroutine_fn raw_co_get_block_status(BlockDriverState *bs,return ret; } +static int raw_zero_init (BlockDriverState *bs,int64_t offset,int64_t length) +{ +BDRVRawState *s = bs->opaque; +int64_t len =bdrv_getlength (bs); + +if (offset + length <0 || offset + length > len) { +return -1 ; + } + +return posix_fallocate (s->fd, offset, length); +} + static coroutine_fn BlockDriverAIOCB *raw_aio_discard(BlockDriverState *bs,int64_t sector_num,int nb_sectors,BlockDriverCompletionFunc *cb,void *opaque)
@@ -1222,6 +1234,7 @@ static BlockDriver bdrv_file = { .bdrv_close = raw_close, .bdrv_create = raw_create, .bdrv_has_zero_init = bdrv_has_zero_init_1, + .bdrv_zero_init = raw_zero_init, .bdrv_co_get_block_status = raw_co_get_block_status, .bdrv_aio_readv = raw_aio_readv,
Signature · 1 line
1.8.3.1
Re: [Qemu-devel] [RFC PATCH 2/4] block/raw-posix: implement bdrv_zero_init
Am 12.11.2013 um 08:47 hat Hu Tao geschrieben:
Quoted 16 lines
Implement
bdrv_zero_initusingposix_fallocate.Signed-off-by Hu Tao <[email protected]>--- block/raw-posix.c | 13 +++++++++++++ 1 file changed, 13 insertions(+)block/raw-posix.c +1
-0@@ -1190,6 +1190,18 @@ static int64_t coroutine_fn raw_co_get_block_status(BlockDriverState *bs,return ret; } +static int raw_zero_init (BlockDriverState *bs,int64_t offset,int64_t length)
block/raw-posix.c static int64_t coroutine_fn raw_co_get_block_status(BlockDriverState *bs, · line 1203
Quoted 10 lines
block/raw-posix.c +10
-0+{ +BDRVRawState *s = bs->opaque; +int64_t len =bdrv_getlength (bs); + +if (offset + length <0 || offset + length > len) { +return -1 ; + } + +return posix_fallocate (s->fd, offset, length); +}
This doesn't really initialise anything to zero. It merely preallocates those parts of a file that aren't allocated yet (and they happen to be zeroed in this case), but leaves already existing parts untouched.
I wonder if this would be a correct implementation for a bdrv_anchor(), though. I also wouldn't call that full preallocation, but it might be useful anyway.
Kevin
Re: [Qemu-devel] [RFC PATCH 2/4] block/raw-posix: implement bdrv_zero_init
Quoted 16 lines · Kevin Wolf
Am 12.11.2013 um 08:47 hat Hu Tao geschrieben:
Quoted 15 lines
Implement
bdrv_zero_initusingposix_fallocate.Signed-off-by Hu Tao <[email protected]>--- block/raw-posix.c | 13 +++++++++++++ 1 file changed, 13 insertions(+)block/raw-posix.c +0
-0@@ -1190,6 +1190,18 @@ static int64_t coroutine_fn raw_co_get_block_status(BlockDriverState *bs,return ret; }
Quoted 15 lines
block/raw-posix.c static int64_t coroutine_fn raw_co_get_block_status(BlockDriverState *bs, · line 1203
Quoted 11 lines
block/raw-posix.c +11
-0+static int raw_zero_init (BlockDriverState *bs,int64_t offset,int64_t length) +{ +BDRVRawState *s = bs->opaque; +int64_t len =bdrv_getlength (bs); + +if (offset + length <0 || offset + length > len) { +return -1 ; + } + +return posix_fallocate (s->fd, offset, length); +}This doesn't really initialise anything to zero. It merely preallocates those parts of a file that aren't allocated yet (and they happen to be zeroed in this case), but leaves already existing parts untouched.
Then the name is inappropriate, how about bdrv_preallocate()?
Quoted 2 lines
I wonder if this would be a correct implementation for a
bdrv_anchor(),
Why?
Quoted 2 lines
though. I also wouldn't call that full preallocation, but it might be useful anyway.
Wouldn't bdrv_preallocate(bs, 0, bdrv_getlength(bs)) be full preallocation?
RFC PATCH 3/4 qcow2: implement bdrv_zero_init
In reply to Hu Tao--- block/qcow2.c | 7 +++++++ 1 file changed, 7 insertions(+)
block/qcow2.c +7 -0
@@ -2180,6 +2180,12 @@ static int qcow2_amend_options(BlockDriverState *bs,return 0 ; } +static int qcow2_zero_init (BlockDriverState *bs,int64_t offset, +int64_t length) +{ +return bdrv_zero_init (bs->file, offset, length); +} +static QEMUOptionParameter qcow2_create_options[] = { { .name =BLOCK_OPT_SIZE ,
@@ -2234,6 +2240,7 @@ static BlockDriver bdrv_qcow2 = { .bdrv_reopen_prepare = qcow2_reopen_prepare, .bdrv_create = qcow2_create, .bdrv_has_zero_init = bdrv_has_zero_init_1, + .bdrv_zero_init = qcow2_zero_init, .bdrv_co_get_block_status = qcow2_co_get_block_status, .bdrv_set_key = qcow2_set_key, .bdrv_make_empty = qcow2_make_empty,
Signature · 1 line
1.8.3.1
RFC PATCH 4/4 qcow2: Add full image preallocation option
In reply to Hu TaoThis adds a preallocation=full mode to qcow2 image creation, which creates a non-sparse image file.
--- block/qcow2.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-)
block/qcow2.c +22 -6
@@ -1385,7 +1385,13 @@ static int qcow2_change_backing_file(BlockDriverState *bs,return qcow2_update_header (bs); } -static int preallocate (BlockDriverState *bs) +enum prealloc_mode { +PREALLOC_OFF =0 , +PREALLOC_METADATA , +PREALLOC_FULL , +}; + +static int preallocate (BlockDriverState *bs,enum prealloc_mode mode) {uint64_t nb_sectors;uint64_t offset;
@@ -1394,9 +1400,12 @@ static int preallocate(BlockDriverState *bs)int ret;QCowL2Meta *meta; +assert (mode !=PREALLOC_OFF ); + nb_sectors =bdrv_getlength (bs) >>9 ; offset =0 ; +/* First allocate metadata in _really_ big chunks */ while (nb_sectors) { num =MIN (nb_sectors,INT_MAX >>9 ); ret = qcow2_alloc_cluster_offset(bs, offset, 0, num, &num,
@@ -1424,6 +1433,11 @@ static int preallocate(BlockDriverState *bs) offset += num <<9 ; } +/* Then write zeros to the cluster data, if requested */ +if (mode ==PREALLOC_FULL ) { +bdrv_zero_init (bs->file, offset,bdrv_getlength (bs)); + } +/* * It is expected that the image file is large enough to actually contain * all of the allocated clusters (otherwise we get failing reads after
@@ -1572,11 +1586,11 @@ static int qcow2_create2(const char *filename, int64_t total_size, } } -/* And if we're supposed to preallocate metadata, do that now */ +/* And if we're supposed to preallocate data, do that now */ if (prealloc) {BDRVQcowState *s = bs->opaque;qemu_co_mutex_lock (&s->lock); - ret =preallocate (bs); + ret =preallocate (bs, prealloc);qemu_co_mutex_unlock (&s->lock);if (ret <0 ) {error_setg_errno (errp, -ret,"Could not preallocate metadata" );
@@ -1629,9 +1643,11 @@ static int qcow2_create(const char *filename, QEMUOptionParameter *options, } }else if (!strcmp (options->name,BLOCK_OPT_PREALLOC )) {if (!options->value.s || !strcmp (options->value.s,"off" )) { - prealloc =0 ; + prealloc =PREALLOC_OFF ; }else if (!strcmp (options->value.s,"metadata" )) { - prealloc =1 ; + prealloc =PREALLOC_METADATA ; + }else if (!strcmp (options->value.s,"full" )) { + prealloc =PREALLOC_FULL ; }else {error_setg (errp,"Invalid preallocation mode: '%s'" , options->value.s);
@@ -2221,7 +2237,7 @@ static QEMUOptionParameter qcow2_create_options[] = { { .name = BLOCK_OPT_PREALLOC, .type = OPT_STRING, - .help ="Preallocation mode (allowed values: off, metadata)" + .help ="Preallocation mode (allowed values: off, metadata, full)" }, { .name = BLOCK_OPT_LAZY_REFCOUNTS,
Signature · 1 line
1.8.3.1
Re: [Qemu-devel] [RFC PATCH 4/4] qcow2: Add full image preallocation option
Am 12.11.2013 um 08:47 hat Hu Tao geschrieben:
Quoted 41 lines
This adds a preallocation=full mode to qcow2 image creation, which creates a non-sparse image file.
Signed-off-by Hu Tao <[email protected]>--- block/qcow2.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-)block/qcow2.c +10
-1@@ -1385,7 +1385,13 @@ static int qcow2_change_backing_file(BlockDriverState *bs,return qcow2_update_header (bs); } -static int preallocate (BlockDriverState *bs) +enum prealloc_mode { +PREALLOC_OFF =0 , +PREALLOC_METADATA , +PREALLOC_FULL , +}; + +static int preallocate (BlockDriverState *bs,enum prealloc_mode mode) {uint64_t nb_sectors;uint64_t offset;@@ -1394,9 +1400,12 @@ static int preallocate(BlockDriverState *bs)int ret;QCowL2Meta *meta; +assert (mode !=PREALLOC_OFF ); + nb_sectors =bdrv_getlength (bs) >>9 ; offset =0 ; +/* First allocate metadata in _really_ big chunks */ while (nb_sectors) { num =MIN (nb_sectors,INT_MAX >>9 ); ret = qcow2_alloc_cluster_offset(bs, offset, 0, num, &num,
block/qcow2.c static int preallocate(BlockDriverState *bs) · line 1439
Quoted 7 lines
block/qcow2.c +4
-0@@ -1424,6 +1433,11 @@ static int preallocate(BlockDriverState *bs) offset += num <<9 ; } +/* Then write zeros to the cluster data, if requested */ +if (mode ==PREALLOC_FULL ) { +bdrv_zero_init (bs->file, offset,bdrv_getlength (bs)); + }
bdrv_zero_init() is completely optional. It is only implemented for raw-posix and errors are not checked. You can't assume that after this point anything is preallocated.
Kevin
Re: [Qemu-devel] [RFC PATCH 4/4] qcow2: Add full image preallocation option
Quoted 45 lines · Kevin Wolf
Am 12.11.2013 um 08:47 hat Hu Tao geschrieben:
Quoted 44 lines
This adds a preallocation=full mode to qcow2 image creation, which creates a non-sparse image file.
Signed-off-by Hu Tao <[email protected]>--- block/qcow2.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-)block/qcow2.c +10
-1@@ -1385,7 +1385,13 @@ static int qcow2_change_backing_file(BlockDriverState *bs,return qcow2_update_header (bs); } -static int preallocate (BlockDriverState *bs) +enum prealloc_mode { +PREALLOC_OFF =0 , +PREALLOC_METADATA , +PREALLOC_FULL , +}; + +static int preallocate (BlockDriverState *bs,enum prealloc_mode mode) {uint64_t nb_sectors;uint64_t offset;@@ -1394,9 +1400,12 @@ static int preallocate(BlockDriverState *bs)int ret;QCowL2Meta *meta; +assert (mode !=PREALLOC_OFF ); + nb_sectors =bdrv_getlength (bs) >>9 ; offset =0 ; +/* First allocate metadata in _really_ big chunks */ while (nb_sectors) { num =MIN (nb_sectors,INT_MAX >>9 ); ret = qcow2_alloc_cluster_offset(bs, offset, 0, num, &num,@@ -1424,6 +1433,11 @@ static int preallocate(BlockDriverState *bs) offset += num <<9 ; }
Quoted 8 lines
block/qcow2.c static int preallocate(BlockDriverState *bs) · line 1439
Quoted 4 lines
block/qcow2.c +4
-0+/* Then write zeros to the cluster data, if requested */ +if (mode ==PREALLOC_FULL ) { +bdrv_zero_init (bs->file, offset,bdrv_getlength (bs)); + }
bdrv_zero_init()is completely optional. It is only implemented for raw-posix and errors are not checked. You can't assume that after this point anything is preallocated.
Yes, errores should have been checked here.
Re: [Qemu-devel] [RFC PATCH 4/4] qcow2: Add full image preallocation option
What is your use case for this seris? QCOW2 creation or converting anything to QCOW2? For the later case you could use "qemu-img convert -S 0 ..."
starting in 1.8.
Peter
Quoted 64 lines · Hu Tao
This adds a preallocation=full mode to qcow2 image creation, which creates a non-sparse image file.
Signed-off-by Hu Tao <[email protected]>--- block/qcow2.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-)block/qcow2.c +7
-1@@ -1385,7 +1385,13 @@ static intqcow2_change_backing_file (BlockDriverState *bs, return qcow2_update_header(bs); } -static int preallocate (BlockDriverState *bs) +enum prealloc_mode { +PREALLOC_OFF =0 , +PREALLOC_METADATA , +PREALLOC_FULL , +}; + +static int preallocate (BlockDriverState *bs,enum prealloc_mode mode) {uint64_t nb_sectors;uint64_t offset;block/qcow2.c +8
-0@@ -1394,9 +1400,12 @@ static int preallocate(BlockDriverState *bs)int ret;QCowL2Meta *meta; +assert (mode !=PREALLOC_OFF ); + nb_sectors =bdrv_getlength (bs) >>9 ; offset =0 ; +/* First allocate metadata in _really_ big chunks */ while (nb_sectors) { num =MIN (nb_sectors,INT_MAX >>9 ); ret = qcow2_alloc_cluster_offset(bs, offset, 0, num, &num,@@ -1424,6 +1433,11 @@ static int preallocate(BlockDriverState *bs) offset += num <<9 ; } +/* Then write zeros to the cluster data, if requested */ +if (mode ==PREALLOC_FULL ) { +bdrv_zero_init (bs->file, offset,bdrv_getlength (bs)); + } +/* * It is expected that the image file is large enough to actually contain
* all of the allocated clusters (otherwise we get failing reads
afterblock/qcow2.c +1
-1@@ -1572,11 +1586,11 @@ static int qcow2_create2(const char *filename, int64_t total_size, } } -/* And if we're supposed to preallocate metadata, do that now */ +/* And if we're supposed to preallocate data, do that now */ if (prealloc) { BDRVQcowState *
Re: [Qemu-devel] [RFC PATCH 4/4] qcow2: Add full image preallocation option
Quoted 4 lines · Peter Lieven
What is your use case for this seris? QCOW2 creation or converting anything to QCOW2? For the later case you could use "qemu-img convert -S 0 ..."
starting in 1.8.
The former. Seems "qemu-img convert -S 0" do the same thing but at converting time, maybe we can share some code. But let me dig the code first.
Quoted 69 lines
Peter
Quoted 64 lines · Hu Tao
This adds a preallocation=full mode to qcow2 image creation, which creates a non-sparse image file.
Signed-off-by Hu Tao <[email protected]>--- block/qcow2.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-)block/qcow2.c +7
-1@@ -1385,7 +1385,13 @@ static intqcow2_change_backing_file (BlockDriverState *bs, return qcow2_update_header(bs); } -static int preallocate (BlockDriverState *bs) +enum prealloc_mode { +PREALLOC_OFF =0 , +PREALLOC_METADATA , +PREALLOC_FULL , +}; + +static int preallocate (BlockDriverState *bs,enum prealloc_mode mode) {uint64_t nb_sectors;uint64_t offset;block/qcow2.c +8
-0@@ -1394,9 +1400,12 @@ static int preallocate(BlockDriverState *bs)int ret;QCowL2Meta *meta; +assert (mode !=PREALLOC_OFF ); + nb_sectors =bdrv_getlength (bs) >>9 ; offset =0 ; +/* First allocate metadata in _really_ big chunks */ while (nb_sectors) { num =MIN (nb_sectors,INT_MAX >>9 ); ret = qcow2_alloc_cluster_offset(bs, offset, 0, num, &num,@@ -1424,6 +1433,11 @@ static int preallocate(BlockDriverState *bs) offset += num <<9 ; } +/* Then write zeros to the cluster data, if requested */ +if (mode ==PREALLOC_FULL ) { +bdrv_zero_init (bs->file, offset,bdrv_getlength (bs)); + } +/* * It is expected that the image file is large enough to actually contain
* all of the allocated clusters (otherwise we get failing reads
afterblock/qcow2.c +1
-1@@ -1572,11 +1586,11 @@ static int qcow2_create2(const char *filename, int64_t total_size, } } -/* And if we're supposed to preallocate metadata, do that now */ +/* And if we're supposed to preallocate data, do that now */ if (prealloc) { BDRVQcowState *
Re: [Qemu-devel] [RFC PATCH 4/4] qcow2: Add full image preallocation option
Quoted 9 lines · Hu Tao
Quoted 4 lines · Peter Lieven
What is your use case for this seris? QCOW2 creation or converting anything to QCOW2? For the later case you could use "qemu-img convert -S 0 ..."
starting in 1.8.The former. Seems "qemu-img convert -S 0" do the same thing but at converting time, maybe we can share some code. But let me dig the code first.
The purpose is to using posix_fallocate() to preallocate space, other than writing zeros to do it. Because the latter is pretty slow.
Quoted 70 lines
Quoted 70 lines
Peter
Quoted 64 lines · Hu Tao
This adds a preallocation=full mode to qcow2 image creation, which creates a non-sparse image file.
Signed-off-by Hu Tao <[email protected]>--- block/qcow2.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-)block/qcow2.c +7
-1@@ -1385,7 +1385,13 @@ static intqcow2_change_backing_file (BlockDriverState *bs, return qcow2_update_header(bs); } -static int preallocate (BlockDriverState *bs) +enum prealloc_mode { +PREALLOC_OFF =0 , +PREALLOC_METADATA , +PREALLOC_FULL , +}; + +static int preallocate (BlockDriverState *bs,enum prealloc_mode mode) {uint64_t nb_sectors;uint64_t offset;block/qcow2.c +8
-0@@ -1394,9 +1400,12 @@ static int preallocate(BlockDriverState *bs)int ret;QCowL2Meta *meta; +assert (mode !=PREALLOC_OFF ); + nb_sectors =bdrv_getlength (bs) >>9 ; offset =0 ; +/* First allocate metadata in _really_ big chunks */ while (nb_sectors) { num =MIN (nb_sectors,INT_MAX >>9 ); ret = qcow2_alloc_cluster_offset(bs, offset, 0, num, &num,@@ -1424,6 +1433,11 @@ static int preallocate(BlockDriverState *bs) offset += num <<9 ; } +/* Then write zeros to the cluster data, if requested */ +if (mode ==PREALLOC_FULL ) { +bdrv_zero_init (bs->file, offset,bdrv_getlength (bs)); + } +/* * It is expected that the image file is large enough to actually contain
* all of the allocated clusters (otherwise we get failing reads
afterblock/qcow2.c +1
-1@@ -1572,11 +1586,11 @@ static int qcow2_create2(const char *filename, int64_t total_size, } } -/* And if we're supposed to preallocate metadata, do that now */ +/* And if we're supposed to preallocate data, do that now */ if (prealloc) { BDRVQcowState *