qcow2 format: keep 'num_free_bytes', and show it upon 'info blockstats'
PATCH 1/1 qcow2 format: keep 'num_free_bytes', and show it upon 'info blockstats'
'num_free_bytes' is the number of non-allocated bytes below highest-allocation.
Added bookkeeping to block-qcow2.c
Export it using BlockDeviceInfo
Show it upon 'info blockstats' if BlockDeviceInfo exists
--- block-qcow2.c | 36 ++++++++++++++++++++++++++++-------- block.c | 5 +++-- block.h | 1 + 3 files changed, 32 insertions(+), 10 deletions(-)
block-qcow2.c +28 -8
@@ -145,6 +145,7 @@ typedef struct BDRVQcowState {AES_KEY aes_decrypt_key;int64_t highest_alloc;/* highest cluester allocated (in clusters) */ +int64_t nc_free;/* num of free clusters below highest_alloc */ uint64_t snapshots_offset;int snapshots_size;
@@ -173,7 +174,7 @@ static void free_clusters(BlockDriverState *bs,#ifdef DEBUG_ALLOC static void check_refcounts (BlockDriverState *bs);#endif -static void scan_refcount (BlockDriverState *bs,int64_t *high); +static void scan_refcount (BlockDriverState *bs,int64_t *high,int64_t *free);static int qcow_probe (const uint8_t *buf,int buf_size,const char *filename)
@@ -283,7 +284,7 @@ static int qcow_open(BlockDriverState *bs, const char *filename, int flags)if (refcount_init (bs) <0 ) goto fail; -scan_refcount (bs, &s->highest_alloc); +scan_refcount (bs, &s->highest_alloc, &s->nc_free);/* read the backing file name */ if (header.backing_file_offset !=0 ) {
@@ -1672,6 +1673,7 @@ static int qcow_get_info(BlockDriverState *bs, BlockDriverInfo *bdi) bdi->vm_state_offset = (int64_t )s->l1_vm_state_index << (s->cluster_bits + s->l2_bits); bdi->highest_alloc = s->highest_alloc << s->cluster_bits; + bdi->num_free_bytes = s->nc_free << s->cluster_bits;return 0 ; }
@@ -2206,25 +2208,35 @@ static int load_refcount_block(BlockDriverState *bs,return 0 ; } -static void scan_refcount (BlockDriverState *bs,int64_t *high) +static void scan_refcount (BlockDriverState *bs,int64_t *high,int64_t *free) {BDRVQcowState *s = bs->opaque; -int64_t refcnt_index, cluster_index, cluster_end, h =0 ; +int64_t refcnt_index, cluster_index, cluster_end, h =0 , f =0 ; +int64_t tail =0 ;/* do not count last consecutive free entries */ for (refcnt_index=0 ; refcnt_index < s->refcount_table_size; refcnt_index++){if (s->refcount_table[refcnt_index] ==0 ) { + f +=1 << (s->cluster_bits -REFCOUNT_SHIFT ); + tail +=1 << (s->cluster_bits -REFCOUNT_SHIFT );continue ; } cluster_index = refcnt_index << (s->cluster_bits -REFCOUNT_SHIFT ); cluster_end = (refcnt_index +1 ) << (s->cluster_bits -REFCOUNT_SHIFT );for ( ; cluster_index < cluster_end; cluster_index++) { -if (get_refcount (bs, cluster_index) ==0 ) -/* do nothing -- reserved for free counting */ ; -else +if (get_refcount (bs, cluster_index) ==0 ) { + f++; + tail++; + } +else { h = cluster_index; + tail =0 ; + } } } + f -= tail; +if (free) + *free = f;if (high) *high = (h+1 ); }
@@ -2270,8 +2282,10 @@ retry: (s->free_cluster_index - nb_clusters) << s->cluster_bits);#endif -if (s->highest_alloc < s->free_cluster_index) +if (s->highest_alloc < s->free_cluster_index) { + s->nc_free += (s->free_cluster_index - s->highest_alloc); s->highest_alloc = s->free_cluster_index; + }return (s->free_cluster_index - nb_clusters) << s->cluster_bits; }
@@ -2448,6 +2462,12 @@ static int update_cluster_refcount(BlockDriverState *bs, block_index = cluster_index & ((1 << (s->cluster_bits -REFCOUNT_SHIFT )) -1 ); refcount =be16_to_cpu (s->refcount_block_cache[block_index]); + +if (refcount ==1 && addend ==-1 ) + s->nc_free +=1 ; +else if (refcount ==0 && addend ==1 ) + s->nc_free -=1 ; + refcount += addend;if (refcount <0 || refcount >0xffff )return -EINVAL ;
block.c +3 -2
@@ -1137,8 +1137,9 @@ void bdrv_info_stats (void) bs->rd_bytes, bs->wr_bytes, bs->rd_ops, bs->wr_ops);if (bdrv_get_info (bs, &bdi) ==0 ) -term_printf (" high=%" PRIu64, - bdi.highest_alloc); +term_printf (" high=%" PRId64 +" bytes_free=%" PRId64, + bdi.highest_alloc, bdi.num_free_bytes);term_printf ("\n" ); } }
block.h +1 -0
@@ -26,6 +26,7 @@ typedef struct BlockDriverInfo {/* offset at which the VM state can be saved (0 if not possible) */ int64_t vm_state_offset;int64_t highest_alloc;/* highest allocated block offset (in bytes) */ +int64_t num_free_bytes;/* below highest_alloc */ } BlockDriverInfo; typedef struct QEMUSnapshotInfo {
Signature · 1 line
1.6.0.6
Quoted 6 lines · Uri Lublin
'
num_free_bytes' is the number of non-allocated bytes below highest-allocation.Added bookkeeping to
block-qcow2.c
Export it using BlockDeviceInfo
Show it upon 'info blockstats' if BlockDeviceInfo exists
What is the use case for this?
Regards,
Anthony Liguori
Quoted 140 lines
Signed-off-by Uri Lublin <[email protected]>--- block-qcow2.c | 36 ++++++++++++++++++++++++++++-------- block.c | 5 +++-- block.h | 1 + 3 files changed, 32 insertions(+), 10 deletions(-)block-qcow2.c +28
-8@@ -145,6 +145,7 @@ typedef struct BDRVQcowState {AES_KEY aes_decrypt_key;int64_t highest_alloc;/* highest cluester allocated (in clusters) */ +int64_t nc_free;/* num of free clusters below highest_alloc */ uint64_t snapshots_offset;int snapshots_size;@@ -173,7 +174,7 @@ static void free_clusters(BlockDriverState *bs,#ifdef DEBUG_ALLOC static void check_refcounts (BlockDriverState *bs);#endif -static void scan_refcount (BlockDriverState *bs,int64_t *high); +static void scan_refcount (BlockDriverState *bs,int64_t *high,int64_t *free);static int qcow_probe (const uint8_t *buf,int buf_size,const char *filename)@@ -283,7 +284,7 @@ static int qcow_open(BlockDriverState *bs, const char *filename, int flags)if (refcount_init (bs) <0 ) goto fail; -scan_refcount (bs, &s->highest_alloc); +scan_refcount (bs, &s->highest_alloc, &s->nc_free);/* read the backing file name */ if (header.backing_file_offset !=0 ) {@@ -1672,6 +1673,7 @@ static int qcow_get_info(BlockDriverState *bs, BlockDriverInfo *bdi) bdi->vm_state_offset = (int64_t )s->l1_vm_state_index << (s->cluster_bits + s->l2_bits); bdi->highest_alloc = s->highest_alloc << s->cluster_bits; + bdi->num_free_bytes = s->nc_free << s->cluster_bits;return 0 ; }@@ -2206,25 +2208,35 @@ static int load_refcount_block(BlockDriverState *bs,return 0 ; } -static void scan_refcount (BlockDriverState *bs,int64_t *high) +static void scan_refcount (BlockDriverState *bs,int64_t *high,int64_t *free) {BDRVQcowState *s = bs->opaque; -int64_t refcnt_index, cluster_index, cluster_end, h =0 ; +int64_t refcnt_index, cluster_index, cluster_end, h =0 , f =0 ; +int64_t tail =0 ;/* do not count last consecutive free entries */ for (refcnt_index=0 ; refcnt_index < s->refcount_table_size; refcnt_index++){if (s->refcount_table[refcnt_index] ==0 ) { + f +=1 << (s->cluster_bits -REFCOUNT_SHIFT ); + tail +=1 << (s->cluster_bits -REFCOUNT_SHIFT );continue ; } cluster_index = refcnt_index << (s->cluster_bits -REFCOUNT_SHIFT ); cluster_end = (refcnt_index +1 ) << (s->cluster_bits -REFCOUNT_SHIFT );for ( ; cluster_index < cluster_end; cluster_index++) { -if (get_refcount (bs, cluster_index) ==0 ) -/* do nothing -- reserved for free counting */ ; -else +if (get_refcount (bs, cluster_index) ==0 ) { + f++; + tail++; + } +else { h = cluster_index; + tail =0 ; + } } } + f -= tail; +if (free) + *free = f;if (high) *high = (h+1 ); }@@ -2270,8 +2282,10 @@ retry: (s->free_cluster_index - nb_clusters) << s->cluster_bits);#endif -if (s->highest_alloc < s->free_cluster_index) +if (s->highest_alloc < s->free_cluster_index) { + s->nc_free += (s->free_cluster_index - s->highest_alloc); s->highest_alloc = s->free_cluster_index; + }return (s->free_cluster_index - nb_clusters) << s->cluster_bits; }@@ -2448,6 +2462,12 @@ static int update_cluster_refcount(BlockDriverState *bs, block_index = cluster_index & ((1 << (s->cluster_bits -REFCOUNT_SHIFT )) -1 ); refcount =be16_to_cpu (s->refcount_block_cache[block_index]); + +if (refcount ==1 && addend ==-1 ) + s->nc_free +=1 ; +else if (refcount ==0 && addend ==1 ) + s->nc_free -=1 ; + refcount += addend;if (refcount <0 || refcount >0xffff )return -EINVAL ;block.c +3
-2@@ -1137,8 +1137,9 @@ void bdrv_info_stats (void) bs->rd_bytes, bs->wr_bytes, bs->rd_ops, bs->wr_ops);if (bdrv_get_info (bs, &bdi) ==0 ) -term_printf (" high=%" PRIu64, - bdi.highest_alloc); +term_printf (" high=%" PRId64 +" bytes_free=%" PRId64, + bdi.highest_alloc, bdi.num_free_bytes);term_printf ("\n" ); } }block.h +1
-0@@ -26,6 +26,7 @@ typedef struct BlockDriverInfo {/* offset at which the VM state can be saved (0 if not possible) */ int64_t vm_state_offset;int64_t highest_alloc;/* highest allocated block offset (in bytes) */ +int64_t num_free_bytes;/* below highest_alloc */ } BlockDriverInfo; typedef struct QEMUSnapshotInfo {
Quoted 9 lines · Anthony Liguori
Quoted 6 lines · Uri Lublin
'
num_free_bytes' is the number of non-allocated bytes below highest-allocation.Added bookkeeping to
block-qcow2.c
Export it using BlockDeviceInfo
Show it upon 'info blockstats' if BlockDeviceInfo existsWhat is the use case for this?
With both highest_alloc and num_free_bytes, we can have a pretty good idea about how fragmented a qcow2 image is, and upon a threshold we can:
1. Allocate more disk space (and not wait for the disk space to run out) or
2. Defragment the image (offline: "qemu-img convert", live: yet to be implemented).
Option 1 assumes one is using LVM, or have other means for disk space allocation. This is especially important for "over subscription" of disk space, meaning the logical size of the image is larger then the disk space allocated for it (the physical size allocated for the image).
Currently num_free_bytes is usually pretty small (images grow a lot but not shrink that much). I am working on a zero-cluster optimization (originally written by Shahar Frank) which provides a way for the guest file system to "tell" qemu when blocks are freed (by writing 0's). An exception is savevm/delvm scenario (which seems to be broken; I'll try to figure out what's wrong with savevm/delvm when I'll have more time available).
Thanks,
Uri.