Skip to contentSkip to conversation

qcow2 info: show highest-allocation and num-free-bytes

PATCH Applied Uri Lublin 2 participants 7 messages Jan 22, 2009

lore
Share & tools
lore

Uri Lublin

author
#0 lore raw

v2 cover qcow2 info: show highest-allocation and num-free-bytes

Hello,

The following 5 enhance "info blockstats" for qcow2 images (and other formats that implement bdrv_get_info), to show higest-allocated-offset and number-of-free-bytes (below the higest-allocated-offset) of the image.

One of the main usage for this information is to know and act upon a case of end-of-storage-space. Instead of waiting for -ENOSPACE (which now may cause the VM to stop), one can define a threshold and extend the diskspace allocated for the image.
This info is also good for management systems, so they can report the user disk space status.

Also with highest_alloc and num_free_bytes, we can have a pretty good idea about how fragmented a qcow2 images is, and try to defragment it later.

Patches 1,2,3, Provides information about the highest allocated byte (offset) Patch 4, Provides information about the number of free bytes
below highest_alloc.
Patch 5, Provides info about highest_alloc and num_free_bytes through
qemu-img info

Changes from v1: better description of use cases.

Regards,
Uri.

author
#1 lore raw

PATCH v2 1/5 block-qcow2: keep highest allocated byte

We want to know the highest written offset for qcow2 images. This gives a pretty good (and easy to calculate) estimation to how much more allocation can be done for the block device.

It can be usefull for allocating more diskspace for that image (if possible, e.g. lvm) before we run out-of-disk-space

---
 block-qcow2.c |   34 ++++++++++++++++++++++++++++++++++
 1 files changed, 34 insertions(+), 0 deletions(-)
block-qcow2.c +34 -0
index d4556ef..e016e8f 100644
@@ -143,6 +143,9 @@ typedef struct BDRVQcowState {
     uint32_t crypt_method_header;
     AES_KEY aes_encrypt_key;
     AES_KEY aes_decrypt_key;
+
+    int64_t highest_alloc; /* highest cluester allocated (in clusters) */
+
     uint64_t snapshots_offset;
     int snapshots_size;
     int nb_snapshots;
@@ -170,6 +173,8 @@ static void free_clusters(BlockDriverState *bs,
 #ifdef DEBUG_ALLOC
 static void check_refcounts(BlockDriverState *bs);
 #endif
+static void scan_refcount(BlockDriverState *bs, int64_t *high);
+
 
 static int qcow_probe(const uint8_t *buf, int buf_size, const char *filename)
 {
@@ -278,6 +283,8 @@ static int qcow_open(BlockDriverState *bs, const char *filename, int flags)
     if (refcount_init(bs) < 0)
         goto fail;
 
+    scan_refcount(bs, &s->highest_alloc);
+
     /* read the backing file name */
     if (header.backing_file_offset != 0) {
         len = header.backing_file_size;
@@ -2206,6 +2213,29 @@ static int load_refcount_block(BlockDriverState *bs,
     return 0;
 }
 
+static void scan_refcount(BlockDriverState *bs, int64_t *high)
+{
+    BDRVQcowState *s = bs->opaque;
+    int64_t refcnt_index, cluster_index, cluster_end, h = 0;
+
+    for (refcnt_index=0; refcnt_index < s->refcount_table_size; refcnt_index++){
+        if (s->refcount_table[refcnt_index] == 0) {
+            continue;
+        }
+        cluster_index = refcnt_index << (s->cluster_bits - REFCOUNT_SHIFT);
+        cluster_end = (refcnt_index + 1) << (s->cluster_bits - REFCOUNT_SHIFT);
+        for ( ; cluster_index < cluster_end; cluster_index++) {
+            if (get_refcount(bs, cluster_index) == 0)
+                /* do nothing -- reserved for free counting */;
+            else
+                h = cluster_index;
+        }
+    }
+
+    if (high)
+        *high = (h+1);
+}
+
 static int get_refcount(BlockDriverState *bs, int64_t cluster_index)
 {
     BDRVQcowState *s = bs->opaque;
@@ -2246,6 +2276,10 @@ retry:
             size,
             (s->free_cluster_index - nb_clusters) << s->cluster_bits);
 #endif
+
+    if (s->highest_alloc < s->free_cluster_index)
+        s->highest_alloc = s->free_cluster_index;
+
     return (s->free_cluster_index - nb_clusters) << s->cluster_bits;
 }
 
Signature · 1 line
1.6.0.6
author
#2 lore raw

PATCH v2 2/5 block-qcow2: export highest_allocated through BlockDriverInfo and get_info()

---
 block-qcow2.c |    1 +
 block.h       |    1 +
 2 files changed, 2 insertions(+), 0 deletions(-)
block-qcow2.c +1 -0
index e016e8f..7c00ee8 100644
@@ -1671,6 +1671,7 @@ static int qcow_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
     bdi->cluster_size = s->cluster_size;
     bdi->vm_state_offset = (int64_t)s->l1_vm_state_index <<
         (s->cluster_bits + s->l2_bits);
+    bdi->highest_alloc = s->highest_alloc << s->cluster_bits;
     return 0;
 }
 
block.h +1 -0
index c3314a1..9c64af3 100644
@@ -25,6 +25,7 @@ typedef struct BlockDriverInfo {
     int cluster_size;
     /* offset at which the VM state can be saved (0 if not possible) */
     int64_t vm_state_offset;
+    int64_t highest_alloc; /* highest allocated block offset (in bytes) */
 } BlockDriverInfo;
 
 typedef struct QEMUSnapshotInfo {
Signature · 1 line
1.6.0.6
author
#3 lore raw

PATCH v2 3/5 info blockstats: show highest_allocated if exists

---
 block.c |    7 ++++++-
 1 files changed, 6 insertions(+), 1 deletions(-)
block.c +6 -1
index 3250327..0058531 100644
@@ -1077,6 +1077,7 @@ void bdrv_info(void)
 void bdrv_info_stats (void)
 {
     BlockDriverState *bs;
+    BlockDriverInfo bdi;
 
     for (bs = bdrv_first; bs != NULL; bs = bs->next) {
 	term_printf ("%s:"
@@ -1084,10 +1085,14 @@ void bdrv_info_stats (void)
 		     " wr_bytes=%" PRIu64
 		     " rd_operations=%" PRIu64
 		     " wr_operations=%" PRIu64
-		     "\n",
+                     ,
 		     bs->device_name,
 		     bs->rd_bytes, bs->wr_bytes,
 		     bs->rd_ops, bs->wr_ops);
+        if (bdrv_get_info(bs, &bdi) == 0)
+            term_printf(" high=%" PRIu64,
+                        bdi.highest_alloc);
+        term_printf("\n");
     }
 }
 
Signature · 1 line
1.6.0.6
author
#4 lore raw

PATCH v2 4/5 qcow2 format: keep 'num_free_bytes', and show it upon 'info blockstats'

'num_free_bytes' is the number of non-allocated bytes below highest-allocation. It's useful, together with the highest-allocation, to figure out how fragmented the image is, and how likely it will run out-of-space soon.

For example when the highest allocation is high (almost end-of-disk), but many bytes (clusters) are free, and can be re-allocated when neeeded, than we know it's probably not going to reach end-of-disk-space soon.

Added bookkeeping to block-qcow2.c
Export it using BlockDeviceInfo
Show it upon 'info blockstats' if BlockDeviceInfo exists

---
 block-qcow2.c |   36 ++++++++++++++++++++++++++++--------
 block.c       |    5 +++--
 block.h       |    1 +
 3 files changed, 32 insertions(+), 10 deletions(-)
block-qcow2.c +28 -8
index 7c00ee8..8a5b621 100644
@@ -145,6 +145,7 @@ typedef struct BDRVQcowState {
     AES_KEY aes_decrypt_key;
 
     int64_t highest_alloc; /* highest cluester allocated (in clusters) */
+    int64_t nc_free;       /* num of free clusters below highest_alloc */
 
     uint64_t snapshots_offset;
     int snapshots_size;
@@ -173,7 +174,7 @@ static void free_clusters(BlockDriverState *bs,
 #ifdef DEBUG_ALLOC
 static void check_refcounts(BlockDriverState *bs);
 #endif
-static void scan_refcount(BlockDriverState *bs, int64_t *high);
+static void scan_refcount(BlockDriverState *bs, int64_t *high, int64_t *free);
 
 
 static int qcow_probe(const uint8_t *buf, int buf_size, const char *filename)
@@ -283,7 +284,7 @@ static int qcow_open(BlockDriverState *bs, const char *filename, int flags)
     if (refcount_init(bs) < 0)
         goto fail;
 
-    scan_refcount(bs, &s->highest_alloc);
+    scan_refcount(bs, &s->highest_alloc, &s->nc_free);
 
     /* read the backing file name */
     if (header.backing_file_offset != 0) {
@@ -1672,6 +1673,7 @@ static int qcow_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
     bdi->vm_state_offset = (int64_t)s->l1_vm_state_index <<
         (s->cluster_bits + s->l2_bits);
     bdi->highest_alloc = s->highest_alloc << s->cluster_bits;
+    bdi->num_free_bytes = s->nc_free  << s->cluster_bits;
     return 0;
 }
 
@@ -2214,25 +2216,35 @@ static int load_refcount_block(BlockDriverState *bs,
     return 0;
 }
 
-static void scan_refcount(BlockDriverState *bs, int64_t *high)
+static void scan_refcount(BlockDriverState *bs, int64_t *high, int64_t *free)
 {
     BDRVQcowState *s = bs->opaque;
-    int64_t refcnt_index, cluster_index, cluster_end, h = 0;
+    int64_t refcnt_index, cluster_index, cluster_end, h = 0, f = 0;
+    int64_t tail = 0; /* do not count last consecutive free entries */
 
     for (refcnt_index=0; refcnt_index < s->refcount_table_size; refcnt_index++){
         if (s->refcount_table[refcnt_index] == 0) {
+            f += 1 << (s->cluster_bits - REFCOUNT_SHIFT);
+            tail += 1 << (s->cluster_bits - REFCOUNT_SHIFT);
             continue;
         }
         cluster_index = refcnt_index << (s->cluster_bits - REFCOUNT_SHIFT);
         cluster_end = (refcnt_index + 1) << (s->cluster_bits - REFCOUNT_SHIFT);
         for ( ; cluster_index < cluster_end; cluster_index++) {
-            if (get_refcount(bs, cluster_index) == 0)
-                /* do nothing -- reserved for free counting */;
-            else
+            if (get_refcount(bs, cluster_index) == 0) {
+                f++;
+                tail++;
+            }
+            else {
                 h = cluster_index;
+                tail = 0;
+            }
         }
     }
 
+    f -= tail;
+    if (free)
+        *free = f;
     if (high)
         *high = (h+1);
 }
@@ -2278,8 +2290,10 @@ retry:
             (s->free_cluster_index - nb_clusters) << s->cluster_bits);
 #endif
 
-    if (s->highest_alloc < s->free_cluster_index)
+    if (s->highest_alloc < s->free_cluster_index) {
+        s->nc_free += (s->free_cluster_index - s->highest_alloc);
         s->highest_alloc = s->free_cluster_index;
+    }
 
     return (s->free_cluster_index - nb_clusters) << s->cluster_bits;
 }
@@ -2456,6 +2470,12 @@ static int update_cluster_refcount(BlockDriverState *bs,
     block_index = cluster_index &
         ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1);
     refcount = be16_to_cpu(s->refcount_block_cache[block_index]);
+
+    if (refcount == 1 && addend == -1)
+        s->nc_free += 1;
+    else if (refcount == 0 && addend == 1)
+        s->nc_free -= 1;
+
     refcount += addend;
     if (refcount < 0 || refcount > 0xffff)
         return -EINVAL;
block.c +3 -2
index 0058531..589c67a 100644
@@ -1090,8 +1090,9 @@ void bdrv_info_stats (void)
 		     bs->rd_bytes, bs->wr_bytes,
 		     bs->rd_ops, bs->wr_ops);
         if (bdrv_get_info(bs, &bdi) == 0)
-            term_printf(" high=%" PRIu64,
-                        bdi.highest_alloc);
+            term_printf(" high=%" PRId64
+                        " bytes_free=%" PRId64,
+                        bdi.highest_alloc, bdi.num_free_bytes);
         term_printf("\n");
     }
 }
block.h +1 -0
index 9c64af3..dca17db 100644
@@ -26,6 +26,7 @@ typedef struct BlockDriverInfo {
     /* offset at which the VM state can be saved (0 if not possible) */
     int64_t vm_state_offset;
     int64_t highest_alloc; /* highest allocated block offset (in bytes) */
+    int64_t num_free_bytes; /* below highest_alloc  */
 } BlockDriverInfo;
 
 typedef struct QEMUSnapshotInfo {
Signature · 1 line
1.6.0.6
author
#5 lore raw

PATCH v2 5/5 qemu-img: info: show highest_alloc and num_free_bytes if exist

---
 qemu-img.c |    4 ++++
 1 files changed, 4 insertions(+), 0 deletions(-)
qemu-img.c +4 -0
index 555ab5f..41ff13b 100644
@@ -727,6 +727,10 @@ static int img_info(int argc, char **argv)
     if (bdrv_get_info(bs, &bdi) >= 0) {
         if (bdi.cluster_size != 0)
             printf("cluster_size: %d\n", bdi.cluster_size);
+        if (bdi.highest_alloc)
+            printf("highest_alloc: %ld\n", bdi.highest_alloc);
+        if (bdi.num_free_bytes)
+            printf("num_free_bytes: %ld\n", bdi.num_free_bytes);
     }
     bdrv_get_backing_filename(bs, backing_filename, sizeof(backing_filename));
     if (backing_filename[0] != '\0') {
Signature · 1 line
1.6.0.6
#6 lore raw
In reply to Uri Lublin
Quoted 17 lines · Uri Lublin

Hello,

The following 5 enhance "info blockstats" for qcow2 images (and
other formats that implement bdrv_get_info), to show higest-allocated-offset and number-of-free-bytes (below the higest-allocated-offset) of the image.

One of the main usage for this information is to know and act upon a case of end-of-storage-space. Instead of waiting for -ENOSPACE (which now may cause the VM to stop), one can define a threshold and extend the diskspace allocated for the image.
This info is also good for management systems, so they can report the user disk space status.

Also with highest_alloc and num_free_bytes, we can have a pretty good idea about how fragmented a qcow2 images is, and try to defragment it later.

Quoted 9 lines

Patches 1,2,3, Provides information about the highest allocated byte (offset) Patch 4, Provides information about the number of free bytes
below highest_alloc.
Patch 5, Provides info about highest_alloc and num_free_bytes through
qemu-img info

Changes from v1: better description of use cases.

Applied all. Thanks.

Regards,

Anthony Liguori

Quoted 6 lines

Regards,
Uri.