Skip to contentSkip to conversation

qcow2 format: keep 'num_free_bytes', and show it upon 'info blockstats'

PATCH In review Uri Lublin 2 participants 3 messages Jan 13, 2009 – Jan 18, 2009

lore
Share & tools
lore

Uri Lublin

author
#0 lore raw

PATCH 1/1 qcow2 format: keep 'num_free_bytes', and show it upon 'info blockstats'

'num_free_bytes' is the number of non-allocated bytes below highest-allocation.

Added bookkeeping to block-qcow2.c
Export it using BlockDeviceInfo
Show it upon 'info blockstats' if BlockDeviceInfo exists

---
 block-qcow2.c |   36 ++++++++++++++++++++++++++++--------
 block.c       |    5 +++--
 block.h       |    1 +
 3 files changed, 32 insertions(+), 10 deletions(-)
block-qcow2.c +28 -8
index b8ef825..f16186e 100644
@@ -145,6 +145,7 @@ typedef struct BDRVQcowState {
     AES_KEY aes_decrypt_key;
 
     int64_t highest_alloc; /* highest cluester allocated (in clusters) */
+    int64_t nc_free;       /* num of free clusters below highest_alloc */
 
     uint64_t snapshots_offset;
     int snapshots_size;
@@ -173,7 +174,7 @@ static void free_clusters(BlockDriverState *bs,
 #ifdef DEBUG_ALLOC
 static void check_refcounts(BlockDriverState *bs);
 #endif
-static void scan_refcount(BlockDriverState *bs, int64_t *high);
+static void scan_refcount(BlockDriverState *bs, int64_t *high, int64_t *free);
 
 
 static int qcow_probe(const uint8_t *buf, int buf_size, const char *filename)
@@ -283,7 +284,7 @@ static int qcow_open(BlockDriverState *bs, const char *filename, int flags)
     if (refcount_init(bs) < 0)
         goto fail;
 
-    scan_refcount(bs, &s->highest_alloc);
+    scan_refcount(bs, &s->highest_alloc, &s->nc_free);
 
     /* read the backing file name */
     if (header.backing_file_offset != 0) {
@@ -1672,6 +1673,7 @@ static int qcow_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
     bdi->vm_state_offset = (int64_t)s->l1_vm_state_index <<
         (s->cluster_bits + s->l2_bits);
     bdi->highest_alloc = s->highest_alloc << s->cluster_bits;
+    bdi->num_free_bytes = s->nc_free  << s->cluster_bits;
     return 0;
 }
 
@@ -2206,25 +2208,35 @@ static int load_refcount_block(BlockDriverState *bs,
     return 0;
 }
 
-static void scan_refcount(BlockDriverState *bs, int64_t *high)
+static void scan_refcount(BlockDriverState *bs, int64_t *high, int64_t *free)
 {
     BDRVQcowState *s = bs->opaque;
-    int64_t refcnt_index, cluster_index, cluster_end, h = 0;
+    int64_t refcnt_index, cluster_index, cluster_end, h = 0, f = 0;
+    int64_t tail = 0; /* do not count last consecutive free entries */
 
     for (refcnt_index=0; refcnt_index < s->refcount_table_size; refcnt_index++){
         if (s->refcount_table[refcnt_index] == 0) {
+            f += 1 << (s->cluster_bits - REFCOUNT_SHIFT);
+            tail += 1 << (s->cluster_bits - REFCOUNT_SHIFT);
             continue;
         }
         cluster_index = refcnt_index << (s->cluster_bits - REFCOUNT_SHIFT);
         cluster_end = (refcnt_index + 1) << (s->cluster_bits - REFCOUNT_SHIFT);
         for ( ; cluster_index < cluster_end; cluster_index++) {
-            if (get_refcount(bs, cluster_index) == 0)
-                /* do nothing -- reserved for free counting */;
-            else
+            if (get_refcount(bs, cluster_index) == 0) {
+                f++;
+                tail++;
+            }
+            else {
                 h = cluster_index;
+                tail = 0;
+            }
         }
     }
 
+    f -= tail;
+    if (free)
+        *free = f;
     if (high)
         *high = (h+1);
 }
@@ -2270,8 +2282,10 @@ retry:
             (s->free_cluster_index - nb_clusters) << s->cluster_bits);
 #endif
 
-    if (s->highest_alloc < s->free_cluster_index)
+    if (s->highest_alloc < s->free_cluster_index) {
+        s->nc_free += (s->free_cluster_index - s->highest_alloc);
         s->highest_alloc = s->free_cluster_index;
+    }
 
     return (s->free_cluster_index - nb_clusters) << s->cluster_bits;
 }
@@ -2448,6 +2462,12 @@ static int update_cluster_refcount(BlockDriverState *bs,
     block_index = cluster_index &
         ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1);
     refcount = be16_to_cpu(s->refcount_block_cache[block_index]);
+
+    if (refcount == 1 && addend == -1)
+        s->nc_free += 1;
+    else if (refcount == 0 && addend == 1)
+        s->nc_free -= 1;
+
     refcount += addend;
     if (refcount < 0 || refcount > 0xffff)
         return -EINVAL;
block.c +3 -2
index 5305b27..a65fc09 100644
@@ -1137,8 +1137,9 @@ void bdrv_info_stats (void)
 		     bs->rd_bytes, bs->wr_bytes,
 		     bs->rd_ops, bs->wr_ops);
         if (bdrv_get_info(bs, &bdi) == 0)
-            term_printf(" high=%" PRIu64,
-                        bdi.highest_alloc);
+            term_printf(" high=%" PRId64
+                        " bytes_free=%" PRId64,
+                        bdi.highest_alloc, bdi.num_free_bytes);
         term_printf("\n");
     }
 }
block.h +1 -0
index 9c64af3..dca17db 100644
@@ -26,6 +26,7 @@ typedef struct BlockDriverInfo {
     /* offset at which the VM state can be saved (0 if not possible) */
     int64_t vm_state_offset;
     int64_t highest_alloc; /* highest allocated block offset (in bytes) */
+    int64_t num_free_bytes; /* below highest_alloc  */
 } BlockDriverInfo;
 
 typedef struct QEMUSnapshotInfo {
Signature · 1 line
1.6.0.6
#1 lore raw
Quoted 6 lines · Uri Lublin

'num_free_bytes' is the number of non-allocated bytes below highest-allocation.

Added bookkeeping to block-qcow2.c
Export it using BlockDeviceInfo
Show it upon 'info blockstats' if BlockDeviceInfo exists

What is the use case for this?

Regards,

Anthony Liguori

Quoted 140 lines
---
 block-qcow2.c |   36 ++++++++++++++++++++++++++++--------
 block.c       |    5 +++--
 block.h       |    1 +
 3 files changed, 32 insertions(+), 10 deletions(-)
block-qcow2.c +28 -8
index b8ef825..f16186e 100644
@@ -145,6 +145,7 @@ typedef struct BDRVQcowState {
     AES_KEY aes_decrypt_key;
 
     int64_t highest_alloc; /* highest cluester allocated (in clusters) */
+    int64_t nc_free;       /* num of free clusters below highest_alloc */
 
     uint64_t snapshots_offset;
     int snapshots_size;
@@ -173,7 +174,7 @@ static void free_clusters(BlockDriverState *bs,
 #ifdef DEBUG_ALLOC
 static void check_refcounts(BlockDriverState *bs);
 #endif
-static void scan_refcount(BlockDriverState *bs, int64_t *high);
+static void scan_refcount(BlockDriverState *bs, int64_t *high, int64_t *free);
 
 
 static int qcow_probe(const uint8_t *buf, int buf_size, const char *filename)
@@ -283,7 +284,7 @@ static int qcow_open(BlockDriverState *bs, const char *filename, int flags)
     if (refcount_init(bs) < 0)
         goto fail;
 
-    scan_refcount(bs, &s->highest_alloc);
+    scan_refcount(bs, &s->highest_alloc, &s->nc_free);
 
     /* read the backing file name */
     if (header.backing_file_offset != 0) {
@@ -1672,6 +1673,7 @@ static int qcow_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
     bdi->vm_state_offset = (int64_t)s->l1_vm_state_index <<
         (s->cluster_bits + s->l2_bits);
     bdi->highest_alloc = s->highest_alloc << s->cluster_bits;
+    bdi->num_free_bytes = s->nc_free  << s->cluster_bits;
     return 0;
 }
 
@@ -2206,25 +2208,35 @@ static int load_refcount_block(BlockDriverState *bs,
     return 0;
 }
 
-static void scan_refcount(BlockDriverState *bs, int64_t *high)
+static void scan_refcount(BlockDriverState *bs, int64_t *high, int64_t *free)
 {
     BDRVQcowState *s = bs->opaque;
-    int64_t refcnt_index, cluster_index, cluster_end, h = 0;
+    int64_t refcnt_index, cluster_index, cluster_end, h = 0, f = 0;
+    int64_t tail = 0; /* do not count last consecutive free entries */
 
     for (refcnt_index=0; refcnt_index < s->refcount_table_size; refcnt_index++){
         if (s->refcount_table[refcnt_index] == 0) {
+            f += 1 << (s->cluster_bits - REFCOUNT_SHIFT);
+            tail += 1 << (s->cluster_bits - REFCOUNT_SHIFT);
             continue;
         }
         cluster_index = refcnt_index << (s->cluster_bits - REFCOUNT_SHIFT);
         cluster_end = (refcnt_index + 1) << (s->cluster_bits - REFCOUNT_SHIFT);
         for ( ; cluster_index < cluster_end; cluster_index++) {
-            if (get_refcount(bs, cluster_index) == 0)
-                /* do nothing -- reserved for free counting */;
-            else
+            if (get_refcount(bs, cluster_index) == 0) {
+                f++;
+                tail++;
+            }
+            else {
                 h = cluster_index;
+                tail = 0;
+            }
         }
     }
 
+    f -= tail;
+    if (free)
+        *free = f;
     if (high)
         *high = (h+1);
 }
@@ -2270,8 +2282,10 @@ retry:
             (s->free_cluster_index - nb_clusters) << s->cluster_bits);
 #endif
 
-    if (s->highest_alloc < s->free_cluster_index)
+    if (s->highest_alloc < s->free_cluster_index) {
+        s->nc_free += (s->free_cluster_index - s->highest_alloc);
         s->highest_alloc = s->free_cluster_index;
+    }
 
     return (s->free_cluster_index - nb_clusters) << s->cluster_bits;
 }
@@ -2448,6 +2462,12 @@ static int update_cluster_refcount(BlockDriverState *bs,
     block_index = cluster_index &
         ((1 << (s->cluster_bits - REFCOUNT_SHIFT)) - 1);
     refcount = be16_to_cpu(s->refcount_block_cache[block_index]);
+
+    if (refcount == 1 && addend == -1)
+        s->nc_free += 1;
+    else if (refcount == 0 && addend == 1)
+        s->nc_free -= 1;
+
     refcount += addend;
     if (refcount < 0 || refcount > 0xffff)
         return -EINVAL;
block.c +3 -2
index 5305b27..a65fc09 100644
@@ -1137,8 +1137,9 @@ void bdrv_info_stats (void)
 		     bs->rd_bytes, bs->wr_bytes,
 		     bs->rd_ops, bs->wr_ops);
         if (bdrv_get_info(bs, &bdi) == 0)
-            term_printf(" high=%" PRIu64,
-                        bdi.highest_alloc);
+            term_printf(" high=%" PRId64
+                        " bytes_free=%" PRId64,
+                        bdi.highest_alloc, bdi.num_free_bytes);
         term_printf("\n");
     }
 }
block.h +1 -0
index 9c64af3..dca17db 100644
@@ -26,6 +26,7 @@ typedef struct BlockDriverInfo {
     /* offset at which the VM state can be saved (0 if not possible) */
     int64_t vm_state_offset;
     int64_t highest_alloc; /* highest allocated block offset (in bytes) */
+    int64_t num_free_bytes; /* below highest_alloc  */
 } BlockDriverInfo;
 
 typedef struct QEMUSnapshotInfo {
author
#2 lore raw
Quoted 9 lines · Anthony Liguori
Quoted 6 lines · Uri Lublin

'num_free_bytes' is the number of non-allocated bytes below highest-allocation.

Added bookkeeping to block-qcow2.c
Export it using BlockDeviceInfo
Show it upon 'info blockstats' if BlockDeviceInfo exists

What is the use case for this?

With both highest_alloc and num_free_bytes, we can have a pretty good idea about how fragmented a qcow2 image is, and upon a threshold we can:
1. Allocate more disk space (and not wait for the disk space to run out) or
2. Defragment the image (offline: "qemu-img convert", live: yet to be implemented).

Option 1 assumes one is using LVM, or have other means for disk space allocation. This is especially important for "over subscription" of disk space, meaning the logical size of the image is larger then the disk space allocated for it (the physical size allocated for the image).

Currently num_free_bytes is usually pretty small (images grow a lot but not shrink that much). I am working on a zero-cluster optimization (originally written by Shahar Frank) which provides a way for the guest file system to "tell" qemu when blocks are freed (by writing 0's). An exception is savevm/delvm scenario (which seems to be broken; I'll try to figure out what's wrong with savevm/delvm when I'll have more time available).

Thanks,
Uri.