VHDX log replay and write support
v2 cover VHDX log replay and write support
This patch series contains the initial VHDX log parsing, replay, and write support.
v2 changes: incorporated Fam's review feedback
This will allow an existing log in a VHDX image to be replayed (e.g., a VHDX image from a Hyper-V host that crashed). In addition, metadata writes are enabled through the log. This allows write support to be enabled for VHDX, as the BAT can be updated safely via the log journal.
These exact patches are available from github, for testing:
https://github.com/codyprime/qemu-kvm-jtc/tree/vhdx-write-v2-upstream
The latest vhdx work (including anything beyond these patches, such as backing/parent file support) can be found at:
https://github.com/codyprime/qemu-kvm-jtc/tree/jtc-vhdx-latest
Jeff Cody (9):
block: vhdx - minor comments and typo correction.
block: vhdx - add header update capability.
block: vhdx code movement - VHDXMetadataEntries and BDRVVHDXState to
header.
block: vhdx - log support struct and defines
block: vhdx - break endian translation functions out
block: vhdx - update log guid in header, and first write tracker
block: vhdx - log parsing, replay, and flush support
block: vhdx - add log write support
block: vhdx write support
block/Makefile.objs | 2 +- block/vhdx-endian.c | 141 ++++++++ block/vhdx-log.c | 1005 +++++++++++++++++++++++++++++++++++++++++++++++++++ block/vhdx.c | 392 ++++++++++++++------ block/vhdx.h | 110 +++++- configure | 13 + 6 files changed, 1552 insertions(+), 111 deletions(-) create mode 100644 block/vhdx-endian.c create mode 100644 block/vhdx-log.c
Signature · 1 line
1.8.1.4
PATCH v2 1/9 block: vhdx - minor comments and typo correction.
Just a couple of minor comments to help note where allocated buffers are freed, and a typo fix.
--- block/vhdx.c | 6 ++++-- block/vhdx.h | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-)
block/vhdx.c +4 -2
@@ -6,9 +6,9 @@* Authors: * Jeff Cody <[email protected]> * -* This is based on the "VHDX Format Specification v0.95", published 4/12/2012 +* This is based on the "VHDX Format Specification v1.00", published 8/25/2012 * by Microsoft: -* https://www.microsoft.com/en-us/download/details.aspx?id=29681 +* https://www.microsoft.com/en-us/download/details.aspx?id=34750 * * This work is licensed under the terms of the GNU LGPL, version 2 or later. * See the COPYING.LIB file in the top-level directory.
@@ -262,6 +262,7 @@ static int vhdx_parse_header(BlockDriverState *bs, BDRVVHDXState *s)uint64_t h2_seq =0 ;uint8_t *buffer; +/* header1 & header2 are freed in vhdx_close() */ header1 =qemu_blockalign (bs,sizeof (VHDXHeader)); header2 =qemu_blockalign (bs,sizeof (VHDXHeader));
@@ -787,6 +788,7 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags) goto fail; } +/* s->bat is freed in vhdx_close() */ s->bat =qemu_blockalign (bs, s->bat_rt.length); ret =bdrv_pread (bs->file, s->bat_offset, s->bat, s->bat_rt.length);
block/vhdx.h +3 -3
@@ -6,9 +6,9 @@* Authors: * Jeff Cody <[email protected]> * -* This is based on the "VHDX Format Specification v0.95", published 4/12/2012 +* This is based on the "VHDX Format Specification v1.00", published 8/25/2012 * by Microsoft: -* https://www.microsoft.com/en-us/download/details.aspx?id=29681 +* https://www.microsoft.com/en-us/download/details.aspx?id=34750 * * This work is licensed under the terms of the GNU LGPL, version 2 or later. * See the COPYING.LIB file in the top-level directory.
@@ -116,7 +116,7 @@ typedef struct QEMU_PACKED VHDXHeader {valid. */ uint16_t log_version;/* version of the log format. Mustn't be zero, unless log_guid is also zero */ -uint16_t version;/* version of th evhdx file. Currently, +uint16_t version;/* version of the vhdx file. Currently, only supported version is "1" */ uint32_t log_length;/* length of the log. Must be multiple of 1MB */
Signature · 1 line
1.8.1.4
PATCH v2 2/9 block: vhdx - add header update capability.
In reply to Jeff CodyThis adds the ability to update the headers in a VHDX image, including generating a new MS-compatible GUID.
As VHDX depends on uuid.h, VHDX is now a configurable build option. If VHDX support is enabled, that will also enable uuid as well. The default is to have VHDX enabled.
To enable/disable VHDX: --enable-vhdx, --disable-vhdx
--- block/Makefile.objs | 2 +- block/vhdx.c | 157 +++++++++++++++++++++++++++++++++++++++++++++++++++- block/vhdx.h | 12 +++- configure | 13 +++++ 4 files changed, 180 insertions(+), 4 deletions(-)
block/Makefile.objs +1 -1
@@ -2,7 +2,7 @@ block-obj-y += raw.o cow.o qcow.o vdi.o vmdk.o cloop.o dmg.o bochs.o vpc.o vvfat block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o qcow2-cache.o block-obj-y += qed.o qed-gencb.o qed-l2-cache.o qed-table.o qed-cluster.o block-obj-y += qed-check.o -block-obj-y += vhdx.o +block-obj-$(CONFIG_VHDX) += vhdx.o block-obj-y += parallels.o blkdebug.o blkverify.o block-obj-y += snapshot.o qapi.o block-obj-$(CONFIG_WIN32) += raw-win32.o win32-aio.o
block/vhdx.c +155 -2
198 lines not on this page · raw message
block/vhdx.h +11 -1
@@ -309,17 +309,27 @@ typedef struct QEMU_PACKED VHDXParentLocatorEntry {/* ----- END VHDX SPECIFICATION STRUCTURES ---- */ +void vhdx_guid_generate (MSGUID *guid); + +uint32_t vhdx_update_checksum (uint8_t *buf,size_t size,int crc_offset);uint32_t vhdx_checksum_calc (uint32_t crc,uint8_t *buf,size_t size,int crc_offset);bool vhdx_checksum_is_valid (uint8_t *buf,size_t size,int crc_offset); -static void leguid_to_cpus (MSGUID *guid) +static inline void leguid_to_cpus (MSGUID *guid) {le32_to_cpus (&guid->data1);le16_to_cpus (&guid->data2);le16_to_cpus (&guid->data3); } +static inline void cpu_to_leguids (MSGUID *guid) +{ +cpu_to_le32s (&guid->data1); +cpu_to_le16s (&guid->data2); +cpu_to_le16s (&guid->data3); +} +#endif
configure +13 -0
@@ -244,6 +244,7 @@ gtk="" gtkabi="2.0" tpm="no" libssh2="" +vhdx="yes" # parse CC options first for opt do
@@ -950,6 +951,11 @@ for opt do ;; --enable-libssh2) libssh2="yes" ;; + --enable-vhdx) vhdx="yes" ; + uuid="yes" + ;; + --disable-vhdx) vhdx="no" + ;; *) echo "ERROR: unknown option $opt"; show_help="yes" ;; esac
@@ -1166,6 +1172,8 @@ echo " --gcov=GCOV use specified gcov [$gcov_tool]" echo " --enable-tpm enable TPM support" echo " --disable-libssh2 disable ssh block device support" echo " --enable-libssh2 enable ssh block device support" +echo " --disable-vhdx disables support for the Microsoft VHDX image format" +echo " --enable-vhdx enable support for the Microsoft VHDX image format" echo "" echo "NOTE: The object files are built at the place where configure is launched" exit 1
@@ -3622,6 +3630,7 @@ echo "TPM support $tpm" echo "libssh2 support $libssh2" echo "TPM passthrough $tpm_passthrough" echo "QOM debugging $qom_cast_debug" +echo "vhdx $vhdx" if test "$sdl_too_old" = "yes"; then echo "-> Your SDL version is too old - please upgrade to have SDL support"
@@ -4012,6 +4021,10 @@ if test "$virtio_blk_data_plane" = "yes" ; then echo 'CONFIG_VIRTIO_BLK_DATA_PLANE=$(CONFIG_VIRTIO)' >> $config_host_mak fi +if test "$vhdx" = "yes" ; then + echo "CONFIG_VHDX=y" >> $config_host_mak +fi + # USB host support case "$usb" in linux)
Signature · 1 line
1.8.1.4
Re: [Qemu-devel] [PATCH v2 2/9] block: vhdx - add header update capability.
block/vhdx.c bool vhdx_checksum_is_valid(uint8_t *buf, size_t size, int crc_offset) · line 250
Quoted 10 lines · Jeff Cody
block/vhdx.c +6
-0@@ -212,6 +242,24 @@ bool vhdx_checksum_is_valid(uint8_t *buf, size_t size, int crc_offset)/* +* This generates a UUID that is compliant with the MS GUIDs used +* in the VHDX spec (and elsewhere). +* +* We can do this with uuid_generate if uuid.h is present, +* however not all systems have uuid and the generation is +* pretty straightforward for the DCE + random usage case
The talk of uuid.h not being available confuses me. The code has no #ifdef and we do not define uuid_generate() ourselves.
Is this an outdated comment?
block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 330
Quoted 7 lines
block/vhdx.c +7
-0+/* Update the VHDX headers +* +* This follows the VHDX spec procedures for header updates. +* +* - non-current header is updated with largest sequence number +*/ +static int vhdx_update_header (BlockDriverState *bs,BDRVVHDXState *s,bool rw)
"rw" should be named "generate_file_write_guid". If you call vhdx_update_header() again later you do not need to update FileWriteGuid again according to the spec. There's probably no harm in doing so but the spec explicitly says "If this is the first header update within the session, use a new value for the FileWriteGuid".
This means that future calls to vhdx_update_header() in the same session should set "generate_file_write_guid" to false. Therefore "rw" is not the right name.
Quoted 12 lines
block/vhdx.c +12
-0+{ +int ret =0 ; +int hdr_idx =0 ; +uint64_t header_offset =VHDX_HEADER1_OFFSET ; + +VHDXHeader *active_header; +VHDXHeader *inactive_header; +VHDXHeader header_le; +uint8_t *buffer; + +/* operate on the non-current header */ +if (s->curr_header ==0 ) {
block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 352
Quoted 10 lines
block/vhdx.c +10
-0+ hdr_idx =1 ; + header_offset =VHDX_HEADER2_OFFSET ; + } + + active_header = s->headers[s->curr_header]; + inactive_header = s->headers[hdr_idx]; + + inactive_header->sequence_number = active_header->sequence_number +1 ; + +/* a new file guid must be generate before any file write, including
s/generate/generated/
Quoted 16 lines
block/vhdx.c +16
-0+* headers */ +memcpy (&inactive_header->file_write_guid, &s->session_guid, +sizeof (MSGUID )); + +/* a new data guid only needs to be generate before any guest-visible +* writes, so update it if the image is opened r/w. */ +if (rw) { +vhdx_guid_generate (&inactive_header->data_write_guid); + } + +/* the header checksum is not over just the packed size of VHDXHeader, +* but rather over the entire 'reserved' range for the header, which is +* 4KB (VHDX_HEADER_SIZE). */ + + buffer =qemu_blockalign (bs,VHDX_HEADER_SIZE ); +/* we can't assume the extra reserved bytes are 0 */
block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 378
Quoted 10 lines
block/vhdx.c +10
-0+ ret =bdrv_pread (bs->file, header_offset, buffer,VHDX_HEADER_SIZE ); +if (ret <0 ) { + goto exit; + } +/* overwrite the actual VHDXHeader portion */ +memcpy (buffer, inactive_header,sizeof (VHDXHeader)); + inactive_header->checksum =vhdx_update_checksum (buffer, +VHDX_HEADER_SIZE ,4 ); +vhdx_header_le_export (inactive_header, &header_le); +bdrv_pwrite_sync (bs->file, header_offset, &header_le,sizeof (VHDXHeader));
This function can fail and it's a good indicator that future I/Os will also be in trouble. Please propagate the error return.
block/vhdx.c static int vhdx_open(BlockDriverState *bs, QDict *options, int flags) · line 958
Quoted 7 lines
block/vhdx.c +1
-2@@ -801,8 +955,7 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags) }if (flags &BDRV_O_RDWR ) { - ret = -ENOTSUP ; - goto fail; +vhdx_update_headers (bs, s, false);
Error handling is missing. At this point it looks like we cannot write to the file. Propagating the error seems reasonable.
Re: [Qemu-devel] [PATCH v2 2/9] block: vhdx - add header update capability.
Quoted 15 lines · Stefan Hajnoczi
block/vhdx.c bool vhdx_checksum_is_valid(uint8_t *buf, size_t size, int crc_offset) · line 250
Quoted 10 lines · Jeff Cody
block/vhdx.c +6
-0@@ -212,6 +242,24 @@ bool vhdx_checksum_is_valid(uint8_t *buf, size_t size, int crc_offset)/* +* This generates a UUID that is compliant with the MS GUIDs used +* in the VHDX spec (and elsewhere). +* +* We can do this with uuid_generate if uuid.h is present, +* however not all systems have uuid and the generation is +* pretty straightforward for the DCE + random usage case The talk of
uuid.hnot being available confuses me. The code has no #ifdef and we do not defineuuid_generate()ourselves.
Quoted 1 line
Is this an outdated comment?
Yes, the comment is outdated from when I had my own UUID generation in the patch - I missed removing the comment.
Quoted 15 lines
block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 330
Quoted 8 lines
block/vhdx.c +7
-0+/* Update the VHDX headers +* +* This follows the VHDX spec procedures for header updates. +* +* - non-current header is updated with largest sequence number +*/ +static int vhdx_update_header (BlockDriverState *bs,BDRVVHDXState *s,bool rw)"rw" should be named "
generate_file_write_guid". If you callvhdx_update_header()again later you do not need to update FileWriteGuid again according to the spec. There's probably no harm in doing so but the spec explicitly says "If this is the first header update within the session, use a new value for the FileWriteGuid".
Quoted 4 lines
This means that future calls to
vhdx_update_header()in the same session should set "generate_file_write_guid" to false. Therefore "rw" is not the right name.
Yes, it is a bit misnamed. I'll change the name. However, redundant generation is protected against with via vhdx_user_visible_write(), introduced in patch 6, which is the only placed that calls it with rw = true (soon to be generate_file_write_guid = true).
Quoted 16 lines
Quoted 16 lines
block/vhdx.c +16
-0+{ +int ret =0 ; +int hdr_idx =0 ; +uint64_t header_offset =VHDX_HEADER1_OFFSET ; + +VHDXHeader *active_header; +VHDXHeader *inactive_header; +VHDXHeader header_le; +uint8_t *buffer; + +/* operate on the non-current header */ +if (s->curr_header ==0 ) { + hdr_idx =1 ; + header_offset =VHDX_HEADER2_OFFSET ; + } +
Quoted 9 lines
block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 352
Quoted 6 lines
block/vhdx.c +6
-0+ active_header = s->headers[s->curr_header]; + inactive_header = s->headers[hdr_idx]; + + inactive_header->sequence_number = active_header->sequence_number +1 ; + +/* a new file guid must be generate before any file write, including s/generate/generated/
Thanks
Quoted 21 lines
Quoted 21 lines
block/vhdx.c +21
-0+* headers */ +memcpy (&inactive_header->file_write_guid, &s->session_guid, +sizeof (MSGUID )); + +/* a new data guid only needs to be generate before any guest-visible +* writes, so update it if the image is opened r/w. */ +if (rw) { +vhdx_guid_generate (&inactive_header->data_write_guid); + } + +/* the header checksum is not over just the packed size of VHDXHeader, +* but rather over the entire 'reserved' range for the header, which is +* 4KB (VHDX_HEADER_SIZE). */ + + buffer =qemu_blockalign (bs,VHDX_HEADER_SIZE ); +/* we can't assume the extra reserved bytes are 0 */ + ret =bdrv_pread (bs->file, header_offset, buffer,VHDX_HEADER_SIZE ); +if (ret <0 ) { + goto exit; + } +/* overwrite the actual VHDXHeader portion */
Quoted 9 lines
block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 378
Quoted 5 lines
block/vhdx.c +5
-0+memcpy (buffer, inactive_header,sizeof (VHDXHeader)); + inactive_header->checksum =vhdx_update_checksum (buffer, +VHDX_HEADER_SIZE ,4 ); +vhdx_header_le_export (inactive_header, &header_le); +bdrv_pwrite_sync (bs->file, header_offset, &header_le,sizeof (VHDXHeader));This function can fail and it's a good indicator that future I/Os will also be in trouble. Please propagate the error return.
You are right, that was unintentional.
Quoted 10 lines
block/vhdx.c static int vhdx_open(BlockDriverState *bs, QDict *options, int flags) · line 958
Quoted 7 lines
block/vhdx.c +1
-2@@ -801,8 +955,7 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags) }if (flags &BDRV_O_RDWR ) { - ret = -ENOTSUP ; - goto fail; +vhdx_update_headers (bs, s, false);Error handling is missing. At this point it looks like we cannot write to the file. Propagating the error seems reasonable.
OK, I move the above change into when the write support is actually in place.
Re: [Qemu-devel] [PATCH v2 2/9] block: vhdx - add header update capability.
In reply to Stefan HajnocziQuoted 31 lines · Stefan Hajnoczi
block/vhdx.c bool vhdx_checksum_is_valid(uint8_t *buf, size_t size, int crc_offset) · line 250
Quoted 10 lines · Jeff Cody
block/vhdx.c +6
-0@@ -212,6 +242,24 @@ bool vhdx_checksum_is_valid(uint8_t *buf, size_t size, int crc_offset)/* +* This generates a UUID that is compliant with the MS GUIDs used +* in the VHDX spec (and elsewhere). +* +* We can do this with uuid_generate if uuid.h is present, +* however not all systems have uuid and the generation is +* pretty straightforward for the DCE + random usage case The talk of
uuid.hnot being available confuses me. The code has no #ifdef and we do not defineuuid_generate()ourselves.Is this an outdated comment?
block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 330
Quoted 7 lines
block/vhdx.c +7
-0+/* Update the VHDX headers +* +* This follows the VHDX spec procedures for header updates. +* +* - non-current header is updated with largest sequence number +*/ +static int vhdx_update_header (BlockDriverState *bs,BDRVVHDXState *s,bool rw)"rw" should be named "
generate_file_write_guid". If you callvhdx_update_header()again later you do not need to update FileWriteGuid again according to the spec. There's probably no harm in doing so but the spec explicitly says "If this is the first header update within the session, use a new value for the FileWriteGuid".
Quoted 4 lines
This means that future calls to
vhdx_update_header()in the same session should set "generate_file_write_guid" to false. Therefore "rw" is not the right name.
Reading through your comments again made me realize there is a misunderstanding (a good sign in itself that 'rw' should be renamed). The 'rw' is not to generate the FileWriteGuid, but rather the DataWriteGuid. FileWriteGuid is copied from our session_guid, so it is only generated once and used through the entire open session. The s->session_guid field is generated upon vhdx_open(), for usage if the file is opened BDRV_O_RDWR.
The DataWriteGuid is generated before any user-visible writes (anything observable via a virtual disk read). And that DataWriteGuid is what the 'rw' field is used for, and the function vhdx_user_visible_write() handles calling vhdx_update_headers() with that field set to true.
I'll rename 'rw' to 'generate_data_write_guid'.
Quoted 65 lines
Quoted 12 lines
block/vhdx.c +12
-0+{ +int ret =0 ; +int hdr_idx =0 ; +uint64_t header_offset =VHDX_HEADER1_OFFSET ; + +VHDXHeader *active_header; +VHDXHeader *inactive_header; +VHDXHeader header_le; +uint8_t *buffer; + +/* operate on the non-current header */ +if (s->curr_header ==0 ) {block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 352
Quoted 10 lines
block/vhdx.c +10
-0+ hdr_idx =1 ; + header_offset =VHDX_HEADER2_OFFSET ; + } + + active_header = s->headers[s->curr_header]; + inactive_header = s->headers[hdr_idx]; + + inactive_header->sequence_number = active_header->sequence_number +1 ; + +/* a new file guid must be generate before any file write, including s/generate/generated/
Quoted 16 lines
block/vhdx.c +16
-0+* headers */ +memcpy (&inactive_header->file_write_guid, &s->session_guid, +sizeof (MSGUID )); + +/* a new data guid only needs to be generate before any guest-visible +* writes, so update it if the image is opened r/w. */ +if (rw) { +vhdx_guid_generate (&inactive_header->data_write_guid); + } + +/* the header checksum is not over just the packed size of VHDXHeader, +* but rather over the entire 'reserved' range for the header, which is +* 4KB (VHDX_HEADER_SIZE). */ + + buffer =qemu_blockalign (bs,VHDX_HEADER_SIZE ); +/* we can't assume the extra reserved bytes are 0 */ block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 378
Quoted 10 lines
block/vhdx.c +10
-0+ ret =bdrv_pread (bs->file, header_offset, buffer,VHDX_HEADER_SIZE ); +if (ret <0 ) { + goto exit; + } +/* overwrite the actual VHDXHeader portion */ +memcpy (buffer, inactive_header,sizeof (VHDXHeader)); + inactive_header->checksum =vhdx_update_checksum (buffer, +VHDX_HEADER_SIZE ,4 ); +vhdx_header_le_export (inactive_header, &header_le); +bdrv_pwrite_sync (bs->file, header_offset, &header_le,sizeof (VHDXHeader));This function can fail and it's a good indicator that future I/Os will also be in trouble. Please propagate the error return.
block/vhdx.c static int vhdx_open(BlockDriverState *bs, QDict *options, int flags) · line 958
Quoted 7 lines
block/vhdx.c +1
-2@@ -801,8 +955,7 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags) }if (flags &BDRV_O_RDWR ) { - ret = -ENOTSUP ; - goto fail; +vhdx_update_headers (bs, s, false);Error handling is missing. At this point it looks like we cannot write to the file. Propagating the error seems reasonable.
Re: [Qemu-devel] [PATCH v2 2/9] block: vhdx - add header update capability.
In reply to Jeff CodyAm 01.08.2013 um 05:23 hat Jeff Cody geschrieben:
Quoted 24 lines
This adds the ability to update the headers in a VHDX image, including generating a new MS-compatible GUID.
As VHDX depends on
uuid.h, VHDX is now a configurable build option. If VHDX support is enabled, that will also enable uuid as well. The default is to have VHDX enabled.To enable/disable VHDX: --enable-vhdx, --disable-vhdx
Signed-off-by Jeff Cody <[email protected]>configure +1
-0@@ -244,6 +244,7 @@ gtk="" gtkabi="2.0" tpm="no" libssh2="" +vhdx="yes" # parse CC options first for opt do
configure for opt do · line 956
Quoted 6 lines
configure +3
-0@@ -950,6 +951,11 @@ for opt do ;; --enable-libssh2) libssh2="yes" ;; + --enable-vhdx) vhdx="yes" ; + uuid="yes" + ;;
What's this? Shouldn't auto-detecting uuid and checking later whether it's available do the right thing?
Test cases:
1. configure --disable-uuid --enable-vhdx
This should result in an error because the options contradict each other. With this patch, the disable flag is overridden.
2. No libuuid headers available; ./configure
Should build without VHDX. With this patch, uuid is left out, vhdx is configured anyway, the build fails.
Kevin
Re: [Qemu-devel] [PATCH v2 2/9] block: vhdx - add header update capability.
Quoted 22 lines · Kevin Wolf
Am 01.08.2013 um 05:23 hat Jeff Cody geschrieben:
Quoted 21 lines
This adds the ability to update the headers in a VHDX image, including generating a new MS-compatible GUID.
As VHDX depends on
uuid.h, VHDX is now a configurable build option. If VHDX support is enabled, that will also enable uuid as well. The default is to have VHDX enabled.To enable/disable VHDX: --enable-vhdx, --disable-vhdx
Signed-off-by Jeff Cody <[email protected]>configure +1
-0@@ -244,6 +244,7 @@ gtk="" gtkabi="2.0" tpm="no" libssh2="" +vhdx="yes"
Quoted 13 lines
configure for opt do · line 956
Quoted 9 lines
configure +3
-0# parse CC options first for opt do@@ -950,6 +951,11 @@ for opt do ;; --enable-libssh2) libssh2="yes" ;; + --enable-vhdx) vhdx="yes" ; + uuid="yes" + ;;What's this? Shouldn't auto-detecting uuid and checking later whether it's available do the right thing?
You are right, thanks. Will fix for v3.
Quoted 11 lines
Test cases:
1. configure --disable-uuid --enable-vhdx
This should result in an error because the options contradict each other. With this patch, the disable flag is overridden.2. No libuuid headers available; ./configure
Should build without VHDX. With this patch, uuid is left out, vhdx is configured anyway, the build fails.Kevin
Re: [Qemu-devel] [PATCH v2 2/9] block: vhdx - add header update capability.
In reply to Jeff CodyAm 01.08.2013 um 05:23 hat Jeff Cody geschrieben:
Quoted 62 lines
This adds the ability to update the headers in a VHDX image, including generating a new MS-compatible GUID.
As VHDX depends on
uuid.h, VHDX is now a configurable build option. If VHDX support is enabled, that will also enable uuid as well. The default is to have VHDX enabled.To enable/disable VHDX: --enable-vhdx, --disable-vhdx
Signed-off-by Jeff Cody <[email protected]>--- block/Makefile.objs | 2 +- block/vhdx.c | 157 +++++++++++++++++++++++++++++++++++++++++++++++++++- block/vhdx.h | 12 +++- configure | 13 +++++ 4 files changed, 180 insertions(+), 4 deletions(-)block/Makefile.objs +1
-1@@ -2,7 +2,7 @@ block-obj-y += raw.o cow.o qcow.o vdi.o vmdk.o cloop.o dmg.o bochs.o vpc.o vvfat block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o qcow2-cache.o block-obj-y += qed.o qed-gencb.o qed-l2-cache.o qed-table.o qed-cluster.o block-obj-y += qed-check.o -block-obj-y += vhdx.o +block-obj-$(CONFIG_VHDX) += vhdx.o block-obj-y += parallels.o blkdebug.o blkverify.o block-obj-y += snapshot.o qapi.o block-obj-$(CONFIG_WIN32) += raw-win32.o win32-aio.oblock/vhdx.c +12
-0@@ -21,6 +21,7 @@#include "qemu/crc32c.h" #include "block/vhdx.h" +#include <uuid/uuid.h> /* Several metadata and region table data entries are identified by * guids in a MS-specific GUID format. */ @@ -156,11 +157,40 @@ typedef struct BDRVVHDXState {VHDXBatEntry *bat;uint64_t bat_offset; +MSGUID session_guid; + +VHDXParentLocatorHeader parent_header;VHDXParentLocatorEntry *parent_entries; } BDRVVHDXState; +/* Calculates new checksum. +* +* Zero is substituted during crc calculation for the original crc field +* crc_offset: byte offset in buf of the buffer crc +* buf: buffer pointer +* size: size of buffer (must be > crc_offset+4) +* +* Note: The resulting checksum is in the CPU endianness, not necessarily
block/vhdx.c typedef struct BDRVVHDXState { · line 185
Quoted 10 lines
block/vhdx.c +10
-0+* in the file format endianness (LE). Any header export to disk should +* make sure that vhdx_header_le_export() is used to convert to the +* correct endianness +*/ +uint32_t vhdx_update_checksum (uint8_t *buf,size_t size,int crc_offset) +{ +uint32_t crc; + +assert (buf !=NULL ); +assert (size > (crc_offset +4 ));
sizeof(crc) for consistency.
Quoted 21 lines
block/vhdx.c +14
-0+ +memset (buf + crc_offset,0 ,sizeof (crc)); + crc =crc32c (0xffffffff , buf, size); +memcpy (buf + crc_offset, &crc,sizeof (crc)); + +return crc; +} +uint32_t vhdx_checksum_calc (uint32_t crc,uint8_t *buf,size_t size,int crc_offset) {@@ -212,6 +242,24 @@ bool vhdx_checksum_is_valid(uint8_t *buf, size_t size, int crc_offset)/* +* This generates a UUID that is compliant with the MS GUIDs used +* in the VHDX spec (and elsewhere). +* +* We can do this with uuid_generate if uuid.h is present, +* however not all systems have uuid and the generation is +* pretty straightforward for the DCE + random usage case
block/vhdx.c bool vhdx_checksum_is_valid(uint8_t *buf, size_t size, int crc_offset) · line 260
Quoted 10 lines
block/vhdx.c +10
-0+* +*/ +void vhdx_guid_generate (MSGUID *guid) +{ +uuid_t uuid; +assert (guid !=NULL ); + +uuid_generate (uuid); +memcpy (guid, uuid,16 ); +}
Should MSGUID be QEMU_PACKED to make sure that this works? Also sizeof(guid) instead of 16?
Quoted 12 lines
block/vhdx.c +5
-0+ +/* * Per the MS VHDX Specification, for every VHDX file: * - The header section is fixed size - 1 MB * - The header section is always the first "object" @@ -249,6 +297,107 @@ static void vhdx_header_le_import(VHDXHeader *h)le64_to_cpus (&h->log_offset); } +/* All VHDX structures on disk are little endian */ +static void vhdx_header_le_export (VHDXHeader *orig_h,VHDXHeader *new_h) +{
block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 312
Quoted 10 lines
block/vhdx.c +10
-0+assert (orig_h !=NULL ); +assert (new_h !=NULL ); + + new_h->signature =cpu_to_le32 (orig_h->signature); + new_h->checksum =cpu_to_le32 (orig_h->checksum); + new_h->sequence_number =cpu_to_le64 (orig_h->sequence_number); + +memcpy (&new_h->file_write_guid, &orig_h->file_write_guid,sizeof (MSGUID )); +memcpy (&new_h->data_write_guid, &orig_h->data_write_guid,sizeof (MSGUID )); +memcpy (&new_h->log_guid, &orig_h->log_guid,sizeof (MSGUID ));
Can we avoid memcpy? I think this should work:
new_h->file_write_guid = orig_h->file_write_guid;new_h->data_write_guid = orig_h->data_write_guid;new_h->log_guid = orig_h->log_guid;
Quoted 33 lines
block/vhdx.c +33
-0+ +cpu_to_leguids (&new_h->file_write_guid); +cpu_to_leguids (&new_h->data_write_guid); +cpu_to_leguids (&new_h->log_guid); + + new_h->log_version =cpu_to_le16 (orig_h->log_version); + new_h->version =cpu_to_le16 (orig_h->version); + new_h->log_length =cpu_to_le32 (orig_h->log_length); + new_h->log_offset =cpu_to_le64 (orig_h->log_offset); +} + +/* Update the VHDX headers +* +* This follows the VHDX spec procedures for header updates. +* +* - non-current header is updated with largest sequence number +*/ +static int vhdx_update_header (BlockDriverState *bs,BDRVVHDXState *s,bool rw) +{ +int ret =0 ; +int hdr_idx =0 ; +uint64_t header_offset =VHDX_HEADER1_OFFSET ; + +VHDXHeader *active_header; +VHDXHeader *inactive_header; +VHDXHeader header_le; +uint8_t *buffer; + +/* operate on the non-current header */ +if (s->curr_header ==0 ) { + hdr_idx =1 ; + header_offset =VHDX_HEADER2_OFFSET ; + }
block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 355
Quoted 10 lines
block/vhdx.c +10
-0+ + active_header = s->headers[s->curr_header]; + inactive_header = s->headers[hdr_idx]; + + inactive_header->sequence_number = active_header->sequence_number +1 ; + +/* a new file guid must be generate before any file write, including +* headers */ +memcpy (&inactive_header->file_write_guid, &s->session_guid, +sizeof (MSGUID ));
Here a simple assignment should do as well.
Quoted 19 lines
block/vhdx.c +19
-0+ +/* a new data guid only needs to be generate before any guest-visible +* writes, so update it if the image is opened r/w. */ +if (rw) { +vhdx_guid_generate (&inactive_header->data_write_guid); + } + +/* the header checksum is not over just the packed size of VHDXHeader, +* but rather over the entire 'reserved' range for the header, which is +* 4KB (VHDX_HEADER_SIZE). */ + + buffer =qemu_blockalign (bs,VHDX_HEADER_SIZE ); +/* we can't assume the extra reserved bytes are 0 */ + ret =bdrv_pread (bs->file, header_offset, buffer,VHDX_HEADER_SIZE ); +if (ret <0 ) { + goto exit; + } +/* overwrite the actual VHDXHeader portion */ +memcpy (buffer, inactive_header,sizeof (VHDXHeader));
block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 384
Quoted 10 lines
block/vhdx.c +10
-0+ inactive_header->checksum =vhdx_update_checksum (buffer, +VHDX_HEADER_SIZE ,4 ); +vhdx_header_le_export (inactive_header, &header_le); +bdrv_pwrite_sync (bs->file, header_offset, &header_le,sizeof (VHDXHeader)); + s->curr_header = hdr_idx; + +exit: +qemu_vfree (buffer); +return ret; +}
Kevin
Re: [Qemu-devel] [PATCH v2 2/9] block: vhdx - add header update capability.
Quoted 67 lines · Kevin Wolf
Am 01.08.2013 um 05:23 hat Jeff Cody geschrieben:
Quoted 66 lines
This adds the ability to update the headers in a VHDX image, including generating a new MS-compatible GUID.
As VHDX depends on
uuid.h, VHDX is now a configurable build option. If VHDX support is enabled, that will also enable uuid as well. The default is to have VHDX enabled.To enable/disable VHDX: --enable-vhdx, --disable-vhdx
Signed-off-by Jeff Cody <[email protected]>--- block/Makefile.objs | 2 +- block/vhdx.c | 157 +++++++++++++++++++++++++++++++++++++++++++++++++++- block/vhdx.h | 12 +++- configure | 13 +++++ 4 files changed, 180 insertions(+), 4 deletions(-)block/Makefile.objs +1
-1@@ -2,7 +2,7 @@ block-obj-y += raw.o cow.o qcow.o vdi.o vmdk.o cloop.o dmg.o bochs.o vpc.o vvfat block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o qcow2-cache.o block-obj-y += qed.o qed-gencb.o qed-l2-cache.o qed-table.o qed-cluster.o block-obj-y += qed-check.o -block-obj-y += vhdx.o +block-obj-$(CONFIG_VHDX) += vhdx.o block-obj-y += parallels.o blkdebug.o blkverify.o block-obj-y += snapshot.o qapi.o block-obj-$(CONFIG_WIN32) += raw-win32.o win32-aio.oblock/vhdx.c +16
-0@@ -21,6 +21,7 @@#include "qemu/crc32c.h" #include "block/vhdx.h" +#include <uuid/uuid.h> /* Several metadata and region table data entries are identified by * guids in a MS-specific GUID format. */ @@ -156,11 +157,40 @@ typedef struct BDRVVHDXState {VHDXBatEntry *bat;uint64_t bat_offset; +MSGUID session_guid; + +VHDXParentLocatorHeader parent_header;VHDXParentLocatorEntry *parent_entries; } BDRVVHDXState; +/* Calculates new checksum. +* +* Zero is substituted during crc calculation for the original crc field +* crc_offset: byte offset in buf of the buffer crc +* buf: buffer pointer +* size: size of buffer (must be > crc_offset+4) +* +* Note: The resulting checksum is in the CPU endianness, not necessarily +* in the file format endianness (LE). Any header export to disk should +* make sure that vhdx_header_le_export() is used to convert to the +* correct endianness +*/
Quoted 9 lines
block/vhdx.c typedef struct BDRVVHDXState { · line 185
Quoted 6 lines
block/vhdx.c +6
-0+uint32_t vhdx_update_checksum (uint8_t *buf,size_t size,int crc_offset) +{ +uint32_t crc; + +assert (buf !=NULL ); +assert (size > (crc_offset +4 ));sizeof(crc) for consistency.
OK
Quoted 26 lines
Quoted 26 lines
block/vhdx.c +19
-0+ +memset (buf + crc_offset,0 ,sizeof (crc)); + crc =crc32c (0xffffffff , buf, size); +memcpy (buf + crc_offset, &crc,sizeof (crc)); + +return crc; +} +uint32_t vhdx_checksum_calc (uint32_t crc,uint8_t *buf,size_t size,int crc_offset) {@@ -212,6 +242,24 @@ bool vhdx_checksum_is_valid(uint8_t *buf, size_t size, int crc_offset)/* +* This generates a UUID that is compliant with the MS GUIDs used +* in the VHDX spec (and elsewhere). +* +* We can do this with uuid_generate if uuid.h is present, +* however not all systems have uuid and the generation is +* pretty straightforward for the DCE + random usage case +* +*/ +void vhdx_guid_generate (MSGUID *guid) +{ +uuid_t uuid;
Quoted 9 lines
block/vhdx.c bool vhdx_checksum_is_valid(uint8_t *buf, size_t size, int crc_offset) · line 260
Quoted 5 lines
block/vhdx.c +5
-0+assert (guid !=NULL ); + +uuid_generate (uuid); +memcpy (guid, uuid,16 ); +}Should MSGUID be
QEMU_PACKEDto make sure that this works? Also sizeof(guid) instead of 16?
Yikes, yes it definitely should be packed.
Quoted 25 lines
Quoted 12 lines
block/vhdx.c +5
-0+ +/* * Per the MS VHDX Specification, for every VHDX file: * - The header section is fixed size - 1 MB * - The header section is always the first "object" @@ -249,6 +297,107 @@ static void vhdx_header_le_import(VHDXHeader *h)le64_to_cpus (&h->log_offset); } +/* All VHDX structures on disk are little endian */ +static void vhdx_header_le_export (VHDXHeader *orig_h,VHDXHeader *new_h) +{block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 312
Quoted 10 lines
block/vhdx.c +10
-0+assert (orig_h !=NULL ); +assert (new_h !=NULL ); + + new_h->signature =cpu_to_le32 (orig_h->signature); + new_h->checksum =cpu_to_le32 (orig_h->checksum); + new_h->sequence_number =cpu_to_le64 (orig_h->sequence_number); + +memcpy (&new_h->file_write_guid, &orig_h->file_write_guid,sizeof (MSGUID )); +memcpy (&new_h->data_write_guid, &orig_h->data_write_guid,sizeof (MSGUID )); +memcpy (&new_h->log_guid, &orig_h->log_guid,sizeof (MSGUID ));Can we avoid memcpy? I think this should work:
Quoted 4 lines
new_h->file_write_guid=orig_h->file_write_guid;new_h->data_write_guid=orig_h->data_write_guid;new_h->log_guid=orig_h->log_guid;
Yes, that should work fine as well - I'll change it to that.
Quoted 77 lines
Quoted 33 lines
block/vhdx.c +33
-0+ +cpu_to_leguids (&new_h->file_write_guid); +cpu_to_leguids (&new_h->data_write_guid); +cpu_to_leguids (&new_h->log_guid); + + new_h->log_version =cpu_to_le16 (orig_h->log_version); + new_h->version =cpu_to_le16 (orig_h->version); + new_h->log_length =cpu_to_le32 (orig_h->log_length); + new_h->log_offset =cpu_to_le64 (orig_h->log_offset); +} + +/* Update the VHDX headers +* +* This follows the VHDX spec procedures for header updates. +* +* - non-current header is updated with largest sequence number +*/ +static int vhdx_update_header (BlockDriverState *bs,BDRVVHDXState *s,bool rw) +{ +int ret =0 ; +int hdr_idx =0 ; +uint64_t header_offset =VHDX_HEADER1_OFFSET ; + +VHDXHeader *active_header; +VHDXHeader *inactive_header; +VHDXHeader header_le; +uint8_t *buffer; + +/* operate on the non-current header */ +if (s->curr_header ==0 ) { + hdr_idx =1 ; + header_offset =VHDX_HEADER2_OFFSET ; + }block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 355
Quoted 10 lines
block/vhdx.c +10
-0+ + active_header = s->headers[s->curr_header]; + inactive_header = s->headers[hdr_idx]; + + inactive_header->sequence_number = active_header->sequence_number +1 ; + +/* a new file guid must be generate before any file write, including +* headers */ +memcpy (&inactive_header->file_write_guid, &s->session_guid, +sizeof (MSGUID ));Here a simple assignment should do as well.
Quoted 19 lines
block/vhdx.c +19
-0+ +/* a new data guid only needs to be generate before any guest-visible +* writes, so update it if the image is opened r/w. */ +if (rw) { +vhdx_guid_generate (&inactive_header->data_write_guid); + } + +/* the header checksum is not over just the packed size of VHDXHeader, +* but rather over the entire 'reserved' range for the header, which is +* 4KB (VHDX_HEADER_SIZE). */ + + buffer =qemu_blockalign (bs,VHDX_HEADER_SIZE ); +/* we can't assume the extra reserved bytes are 0 */ + ret =bdrv_pread (bs->file, header_offset, buffer,VHDX_HEADER_SIZE ); +if (ret <0 ) { + goto exit; + } +/* overwrite the actual VHDXHeader portion */ +memcpy (buffer, inactive_header,sizeof (VHDXHeader));block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 384
Quoted 10 lines
block/vhdx.c +10
-0+ inactive_header->checksum =vhdx_update_checksum (buffer, +VHDX_HEADER_SIZE ,4 ); +vhdx_header_le_export (inactive_header, &header_le); +bdrv_pwrite_sync (bs->file, header_offset, &header_le,sizeof (VHDXHeader)); + s->curr_header = hdr_idx; + +exit: +qemu_vfree (buffer); +return ret; +}Kevin
Re: [Qemu-devel] [PATCH v2 2/9] block: vhdx - add header update capability.
In reply to Jeff CodyAm 01.08.2013 um 05:23 hat Jeff Cody geschrieben:
Quoted 10 lines
This adds the ability to update the headers in a VHDX image, including generating a new MS-compatible GUID.
As VHDX depends on
uuid.h, VHDX is now a configurable build option. If VHDX support is enabled, that will also enable uuid as well. The default is to have VHDX enabled.To enable/disable VHDX: --enable-vhdx, --disable-vhdx
Signed-off-by Jeff Cody <[email protected]>
I knew I'd forget to mention something I had seen...
block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 376
Quoted 4 lines
block/vhdx.c +4
-0+/* overwrite the actual VHDXHeader portion */ +memcpy (buffer, inactive_header,sizeof (VHDXHeader)); + inactive_header->checksum =vhdx_update_checksum (buffer, +VHDX_HEADER_SIZE ,4 );
This I would prefer as offsetof(VHDXHeader, checksum) instead of a magic number 4.
Kevin
Re: [Qemu-devel] [PATCH v2 2/9] block: vhdx - add header update capability.
Quoted 14 lines · Kevin Wolf
Am 01.08.2013 um 05:23 hat Jeff Cody geschrieben:
Quoted 10 lines
This adds the ability to update the headers in a VHDX image, including generating a new MS-compatible GUID.
As VHDX depends on
uuid.h, VHDX is now a configurable build option. If VHDX support is enabled, that will also enable uuid as well. The default is to have VHDX enabled.To enable/disable VHDX: --enable-vhdx, --disable-vhdx
Signed-off-by Jeff Cody <[email protected]>I knew I'd forget to mention something I had seen...
Quoted 7 lines
block/vhdx.c static void vhdx_header_le_import(VHDXHeader *h) · line 376
Quoted 4 lines
block/vhdx.c +4
-0+/* overwrite the actual VHDXHeader portion */ +memcpy (buffer, inactive_header,sizeof (VHDXHeader)); + inactive_header->checksum =vhdx_update_checksum (buffer, +VHDX_HEADER_SIZE ,4 );This I would prefer as
offsetof(VHDXHeader, checksum)instead of a magic number 4.
Who doesn't like a little bit of magic? :) OK, I'll use offsetof() (I'll also look through the rest of my patches for similar occurrences as well).
Jeff
Re: [Qemu-devel] [PATCH v2 2/9] block: vhdx - add header update capability.
Am 05.08.2013 um 17:09 hat Jeff Cody geschrieben:
Quoted 20 lines · Kevin Wolf
Quoted 19 lines
Am 01.08.2013 um 05:23 hat Jeff Cody geschrieben:
Quoted 10 lines
This adds the ability to update the headers in a VHDX image, including generating a new MS-compatible GUID.
As VHDX depends on
uuid.h, VHDX is now a configurable build option. If VHDX support is enabled, that will also enable uuid as well. The default is to have VHDX enabled.To enable/disable VHDX: --enable-vhdx, --disable-vhdx
Signed-off-by Jeff Cody <[email protected]>I knew I'd forget to mention something I had seen...
Quoted 4 lines
block/vhdx.c +4
-0+/* overwrite the actual VHDXHeader portion */ +memcpy (buffer, inactive_header,sizeof (VHDXHeader)); + inactive_header->checksum =vhdx_update_checksum (buffer, +VHDX_HEADER_SIZE ,4 );
Quoted 4 lines
Quoted 2 lines
This I would prefer as
offsetof(VHDXHeader, checksum)instead of a magic number 4.Who doesn't like a little bit of magic? :)
I think I would have accepted a properly implemented qemu_rand(). :-)
Kevin
PATCH v2 3/9 block: vhdx code movement - VHDXMetadataEntries and BDRVVHDXState to header.
In reply to Jeff CodyIn preparation for VHDX log support, move these structures to the header.
--- block/vhdx.c | 51 --------------------------------------------------- block/vhdx.h | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 51 deletions(-)
block/vhdx.c +0 -51
@@ -104,16 +104,6 @@ static const MSGUID parent_vhdx_guid = { .data1 = 0xb04aefb7, META_PAGE_83_PRESENT | META_LOGICAL_SECTOR_SIZE_PRESENT | \ META_PHYS_SECTOR_SIZE_PRESENT) -typedef struct VHDXMetadataEntries { -VHDXMetadataTableEntry file_parameters_entry; -VHDXMetadataTableEntry virtual_disk_size_entry; -VHDXMetadataTableEntry page83_data_entry; -VHDXMetadataTableEntry logical_sector_size_entry; -VHDXMetadataTableEntry phys_sector_size_entry; -VHDXMetadataTableEntry parent_locator_entry; -uint16_t present; -}VHDXMetadataEntries ; - typedef struct VHDXSectorInfo { uint32_t bat_idx; /* BAT entry index */
@@ -124,47 +114,6 @@ typedef struct VHDXSectorInfo {uint64_t block_offset;/* block offset, in bytes */ } VHDXSectorInfo; - - -typedef struct BDRVVHDXState { -CoMutex lock; - -int curr_header; -VHDXHeader *headers[2 ]; - -VHDXRegionTableHeader rt; -VHDXRegionTableEntry bat_rt;/* region table for the BAT */ -VHDXRegionTableEntry metadata_rt;/* region table for the metadata */ - -VHDXMetadataTableHeader metadata_hdr; -VHDXMetadataEntries metadata_entries; - -VHDXFileParameters params; -uint32_t block_size; -uint32_t block_size_bits; -uint32_t sectors_per_block; -uint32_t sectors_per_block_bits; - -uint64_t virtual_disk_size; -uint32_t logical_sector_size; -uint32_t physical_sector_size; - -uint64_t chunk_ratio; -uint32_t chunk_ratio_bits; -uint32_t logical_sector_size_bits; - -uint32_t bat_entries; -VHDXBatEntry *bat; -uint64_t bat_offset; - -MSGUID session_guid; - - -VHDXParentLocatorHeader parent_header; -VHDXParentLocatorEntry *parent_entries; - -}BDRVVHDXState ; -/* Calculates new checksum. * * Zero is substituted during crc calculation for the original crc field
block/vhdx.h +47 -0
53 lines not on this page · raw message
Signature · 1 line
1.8.1.4
PATCH v2 4/9 block: vhdx - log support struct and defines
In reply to Jeff CodyThis adds some magic number defines, and internal structure definitions for VHDX log replay support.
--- block/vhdx.h | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-)
block/vhdx.h +20 -1
@@ -151,7 +151,10 @@ typedef struct QEMU_PACKED VHDXRegionTableEntry {/* ---- LOG ENTRY STRUCTURES ---- */ +#define VHDX_LOG_MIN_SIZE (1024*1024) +#define VHDX_LOG_SECTOR_SIZE 4096#define VHDX_LOG_HDR_SIZE 64 +#define VHDX_LOG_SIGNATURE 0x65676f6c typedef struct QEMU_PACKED VHDXLogEntryHeader {uint32_t signature;/* "loge" in ASCII */ uint32_t checksum;/* CRC-32C hash of the 64KB table */
@@ -174,7 +177,8 @@ typedef struct QEMU_PACKED VHDXLogEntryHeader { } VHDXLogEntryHeader;#define VHDX_LOG_DESC_SIZE 32 - +#define VHDX_LOG_DESC_SIGNATURE 0x63736564 +#define VHDX_LOG_ZERO_SIGNATURE 0x6f72657a typedef struct QEMU_PACKED VHDXLogDescriptor {uint32_t signature;/* "zero" or "desc" in ASCII */ union {
@@ -194,6 +198,7 @@ typedef struct QEMU_PACKED VHDXLogDescriptor {vhdx_log_entry_header */ } VHDXLogDescriptor; +#define VHDX_LOG_DATA_SIGNATURE 0x61746164 typedef struct QEMU_PACKED VHDXLogDataSector {uint32_t data_signature;/* "data" in ASCII */ uint32_t sequence_high;/* 4 MSB of 8 byte sequence_number */
@@ -318,6 +323,18 @@ typedef struct VHDXMetadataEntries {uint16_t present; } VHDXMetadataEntries; +typedef struct VHDXLogEntries { +uint64_t offset; +uint64_t length; +uint32_t head; +uint32_t tail; +}VHDXLogEntries ; + +typedef struct VHDXLogEntryInfo { +uint64_t sector_start; +uint32_t desc_count; +}VHDXLogEntryInfo ; + typedef struct BDRVVHDXState { CoMutex lock;
@@ -351,6 +368,8 @@ typedef struct BDRVVHDXState {MSGUID session_guid; +VHDXLogEntries log; +VHDXParentLocatorHeader parent_header;VHDXParentLocatorEntry *parent_entries;
Signature · 1 line
1.8.1.4
Re: [Qemu-devel] [PATCH v2 4/9] block: vhdx - log support struct and defines
Quoted 5 lines · Jeff Cody
block/vhdx.h +1
-0@@ -318,6 +323,18 @@ typedef struct VHDXMetadataEntries {uint16_t present; } VHDXMetadataEntries; +typedef struct VHDXLogEntries {
block/vhdx.h typedef struct VHDXMetadataEntries { · line 336
Quoted 10 lines
block/vhdx.h +10
-0+uint64_t offset; +uint64_t length; +uint32_t head; +uint32_t tail; +} VHDXLogEntries; + +typedef struct VHDXLogEntryInfo { +uint64_t sector_start; +uint32_t desc_count; +}VHDXLogEntryInfo ;
An array of VHDXLogEntryInfo structs would be affected by alignment on x86_64. &a[1] != (void*)a + sizeof(VHDXLogEntryInfo).
IMO all on-disk structs should be QEMU_PACKED for safety.
Re: [Qemu-devel] [PATCH v2 4/9] block: vhdx - log support struct and defines
Quoted 20 lines · Stefan Hajnoczi
Quoted 5 lines · Jeff Cody
block/vhdx.h +1
-0@@ -318,6 +323,18 @@ typedef struct VHDXMetadataEntries {uint16_t present; } VHDXMetadataEntries; +typedef struct VHDXLogEntries {block/vhdx.h typedef struct VHDXMetadataEntries { · line 336
Quoted 10 lines
block/vhdx.h +10
-0+uint64_t offset; +uint64_t length; +uint32_t head; +uint32_t tail; +} VHDXLogEntries; + +typedef struct VHDXLogEntryInfo { +uint64_t sector_start; +uint32_t desc_count; +}VHDXLogEntryInfo ;An array of VHDXLogEntryInfo structs would be affected by alignment on
x86_64. &a[1] != (void*)a + sizeof(VHDXLogEntryInfo).
Quoted 1 line
IMO all on-disk structs should be
QEMU_PACKEDfor safety.
Neither of the two structs above reflect on-disk structs (actually VHDXLogEntryInfo is currently unused, so I should drop it from this patch).
I should mention this in the commit message, since the patch context isn't enough to show it, but the structs introduced in this patch are below a comment demarcation line that calls out an end to the on-disk structs:
/* ----- END VHDX SPECIFICATION STRUCTURES ---- */
Re: [Qemu-devel] [PATCH v2 4/9] block: vhdx - log support struct and defines
In reply to Jeff CodyAm 01.08.2013 um 05:23 hat Jeff Cody geschrieben:
Quoted 13 lines
This adds some magic number defines, and internal structure definitions for VHDX log replay support.
Signed-off-by Jeff Cody <[email protected]>--- block/vhdx.h | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-)block/vhdx.h +0
-0
block/vhdx.h typedef struct QEMU_PACKED VHDXRegionTableEntry { · line 154
Quoted 4 lines
block/vhdx.h +1
-0@@ -151,7 +151,10 @@ typedef struct QEMU_PACKED VHDXRegionTableEntry {/* ---- LOG ENTRY STRUCTURES ---- */ +#define VHDX_LOG_MIN_SIZE (1024*1024)
There should be spaces around the operator.
Also, VHDX_LOG_ENTRY_MIN_SIZE? I thought at first that this was the minimum size of the whole log (or does that happen to be the same and I just missed it in the spec?)
Kevin
Re: [Qemu-devel] [PATCH v2 4/9] block: vhdx - log support struct and defines
Quoted 16 lines · Kevin Wolf
Am 01.08.2013 um 05:23 hat Jeff Cody geschrieben:
Quoted 15 lines
This adds some magic number defines, and internal structure definitions for VHDX log replay support.
Signed-off-by Jeff Cody <[email protected]>--- block/vhdx.h | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-)block/vhdx.h +0
-0@@ -151,7 +151,10 @@ typedef struct QEMU_PACKED VHDXRegionTableEntry {
Quoted 5 lines
block/vhdx.h typedef struct QEMU_PACKED VHDXRegionTableEntry { · line 154
Quoted 2 lines
block/vhdx.h +1
-0/* ---- LOG ENTRY STRUCTURES ---- */ +#define VHDX_LOG_MIN_SIZE (1024*1024)There should be spaces around the operator.
OK. Checkpatch.pl missed this, I guess it doesn't check macros?
Quoted 4 lines
Also,
VHDX_LOG_ENTRY_MIN_SIZE? I thought at first that this was the minimum size of the whole log (or does that happen to be the same and I just missed it in the spec?)
This is indeed the minimum size for the entire log - once you get to patch 7, you'll see it used. But an interesting thing about this, is the original 0.95 spec said that 1MB was also the minimum log *entry* size as well. But that was incorrect, and the 1.00 spec notes that a log entry size min is 4KB (same as the log sector size).
Jeff
PATCH v2 5/9 block: vhdx - break endian translation functions out
In reply to Jeff CodyThis moves the endian translation functions out from the vhdx.c source, into a separate source file. In addition to the previously defined endian functions, new endian translation functions for log support are added as well.
--- block/Makefile.objs | 2 +- block/vhdx-endian.c | 141 ++++++++++++++++++++++++++++++++++++++++++++++++++++ block/vhdx.c | 43 ---------------- block/vhdx.h | 13 +++++ 4 files changed, 155 insertions(+), 44 deletions(-) create mode 100644 block/vhdx-endian.c
block/Makefile.objs +1 -1
@@ -2,7 +2,7 @@ block-obj-y += raw.o cow.o qcow.o vdi.o vmdk.o cloop.o dmg.o bochs.o vpc.o vvfat block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o qcow2-cache.o block-obj-y += qed.o qed-gencb.o qed-l2-cache.o qed-table.o qed-cluster.o block-obj-y += qed-check.o -block-obj-$(CONFIG_VHDX) += vhdx.o +block-obj-$(CONFIG_VHDX) += vhdx.o vhdx-endian.o block-obj-y += parallels.o blkdebug.o blkverify.o block-obj-y += snapshot.o qapi.o block-obj-$(CONFIG_WIN32) += raw-win32.o win32-aio.o
block/vhdx-endian.c new file +141 -0
141 lines not on this page · raw message
block/vhdx.c +0 -43
49 lines not on this page · raw message
block/vhdx.h +13 -0
@@ -398,4 +398,17 @@ static inline void cpu_to_leguids(MSGUID *guid) cpu_to_le16s(&guid->data3); } +void vhdx_header_le_import (VHDXHeader *h); +void vhdx_header_le_export (VHDXHeader *orig_h,VHDXHeader *new_h); +void vhdx_log_desc_le_import (VHDXLogDescriptor *d); +void vhdx_log_desc_le_export (VHDXLogDescriptor *d); +void vhdx_log_data_le_export (VHDXLogDataSector *d); +void vhdx_log_entry_hdr_le_import (VHDXLogEntryHeader *hdr); +void vhdx_log_entry_hdr_le_export (VHDXLogEntryHeader *hdr); + + + + + +#endif
Signature · 1 line
1.8.1.4
Re: [Qemu-devel] [PATCH v2 5/9] block: vhdx - break endian translation functions out
Quoted 12 lines · Jeff Cody
block/vhdx.h +4
-0@@ -398,4 +398,17 @@ static inline void cpu_to_leguids(MSGUID *guid)cpu_to_le16s (&guid->data3); } +void vhdx_header_le_import (VHDXHeader *h); +void vhdx_header_le_export (VHDXHeader *orig_h,VHDXHeader *new_h); +void vhdx_log_desc_le_import (VHDXLogDescriptor *d); +void vhdx_log_desc_le_export (VHDXLogDescriptor *d);
block/vhdx.h static inline void cpu_to_leguids(MSGUID *guid) · line 414
Quoted 10 lines
block/vhdx.h +9
-0+void vhdx_log_data_le_export (VHDXLogDataSector *d); +void vhdx_log_entry_hdr_le_import (VHDXLogEntryHeader *hdr); +void vhdx_log_entry_hdr_le_export (VHDXLogEntryHeader *hdr); + + + + + +#endif
You are a generous man when it comes to newlines!
Re: [Qemu-devel] [PATCH v2 5/9] block: vhdx - break endian translation functions out
Quoted 9 lines · Stefan Hajnoczi
Quoted 8 lines · Jeff Cody
block/vhdx.h +0
-0@@ -398,4 +398,17 @@ static inline void cpu_to_leguids(MSGUID *guid)cpu_to_le16s (&guid->data3); }
Quoted 16 lines
Quoted 4 lines
block/vhdx.h +4
-0+void vhdx_header_le_import (VHDXHeader *h); +void vhdx_header_le_export (VHDXHeader *orig_h,VHDXHeader *new_h); +void vhdx_log_desc_le_import (VHDXLogDescriptor *d); +void vhdx_log_desc_le_export (VHDXLogDescriptor *d);block/vhdx.h static inline void cpu_to_leguids(MSGUID *guid) · line 414
Quoted 10 lines
block/vhdx.h +9
-0+void vhdx_log_data_le_export (VHDXLogDataSector *d); +void vhdx_log_entry_hdr_le_import (VHDXLogEntryHeader *hdr); +void vhdx_log_entry_hdr_le_export (VHDXLogEntryHeader *hdr); + + + + + +#endif You are a generous man when it comes to newlines!
Thanks!
-Jeff
Re: [Qemu-devel] [PATCH v2 5/9] block: vhdx - break endian translation functions out
In reply to Jeff CodyAm 01.08.2013 um 05:23 hat Jeff Cody geschrieben:
Quoted 26 lines
This moves the endian translation functions out from the
vhdx.csource, into a separate source file. In addition to the previously defined endian functions, new endian translation functions for log support are added as well.Signed-off-by Jeff Cody <[email protected]>block/vhdx-endian.c new file +19
-0+void vhdx_log_entry_hdr_le_import (VHDXLogEntryHeader *hdr) +{ +assert (hdr !=NULL ); + +le32_to_cpus (&hdr->signature); +le32_to_cpus (&hdr->checksum); +le32_to_cpus (&hdr->entry_length); +le32_to_cpus (&hdr->tail); +le64_to_cpus (&hdr->sequence_number); +le32_to_cpus (&hdr->descriptor_count); +leguid_to_cpus (&hdr->log_guid); +le64_to_cpus (&hdr->flushed_file_offset); +le64_to_cpus (&hdr->last_file_offset); +} + +void vhdx_log_entry_hdr_le_export (VHDXLogEntryHeader *hdr) +{ +assert (hdr !=NULL ); +
block/vhdx-endian.c line 139
Quoted 10 lines
block/vhdx-endian.c new file +10
-0+cpu_to_le32s (&hdr->signature); +cpu_to_le32s (&hdr->checksum); +cpu_to_le32s (&hdr->entry_length); +cpu_to_le32s (&hdr->tail); +cpu_to_le64s (&hdr->sequence_number); +cpu_to_le32s (&hdr->descriptor_count); +cpu_to_le64s (&hdr->flushed_file_offset); +cpu_to_le64s (&hdr->last_file_offset); +cpu_to_leguids (&hdr->log_guid); +}
Almost as critical as Stefan's comment: Can you keep the order of fields consistent between import and export?
Kevin
Re: [Qemu-devel] [PATCH v2 5/9] block: vhdx - break endian translation functions out
Quoted 32 lines · Kevin Wolf
Am 01.08.2013 um 05:23 hat Jeff Cody geschrieben:
Quoted 31 lines
This moves the endian translation functions out from the
vhdx.csource, into a separate source file. In addition to the previously defined endian functions, new endian translation functions for log support are added as well.Signed-off-by Jeff Cody <[email protected]>block/vhdx-endian.c new file +24
-0+void vhdx_log_entry_hdr_le_import (VHDXLogEntryHeader *hdr) +{ +assert (hdr !=NULL ); + +le32_to_cpus (&hdr->signature); +le32_to_cpus (&hdr->checksum); +le32_to_cpus (&hdr->entry_length); +le32_to_cpus (&hdr->tail); +le64_to_cpus (&hdr->sequence_number); +le32_to_cpus (&hdr->descriptor_count); +leguid_to_cpus (&hdr->log_guid); +le64_to_cpus (&hdr->flushed_file_offset); +le64_to_cpus (&hdr->last_file_offset); +} + +void vhdx_log_entry_hdr_le_export (VHDXLogEntryHeader *hdr) +{ +assert (hdr !=NULL ); + +cpu_to_le32s (&hdr->signature); +cpu_to_le32s (&hdr->checksum); +cpu_to_le32s (&hdr->entry_length); +cpu_to_le32s (&hdr->tail); +cpu_to_le64s (&hdr->sequence_number);
Quoted 9 lines
block/vhdx-endian.c line 139
Quoted 5 lines
block/vhdx-endian.c new file +5
-0+cpu_to_le32s (&hdr->descriptor_count); +cpu_to_le64s (&hdr->flushed_file_offset); +cpu_to_le64s (&hdr->last_file_offset); +cpu_to_leguids (&hdr->log_guid); +}Almost as critical as Stefan's comment: Can you keep the order of fields consistent between import and export?
Absolutely! :)
PATCH v2 6/9 block: vhdx - update log guid in header, and first write tracker
In reply to Jeff CodyAllow tracking of first file write in the VHDX image, as well as the ability to update the GUID in the header. This is in preparation for log support.
--- block/vhdx.c | 28 +++++++++++++++++++++++----- block/vhdx.h | 7 +++++-- 2 files changed, 28 insertions(+), 7 deletions(-)
block/vhdx.c +23 -5
@@ -233,7 +233,8 @@ static int vhdx_probe(const uint8_t *buf, int buf_size, const char *filename)* * - non-current header is updated with largest sequence number */ -static int vhdx_update_header (BlockDriverState *bs,BDRVVHDXState *s,bool rw) +static int vhdx_update_header (BlockDriverState *bs,BDRVVHDXState *s,bool rw, +MSGUID *log_guid) {int ret =0 ;int hdr_idx =0 ;
@@ -266,6 +267,11 @@ static int vhdx_update_header(BlockDriverState *bs, BDRVVHDXState *s, bool rw)vhdx_guid_generate (&inactive_header->data_write_guid); } +/* update the log guid if present */ +if (log_guid) { +memcpy (&inactive_header->log_guid, log_guid,sizeof (MSGUID )); + } +/* the header checksum is not over just the packed size of VHDXHeader, * but rather over the entire 'reserved' range for the header, which is * 4KB (VHDX_HEADER_SIZE). */
@@ -293,15 +299,16 @@ exit:* The VHDX spec calls for header updates to be performed twice, so that both * the current and non-current header have valid info */ -static int vhdx_update_headers (BlockDriverState *bs,BDRVVHDXState *s,bool rw) +int vhdx_update_headers (BlockDriverState *bs,BDRVVHDXState *s,bool rw, +MSGUID *log_guid) {int ret; - ret =vhdx_update_header (bs, s, rw); + ret =vhdx_update_header (bs, s, rw, log_guid);if (ret <0 ) {return ret; } - ret =vhdx_update_header (bs, s, rw); + ret =vhdx_update_header (bs, s, rw, log_guid);return ret; }
@@ -781,6 +788,7 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags) s->bat =NULL ; + s->first_visible_write = true;qemu_co_mutex_init (&s->lock);
@@ -861,7 +869,7 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags) }if (flags &BDRV_O_RDWR ) { -vhdx_update_headers (bs, s, false); +vhdx_update_headers (bs, s, false,NULL ); }/* TODO: differencing files, write */
@@ -998,6 +1006,16 @@ exit: +/* Per the spec, on the first write of guest-visible data to the file the +* data write guid must be updated in the header */ +void vhdx_user_visible_write (BlockDriverState *bs,BDRVVHDXState *s) +{ +if (s->first_visible_write) { + s->first_visible_write = false; +vhdx_update_headers (bs, s, true,NULL ); + } +} + static coroutine_fn int vhdx_co_writev(BlockDriverState *bs, int64_t sector_num,int nb_sectors,QEMUIOVector *qiov) {
block/vhdx.h +5 -2
@@ -366,6 +366,7 @@ typedef struct BDRVVHDXState {VHDXBatEntry *bat;uint64_t bat_offset; +bool first_visible_write;MSGUID session_guid;VHDXLogEntries log;
@@ -377,6 +378,9 @@ typedef struct BDRVVHDXState {void vhdx_guid_generate (MSGUID *guid); +int vhdx_update_headers (BlockDriverState *bs,BDRVVHDXState *s,bool rw, +MSGUID *log_guid); +uint32_t vhdx_update_checksum (uint8_t *buf,size_t size,int crc_offset);uint32_t vhdx_checksum_calc (uint32_t crc,uint8_t *buf,size_t size,int crc_offset);
@@ -406,8 +410,7 @@ void vhdx_log_data_le_export(VHDXLogDataSector *d);void vhdx_log_entry_hdr_le_import (VHDXLogEntryHeader *hdr);void vhdx_log_entry_hdr_le_export (VHDXLogEntryHeader *hdr); - - +void vhdx_user_visible_write (BlockDriverState *bs,BDRVVHDXState *s);
Signature · 1 line
1.8.1.4
Re: [Qemu-devel] [PATCH v2 6/9] block: vhdx - update log guid in header, and first write tracker
block/vhdx.c exit: · line 1015
Quoted 11 lines · Jeff Cody
block/vhdx.c +7
-0@@ -998,6 +1006,16 @@ exit: +/* Per the spec, on the first write of guest-visible data to the file the +* data write guid must be updated in the header */ +void vhdx_user_visible_write (BlockDriverState *bs,BDRVVHDXState *s) +{ +if (s->first_visible_write) { + s->first_visible_write = false; +vhdx_update_headers (bs, s, true,NULL );
Error handling is missing.
PATCH v2 7/9 block: vhdx - log parsing, replay, and flush support
In reply to Jeff CodyThis adds support for VHDX v0 logs, as specified in Microsoft's VHDX Specification Format v1.00:
https://www.microsoft.com/en-us/download/details.aspx?id=34750
The following support is added:
* Log parsing, and validation - validate that an existing log is correct.
* Log search - search through an existing log, to find any valid sequence of entries.
* Log replay and flush - replay an existing log, and flush/clear the log when complete.
The VHDX log is a circular buffer, with elements (sectors) of 4KB.
A log entry is a variably-length number of sectors, that is comprised of a header and 'descriptors', that describe each sector.
A log may contain multiple entries, know as a log sequence. In a log sequence, each log entry immediately follows the previous entry, with an incrementing sequence number. There can only ever be one active and valid sequence in the log.
Each log entry must match the file log GUID in order to be valid (along with other criteria). Once we have flushed all valid log entries, we marked the file log GUID to be zero, which indicates a buffer with no valid entries.
--- block/Makefile.objs | 2 +- block/vhdx-log.c | 731 ++++++++++++++++++++++++++++++++++++++++++++++++++++ block/vhdx.c | 44 +--- block/vhdx.h | 7 +- 4 files changed, 740 insertions(+), 44 deletions(-) create mode 100644 block/vhdx-log.c
block/Makefile.objs +1 -1
@@ -2,7 +2,7 @@ block-obj-y += raw.o cow.o qcow.o vdi.o vmdk.o cloop.o dmg.o bochs.o vpc.o vvfat block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o qcow2-cache.o block-obj-y += qed.o qed-gencb.o qed-l2-cache.o qed-table.o qed-cluster.o block-obj-y += qed-check.o -block-obj-$(CONFIG_VHDX) += vhdx.o vhdx-endian.o +block-obj-$(CONFIG_VHDX) += vhdx.o vhdx-endian.o vhdx-log.o block-obj-y += parallels.o blkdebug.o blkverify.o block-obj-y += snapshot.o qapi.o block-obj-$(CONFIG_WIN32) += raw-win32.o win32-aio.o
block/vhdx-log.c new file +731 -0
731 lines not on this page · raw message
block/vhdx.c +2 -42
62 lines not on this page · raw message
block/vhdx.h +6 -1
@@ -326,7 +326,11 @@ typedef struct VHDXMetadataEntries {typedef struct VHDXLogEntries {uint64_t offset;uint64_t length; -uint32_t head; +uint32_t write; +uint32_t read; +VHDXLogEntryHeader *hdr; +void *desc_buffer; +uint64_t sequence;uint32_t tail; }VHDXLogEntries ;
@@ -387,6 +391,7 @@ uint32_t vhdx_checksum_calc(uint32_t crc, uint8_t *buf, size_t size,bool vhdx_checksum_is_valid (uint8_t *buf,size_t size,int crc_offset); +int vhdx_parse_log (BlockDriverState *bs,BDRVVHDXState *s);static inline void leguid_to_cpus (MSGUID *guid) {
Signature · 1 line
1.8.1.4
PATCH v2 8/9 block: vhdx - add log write support
In reply to Jeff CodyThis adds support for writing to the VHDX log.
For spec details, see VHDX Specification Format v1.00:
https://www.microsoft.com/en-us/download/details.aspx?id=34750
There are a few limitations to this log support:
1.) There is no caching yet
2.) The log is flushed after each entry
The primary write interface, vhdx_log_write_and_flush(), performs a log write followed by an immediate flush of the log.
As each log entry sector is a minimum of 4KB, partial sector writes are filled in with data from the disk write destination.
If the current file log GUID is 0, a new GUID is generated and updated in the header.
--- block/vhdx-log.c | 274 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ block/vhdx.h | 3 + 2 files changed, 277 insertions(+)
block/vhdx-log.c +274 -0
@@ -170,6 +170,53 @@ exit:return ret; } +/* Writes num_sectors to the log (all log sectors are 4096 bytes), +* from buffer 'buffer'. Upon return, *sectors_written will contain +* the number of sectors successfully written. +* +* It is assumed that 'buffer' is at least 4096*num_sectors large. +* +* 0 is returned on success, -errno otherwise */ +static int vhdx_log_write_sectors (BlockDriverState *bs,VHDXLogEntries *log, +uint32_t *sectors_written,void *buffer, +uint32_t num_sectors) +{ +int ret =0 ; +uint64_t offset; +uint32_t write; +void *buffer_tmp; +BDRVVHDXState *s = bs->opaque; + +vhdx_user_visible_write (bs, s); + + write = log->write; + + buffer_tmp = buffer; +while (num_sectors) { + + offset = log->offset + write; + write =vhdx_log_inc_idx (write, log->length); +if (write == log->read) { +/* full */ +break ; + } + ret =bdrv_pwrite_sync (bs->file, offset, buffer_tmp, +VHDX_LOG_SECTOR_SIZE ); +if (ret <0 ) { + goto exit; + } + buffer_tmp +=VHDX_LOG_SECTOR_SIZE ; + + log->write = write; + *sectors_written = *sectors_written +1 ; + num_sectors--; + } + +exit: +return ret; +} + +/* Validates a log entry header */ static bool vhdx_log_hdr_is_valid (VHDXLogEntries *log,VHDXLogEntryHeader *hdr,BDRVVHDXState *s)
@@ -729,3 +776,230 @@ exit:return ret; } + + +static void vhdx_log_raw_to_le_sector (VHDXLogDescriptor *desc, +VHDXLogDataSector *sector,void *data, +uint64_t seq) +{ +/* 8 + 4084 + 4 = 4096, 1 log sector */ +memcpy (&desc->leading_bytes, data,8 ); + data +=8 ; +cpu_to_le64s (&desc->leading_bytes); +memcpy (sector->data, data,4084 ); + data +=4084 ; +memcpy (&desc->trailing_bytes, data,4 ); +cpu_to_le32s (&desc->trailing_bytes); + data +=4 ; + + sector->sequence_high = (uint32_t ) (seq >>32 ); + sector->sequence_low = (uint32_t ) (seq &0xffffffff ); + sector->data_signature =VHDX_LOG_DATA_SIGNATURE ; + +vhdx_log_desc_le_export (desc); +vhdx_log_data_le_export (sector); +} + + +static int vhdx_log_write (BlockDriverState *bs,BDRVVHDXState *s, +void *data,uint32_t length,uint64_t offset) +{ +int ret =0 ; +void *buffer =NULL ; +void *merged_sector =NULL ; +void *data_tmp, *sector_write; +unsigned int i; +int sector_offset; +uint32_t desc_sectors, sectors, total_length; +uint32_t sectors_written =0 ; +uint32_t aligned_length; +uint32_t leading_length =0 ; +uint32_t trailing_length =0 ; +uint32_t partial_sectors =0 ; +uint32_t bytes_written =0 ; +uint64_t file_offset; +VHDXHeader *header; +VHDXLogEntryHeader new_hdr; +VHDXLogDescriptor *new_desc =NULL ; +VHDXLogDataSector *data_sector =NULL ; +MSGUID new_guid = {0 }; + + header = s->headers[s->curr_header]; + +/* need to have offset read data, and be on 4096 byte boundary */ + +if (length > header->log_length) { +/* no log present. we could create a log here instead of failing */ + ret = -EINVAL ; + goto exit; + } + +if (vhdx_log_guid_is_zero (&header->log_guid)) { +vhdx_guid_generate (&new_guid); +vhdx_update_headers (bs, s, false, &new_guid); + }else { +/* currently, we require that the log be flushed after +* every write. */ + ret = -ENOTSUP ; + } + +/* 0 is an invalid sequence number, but may also represent the first +* log write (or a wrapped seq) */ +if (s->log.sequence ==0 ) { + s->log.sequence =1 ; + } + + sector_offset = offset %VHDX_LOG_SECTOR_SIZE ; + file_offset = (offset /VHDX_LOG_SECTOR_SIZE ) *VHDX_LOG_SECTOR_SIZE ; + + aligned_length = length; + +/* add in the unaligned head and tail bytes */ +if (sector_offset) { + leading_length = (VHDX_LOG_SECTOR_SIZE - sector_offset); + leading_length = leading_length > length ? length : leading_length; + aligned_length -= leading_length; + partial_sectors++; + } + + sectors = aligned_length /VHDX_LOG_SECTOR_SIZE ; + trailing_length = aligned_length - (sectors *VHDX_LOG_SECTOR_SIZE ); +if (trailing_length) { + partial_sectors++; + } + + sectors += partial_sectors; + +/* sectors is now how many sectors the data itself takes, not +* including the header and descriptor metadata */ + + new_hdr = (VHDXLogEntryHeader ) { + .signature =VHDX_LOG_SIGNATURE , + .tail = s->log.tail, + .sequence_number = s->log.sequence, + .descriptor_count = sectors, + .reserved =0 , + .flushed_file_offset =bdrv_getlength (bs->file), + .last_file_offset =bdrv_getlength (bs->file), + }; + +memcpy (&new_hdr.log_guid, &header->log_guid,sizeof (MSGUID )); + + desc_sectors =vhdx_compute_desc_sectors (new_hdr.descriptor_count); + + total_length = (desc_sectors + sectors) *VHDX_LOG_SECTOR_SIZE ; + new_hdr.entry_length = total_length; + +vhdx_log_entry_hdr_le_export (&new_hdr); + + buffer =qemu_blockalign (bs, total_length); +memcpy (buffer, &new_hdr,sizeof (new_hdr)); + + new_desc = (VHDXLogDescriptor *) (buffer +sizeof (new_hdr)); + data_sector = buffer + (desc_sectors *VHDX_LOG_SECTOR_SIZE ); + data_tmp = data; + +/* All log sectors are 4KB, so for any partial sectors we must +* merge the data with preexisting data from the final file +* destination */ + merged_sector =qemu_blockalign (bs,VHDX_LOG_SECTOR_SIZE ); + +for (i =0 ; i < sectors; i++) { + new_desc->signature =VHDX_LOG_DESC_SIGNATURE ; + new_desc->sequence_number = s->log.sequence; + new_desc->file_offset = file_offset; + +if (i ==0 && leading_length) { +/* partial sector at the front of the buffer */ + ret =bdrv_pread (bs->file, file_offset, merged_sector, +VHDX_LOG_SECTOR_SIZE ); +if (ret <0 ) { + goto exit; + } +memcpy (merged_sector + sector_offset, data_tmp, leading_length); + bytes_written = leading_length; + sector_write = merged_sector; + }else if (i == sectors -1 && trailing_length) { +/* partial sector at the end of the buffer */ + ret =bdrv_pread (bs->file, + file_offset, + merged_sector + trailing_length, +VHDX_LOG_SECTOR_SIZE - trailing_length); +if (ret <0 ) { + goto exit; + } +memcpy (merged_sector, data_tmp, trailing_length); + bytes_written = trailing_length; + sector_write = merged_sector; + }else { + bytes_written =VHDX_LOG_SECTOR_SIZE ; + sector_write = data_tmp; + } + +/* populate the raw sector data into the proper structures, +* as well as update the descriptor, and convert to proper +* endianness */ +vhdx_log_raw_to_le_sector (new_desc, data_sector, sector_write, + s->log.sequence); + + data_tmp += bytes_written; + data_sector++; + new_desc++; + file_offset +=VHDX_LOG_SECTOR_SIZE ; + } + +/* checksum covers entire entry, from the log header through the +* last data sector */ +vhdx_update_checksum (buffer, total_length,4 ); +cpu_to_le32s ((uint32_t *)(buffer +4 )); + +/* now write to the log */ +vhdx_log_write_sectors (bs, &s->log, §ors_written, buffer, + desc_sectors + sectors); +if (ret <0 ) { + goto exit; + } + +if (sectors_written != desc_sectors + sectors) { +/* instead of failing, we could flush the log here */ + ret = -EINVAL ; + goto exit; + } + + s->log.sequence++; +/* write new tail */ + s->log.tail = s->log.write; + +exit: +qemu_vfree (buffer); +qemu_vfree (merged_sector); +return ret; +} + +/* Perform a log write, and then immediately flush the entire log */ +int vhdx_log_write_and_flush (BlockDriverState *bs,BDRVVHDXState *s, +void *data,uint32_t length,uint64_t offset) +{ +int ret =0 ; +VHDXLogSequence logs = { .valid = true, + .count =1 , + .hdr = {0 } }; + + + ret =vhdx_log_write (bs, s, data, length, offset); +if (ret <0 ) { + goto exit; + } + logs.log = s->log; + + ret =vhdx_log_flush (bs, s, &logs); +if (ret <0 ) { + goto exit; + } + + s->log = logs.log; + +exit: +return ret; +} +
block/vhdx.h +3 -0
@@ -393,6 +393,9 @@ bool vhdx_checksum_is_valid(uint8_t *buf, size_t size, int crc_offset);int vhdx_parse_log (BlockDriverState *bs,BDRVVHDXState *s); +int vhdx_log_write_and_flush (BlockDriverState *bs,BDRVVHDXState *s, +void *data,uint32_t length,uint64_t offset); +static inline void leguid_to_cpus (MSGUID *guid) {le32_to_cpus (&guid->data1);
Signature · 1 line
1.8.1.4
PATCH v2 9/9 block: vhdx write support
In reply to Jeff CodyThis adds support for writing to VHDX image files, using coroutines. Writes into the BAT table goes through the VHDX log. Currently, BAT table writes occur when expanding a dynamic VHDX file, and allocating a new BAT entry.
--- block/vhdx.c | 147 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 2 deletions(-)
block/vhdx.c +145 -2
@@ -831,7 +831,7 @@ static int vhdx_open(BlockDriverState *bs, QDict *options, int flags)vhdx_update_headers (bs, s, false,NULL ); } -/* TODO: differencing files, write */ +/* TODO: differencing files */ return 0 ; fail:
@@ -963,7 +963,43 @@ exit:return ret; } +/* +* Allocate a new payload block at the end of the file. +* +* Allocation will happen at 1MB alignment inside the file +* +* Returns the file offset start of the new payload block +*/ +static int vhdx_allocate_block (BlockDriverState *bs,BDRVVHDXState *s, +uint64_t *new_offset) +{ + *new_offset =bdrv_getlength (bs->file); + +/* per the spec, the address for a block is in units of 1MB */ + *new_offset =ROUND_UP (*new_offset,1024 *1024 ); + +return bdrv_truncate (bs->file, *new_offset + s->block_size); +} + +/* +* Update the BAT tablet entry with the new file offset, and the new entry +* state */ +static void vhdx_update_bat_table_entry (BlockDriverState *bs,BDRVVHDXState *s, +VHDXSectorInfo *sinfo, +uint64_t *bat_entry, +uint64_t *bat_offset,int state) +{ +/* The BAT entry is a uint64, with 44 bits for the file offset in units of +* 1MB, and 3 bits for the block state. */ + s->bat[sinfo->bat_idx] = ((sinfo->file_offset>>20 ) << +VHDX_BAT_FILE_OFF_BITS ); + + s->bat[sinfo->bat_idx] |= state &VHDX_BAT_STATE_BIT_MASK ; + *bat_entry =cpu_to_le64 (s->bat[sinfo->bat_idx]); + *bat_offset = s->bat_offset + sinfo->bat_idx *sizeof (VHDXBatEntry); + +}/* Per the spec, on the first write of guest-visible data to the file the * data write guid must be updated in the header */
@@ -978,7 +1014,114 @@ void vhdx_user_visible_write(BlockDriverState *bs, BDRVVHDXState *s) static coroutine_fn int vhdx_co_writev(BlockDriverState *bs, int64_t sector_num,int nb_sectors,QEMUIOVector *qiov) { -return -ENOTSUP ; +int ret = -ENOTSUP ; +BDRVVHDXState *s = bs->opaque; +VHDXSectorInfo sinfo; +uint64_t bytes_done =0 ; +uint64_t bat_entry =0 ; +uint64_t bat_entry_offset =0 ; +bool bat_update; +QEMUIOVector hd_qiov; + +qemu_iovec_init (&hd_qiov, qiov->niov); + +qemu_co_mutex_lock (&s->lock); + +vhdx_user_visible_write (bs, s); + +while (nb_sectors >0 ) { +if (s->params.data_bits &VHDX_PARAMS_HAS_PARENT ) { +/* not supported yet */ + ret = -ENOTSUP ; + goto exit; + }else { + bat_update = false; +vhdx_block_translate (s, sector_num, nb_sectors, &sinfo); + +qemu_iovec_reset (&hd_qiov); +qemu_iovec_concat (&hd_qiov, qiov, bytes_done, sinfo.bytes_avail); +/* check the payload block state */ +switch (s->bat[sinfo.bat_idx] &VHDX_BAT_STATE_BIT_MASK ) { +case PAYLOAD_BLOCK_ZERO : +/* in this case, we need to preserve zero writes for +* data that is not part of this write, so we must pad +* the rest of the buffer to zeroes */ + +/* if we are on a posix system with ftruncate() that extends +* a file, then it is zero-filled for us. On Win32, the raw +* layer uses SetFilePointer and SetFileEnd, which does not +* zero fill AFAIK */ + +/* TODO: queue another write of zero buffers if the host OS does +* not zero-fill on file extension */ + +/* fall through */ +case PAYLOAD_BLOCK_NOT_PRESENT :/* fall through */ +case PAYLOAD_BLOCK_UNMAPPED :/* fall through */ +case PAYLOAD_BLOCK_UNDEFINED :/* fall through */ + ret =vhdx_allocate_block (bs, s, &sinfo.file_offset); +if (ret <0 ) { + goto exit; + } +/* once we support differencing files, this may also be +* partially present */ +/* update block state to the newly specified state */ +vhdx_update_bat_table_entry (bs, s, &sinfo, &bat_entry, + &bat_entry_offset, +PAYLOAD_BLOCK_FULL_PRESENT ); + bat_update = true; +/* since we just allocated a block, file_offset is the +* beginning of the payload block. It needs to be the +* write address, which includes the offset into the block */ + sinfo.file_offset += sinfo.block_offset; +/* fall through */ +case PAYLOAD_BLOCK_FULL_PRESENT : +/* if the file offset address is in the header zone, +* there is a problem */ +if (sinfo.file_offset < (1024 *1024 )) { + ret = -EFAULT ; + goto exit; + } +/* block exists, so we can just overwrite it */ +qemu_co_mutex_unlock (&s->lock); + ret =bdrv_co_writev (bs->file, + sinfo.file_offset>>BDRV_SECTOR_BITS , + sinfo.sectors_avail, &hd_qiov); +qemu_co_mutex_lock (&s->lock); +if (ret <0 ) { + goto exit; + } +break ; +case PAYLOAD_BLOCK_PARTIALLY_PRESENT : +/* we don't yet support difference files, fall through +* to error */ +default : + ret = -EIO ; + goto exit; +break ; + } + +if (bat_update) { +/* this will update the BAT entry into the log journal, and +* then flush the log journal out to disk */ + ret =vhdx_log_write_and_flush (bs, s, &bat_entry, +sizeof (VHDXBatEntry), + bat_entry_offset); +if (ret <0 ) { + goto exit; + } + } + + nb_sectors -= sinfo.sectors_avail; + sector_num += sinfo.sectors_avail; + bytes_done += sinfo.bytes_avail; + + } + } + +exit: +qemu_co_mutex_unlock (&s->lock); +return ret; }
Signature · 1 line
1.8.1.4