Skip to contentSkip to conversation

qemu-img: add seek and -n option to dd command

PATCH In review Peter Lieven 5 participants 10 messages Jan 28, 2021 – Feb 5, 2021

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 qemu-img: add seek and -n option to dd command

docs/tools/qemu-img.rst +8 -0
index b615aa8419..7d4564c2b8 100644
@@ -209,6 +209,10 @@ Parameters to dd subcommand:
 
 .. program:: qemu-img-dd
 
+.. option:: -n
+
+  Skip the creation of the output file
+
 .. option:: bs=BLOCK_SIZE
 
   Defines the block size
@@ -229,6 +233,10 @@ Parameters to dd subcommand:
 
   Sets the number of input blocks to skip
 
+.. option:: sseek=BLOCKS
+
+  Sets the number of blocks to seek into the output
+
 Parameters to snapshot subcommand:
 
 .. program:: qemu-img-snapshot
qemu-img.c +63 -34

164 lines not on this page · raw message

Signature · 1 line
2.17.1
#1 lore raw
Quoted 33 lines · Peter Lieven
docs/tools/qemu-img.rst +8 -0
index b615aa8419..7d4564c2b8 100644
@@ -209,6 +209,10 @@ Parameters to dd subcommand:
 
 .. program:: qemu-img-dd
 
+.. option:: -n
+
+  Skip the creation of the output file
+
 .. option:: bs=BLOCK_SIZE
 
   Defines the block size
@@ -229,6 +233,10 @@ Parameters to dd subcommand:
 
   Sets the number of input blocks to skip
 
+.. option:: sseek=BLOCKS
+
+  Sets the number of blocks to seek into the output
+
 Parameters to snapshot subcommand:
 
 .. program:: qemu-img-snapshot
qemu-img.c +0 -0
index 8597d069af..d7f390e382 100644

qemu-img.c static void QEMU_NORETURN help(void) · line 217

Quoted 5 lines
qemu-img.c +2 -0
@@ -213,12 +213,17 @@ static void QEMU_NORETURN help(void)
            "  '-s' run in Strict mode - fail on different image size or sector allocation\n"
            "\n"
            "Parameters to dd subcommand:\n"
+           "  '-n' skips the target volume creation (useful if the volume is created\n"
+           "       prior to running qemu-img). Note that he behaviour is not identical to\n"

s/he/the/

Quoted 170 lines
qemu-img.c +61 -34
+           "       original dd option conv=nocreat. The output is neither truncated nor\n"
+           "       is it possible to write past the end of an existing file.\n"
            "  'bs=BYTES' read and write up to BYTES bytes at a time "
            "(default: 512)\n"
            "  'count=N' copy only N input blocks\n"
            "  'if=FILE' read from FILE\n"
            "  'of=FILE' write to FILE\n"
-           "  'skip=N' skip N bs-sized blocks at the start of input\n";
+           "  'skip=N' skip N bs-sized blocks at the start of input\n"
+           "  'seek=N' seek N bs-sized blocks into the output\n";
 
     printf("%s\nSupported formats:", help_msg);
     bdrv_iterate_format(format_print, NULL, false);
@@ -4885,6 +4890,7 @@ static int img_bitmap(int argc, char **argv)
 #define C_IF      04
 #define C_OF      010
 #define C_SKIP    020
+#define C_SEEK    040
 
 struct DdInfo {
     unsigned int flags;
@@ -4964,6 +4970,19 @@ static int img_dd_skip(const char *arg,
     return 0;
 }
 
+static int img_dd_seek(const char *arg,
+                       struct DdIo *in, struct DdIo *out,
+                       struct DdInfo *dd)
+{
+    out->offset = cvtnum("seek", arg);
+
+    if (in->offset < 0) {
+        return 1;
+    }
+
+    return 0;
+}
+
 static int img_dd(int argc, char **argv)
 {
     int ret = 0;
@@ -4980,7 +4999,7 @@ static int img_dd(int argc, char **argv)
     const char *fmt = NULL;
     int64_t size = 0;
     int64_t block_count = 0, out_pos, in_pos;
-    bool force_share = false;
+    bool force_share = false, skip_create = false;
     struct DdInfo dd = {
         .flags = 0,
         .count = 0,
@@ -5004,6 +5023,7 @@ static int img_dd(int argc, char **argv)
         { "if", img_dd_if, C_IF },
         { "of", img_dd_of, C_OF },
         { "skip", img_dd_skip, C_SKIP },
+        { "seek", img_dd_seek, C_SEEK },
         { NULL, NULL, 0 }
     };
     const struct option long_options[] = {
@@ -5014,7 +5034,7 @@ static int img_dd(int argc, char **argv)
         { 0, 0, 0, 0 }
     };
 
-    while ((c = getopt_long(argc, argv, ":hf:O:U", long_options, NULL))) {
+    while ((c = getopt_long(argc, argv, ":hf:O:Un", long_options, NULL))) {
         if (c == EOF) {
             break;
         }
@@ -5037,6 +5057,9 @@ static int img_dd(int argc, char **argv)
         case 'U':
             force_share = true;
             break;
+        case 'n':
+            skip_create = true;
+            break;
         case OPTION_OBJECT:
             if (!qemu_opts_parse_noisily(&qemu_object_opts, optarg, true)) {
                 ret = -1;
@@ -5116,22 +5139,25 @@ static int img_dd(int argc, char **argv)
         ret = -1;
         goto out;
     }
-    if (!drv->create_opts) {
-        error_report("Format driver '%s' does not support image creation",
-                     drv->format_name);
-        ret = -1;
-        goto out;
-    }
-    if (!proto_drv->create_opts) {
-        error_report("Protocol driver '%s' does not support image creation",
-                     proto_drv->format_name);
-        ret = -1;
-        goto out;
-    }
-    create_opts = qemu_opts_append(create_opts, drv->create_opts);
-    create_opts = qemu_opts_append(create_opts, proto_drv->create_opts);
 
-    opts = qemu_opts_create(create_opts, NULL, 0, &error_abort);
+    if (!skip_create) {
+        if (!drv->create_opts) {
+            error_report("Format driver '%s' does not support image creation",
+                         drv->format_name);
+            ret = -1;
+            goto out;
+        }
+        if (!proto_drv->create_opts) {
+            error_report("Protocol driver '%s' does not support image creation",
+                         proto_drv->format_name);
+            ret = -1;
+            goto out;
+        }
+        create_opts = qemu_opts_append(create_opts, drv->create_opts);
+        create_opts = qemu_opts_append(create_opts, proto_drv->create_opts);
+
+        opts = qemu_opts_create(create_opts, NULL, 0, &error_abort);
+    }
 
     size = blk_getlength(blk1);
     if (size < 0) {
@@ -5145,22 +5171,25 @@ static int img_dd(int argc, char **argv)
         size = dd.count * in.bsz;
     }
 
-    /* Overflow means the specified offset is beyond input image's size */
-    if (dd.flags & C_SKIP && (in.offset > INT64_MAX / in.bsz ||
-                              size < in.bsz * in.offset)) {
-        qemu_opt_set_number(opts, BLOCK_OPT_SIZE, 0, &error_abort);
-    } else {
-        qemu_opt_set_number(opts, BLOCK_OPT_SIZE,
-                            size - in.bsz * in.offset, &error_abort);
-    }
+    if (!skip_create) {
+        /* Overflow means the specified offset is beyond input image's size */
+        if (dd.flags & C_SKIP && (in.offset > INT64_MAX / in.bsz ||
+                                  size < in.bsz * in.offset)) {
+            qemu_opt_set_number(opts, BLOCK_OPT_SIZE, 0, &error_abort);
+        } else {
+            qemu_opt_set_number(opts, BLOCK_OPT_SIZE,
+                                out.bsz * out.offset + size -
+                                in.bsz * in.offset, &error_abort);
+        }
 
-    ret = bdrv_create(drv, out.filename, opts, &local_err);
-    if (ret < 0) {
-        error_reportf_err(local_err,
-                          "%s: error while creating output image: ",
-                          out.filename);
-        ret = -1;
-        goto out;
+        ret = bdrv_create(drv, out.filename, opts, &local_err);
+        if (ret < 0) {
+            error_reportf_err(local_err,
+                              "%s: error while creating output image: ",
+                              out.filename);
+            ret = -1;
+            goto out;
+        }
     }
 
     /* TODO, we can't honour --image-opts for the target,
@@ -5189,7 +5218,7 @@ static int img_dd(int argc, char **argv)
 
     in.buf = g_new(uint8_t, in.bsz);
 
-    for (out_pos = 0; in_pos < size; block_count++) {
+    for (out_pos = out.offset * out.bsz; in_pos < size; block_count++) {
         int in_ret, out_ret;
 
         if (in_pos + in.bsz > size) {
Signature · 1 line
2.17.1

dme.

Signature · 1 line
But they'll laugh at you in Jackson, and I'll be dancin' on a Pony Keg.
#2 lore raw
In reply to Peter Lieven
Quoted 1 line · Peter Lieven

Your commit message says 'what', but not 'why'. Generally, the one-line 'what' works well as the subject line, but you want the commit body to give an argument why your patch should be applied, rather than blank.

Here's the last time we tried to improve qemu-img dd:
https://lists.gnu.org/archive/html/qemu-devel/2018-08/msg02618.html

where I also proposed adding seek=, and fixing skip= with count=. Your patch does not do the latter. But the bigger complaint back then was that 'qemu-img copy' should be able to do everything, and that qemu-img dd should then just be a thin shim around 'qemu-img copy', rather than having two parallel projects that diverge in their implementations.

Your patch does not have the typical '---' divider and diffstat between the commit message and the patch proper; this may be a factor of which git packages you have installed, but having the diffstat present makes it easier to see at a glance what your patch touches without reading the entire email. I had to go hunting to learn if you added iotest coverage of this new feature...

...and the answer was no, you didn't. You'll need to add that in v2 (see the link to my earlier attempt at modifying dd for an example).

Quoted 17 lines
docs/tools/qemu-img.rst +4 -0
index b615aa8419..7d4564c2b8 100644
@@ -209,6 +209,10 @@ Parameters to dd subcommand:
 
 .. program:: qemu-img-dd
 
+.. option:: -n
+
+  Skip the creation of the output file
+
 .. option:: bs=BLOCK_SIZE
 
   Defines the block size

docs/tools/qemu-img.rst Parameters to dd subcommand: · line 236

Quoted 4 lines
docs/tools/qemu-img.rst +1 -0
@@ -229,6 +233,10 @@ Parameters to dd subcommand:
 
   Sets the number of input blocks to skip
 
+.. option:: sseek=BLOCKS

Typo: seek=

Quoted 11 lines
docs/tools/qemu-img.rst +3 -0
+
+  Sets the number of blocks to seek into the output
+
 Parameters to snapshot subcommand:
 
 .. program:: qemu-img-snapshot
qemu-img.c +0 -0
index 8597d069af..d7f390e382 100644

qemu-img.c static void QEMU_NORETURN help(void) · line 217

Quoted 5 lines
qemu-img.c +2 -0
@@ -213,12 +213,17 @@ static void QEMU_NORETURN help(void)
            "  '-s' run in Strict mode - fail on different image size or sector allocation\n"
            "\n"
            "Parameters to dd subcommand:\n"
+           "  '-n' skips the target volume creation (useful if the volume is created\n"
+           "       prior to running qemu-img). Note that he behaviour is not identical to\n"

the

qemu-img.c static void QEMU_NORETURN help(void) · line 219

Quoted 2 lines
qemu-img.c +2 -0
+           "       original dd option conv=nocreat. The output is neither truncated nor\n"
+           "       is it possible to write past the end of an existing file.\n"

and hence why you spell it -n rather than the more dd-like conv=nocreat.
We absolutely want a different spelling to emphasize the different
behavior.

Is it worth splitting this patch in two parts, one for -n, the other for seek=?

Signature · 3 lines
Eric Blake, Principal Software Engineer
Red Hat, Inc.           +1-919-301-3226
Virtualization:  qemu.org | libvirt.org
#3 lore raw

Am 02.02.21 um 16:51 schrieb Eric Blake:

Quoted 8 lines · Peter Lieven
Quoted 1 line

Your commit message says 'what', but not 'why'. Generally, the one-line 'what' works well as the subject line, but you want the commit body to give an argument why your patch should be applied, rather than blank.

Here's the last time we tried to improve qemu-img dd:
https://lists.gnu.org/archive/html/qemu-devel/2018-08/msg02618.html

I was not aware of that story. My use case is that I want to be

able to "patch" an image that Qemu is able to handle by overwriting

certain sectors. And I especially do not want to "mount" that image

via qemu-nbd because I might not trust it. I totally want to avoid that the host

system tries to analyse that image in terms of scanning the bootsector, partprobe,

lvm etc. pp.

Quoted 6 lines

where I also proposed adding seek=, and fixing skip= with count=. Your patch does not do the latter. But the bigger complaint back then was that 'qemu-img copy' should be able to do everything, and that qemu-img dd should then just be a thin shim around 'qemu-img copy', rather than having two parallel projects that diverge in their implementations.

understood. I was not aware of an issue with skip and count.

The patch works for me and I wanted to share it. But when I read

the thread it seems that it would be a difficult task to get it merged.

Quoted 10 lines

Your patch does not have the typical '---' divider and diffstat between the commit message and the patch proper; this may be a factor of which git packages you have installed, but having the diffstat present makes it easier to see at a glance what your patch touches without reading the entire email. I had to go hunting to learn if you added iotest coverage of this new feature...

...and the answer was no, you didn't. You'll need to add that in v2 (see the link to my earlier attempt at modifying dd for an example).

I did not. Maybe I accidently killed the '---' divider. If I will make a V2 I will add

an I/O test.

Thanks for your suggestions,

Peter

#4 lore raw
Quoted 15 lines · Peter Lieven

Am 02.02.21 um 16:51 schrieb Eric Blake:

Quoted 8 lines · Peter Lieven
Quoted 1 line

Your commit message says 'what', but not 'why'. Generally, the one-line 'what' works well as the subject line, but you want the commit body to give an argument why your patch should be applied, rather than blank.

Here's the last time we tried to improve qemu-img dd:
https://lists.gnu.org/archive/html/qemu-devel/2018-08/msg02618.html

I was not aware of that story. My use case is that I want to be

able to "patch" an image that Qemu is able to handle by overwriting

Quoted 7 lines

certain sectors. And I especially do not want to "mount" that image

via qemu-nbd because I might not trust it. I totally want to avoid that the host

system tries to analyse that image in terms of scanning the bootsector, partprobe,

lvm etc. pp.

qemu-nbd does not have to mount an image (yes, one use of qemu-nbd is to use -c /dev/nbdX to get the kernel to mount it; but other uses are to expose the NBD image in user-space only with no kernel involvement, and therefore no system mount efforts).

Another thing you might try is libnbd, which now includes a utility nbdcopy. It should make it easier to overwrite a portion of an NBD image using only user-space actions. I'm not sure if Rich has got it doing partial file overwrites yet (.../me goes and compiles the latest git checkout... nope, still a TODO item to implement subsetting), but it may be possible to combine nbdkit's --filter=offset with the full NBD image in order to then easily point nbdcopy to only the subset you care about. Definitely some ideas worthy of implementation.

Quoted 9 lines
Quoted 7 lines

where I also proposed adding seek=, and fixing skip= with count=. Your patch does not do the latter. But the bigger complaint back then was that 'qemu-img copy' should be able to do everything, and that qemu-img dd should then just be a thin shim around 'qemu-img copy', rather than having two parallel projects that diverge in their implementations.

Quoted 5 lines

understood. I was not aware of an issue with skip and count.

The patch works for me and I wanted to share it. But when I read

the thread it seems that it would be a difficult task to get it merged.

Just because we're reluctant to improve qemu-img dd without also improving qemu-img copy does not mean that your improvements are unwanted. And hacking on nbdcopy may be faster than waiting for qemu-img to catch up to where we want it to go.

Quoted 14 lines
Quoted 10 lines

Your patch does not have the typical '---' divider and diffstat between the commit message and the patch proper; this may be a factor of which git packages you have installed, but having the diffstat present makes it easier to see at a glance what your patch touches without reading the entire email. I had to go hunting to learn if you added iotest coverage of this new feature...

Quoted 2 lines

...and the answer was no, you didn't. You'll need to add that in v2 (see the link to my earlier attempt at modifying dd for an example).

Quoted 6 lines

I did not. Maybe I accidently killed the '---' divider. If I will make a V2 I will add

an I/O test.

Thanks for your suggestions,

Good luck!

Quoted 5 lines

Peter

Signature · 3 lines
Eric Blake, Principal Software Engineer
Red Hat, Inc.           +1-919-301-3226
Virtualization:  qemu.org | libvirt.org
#6 lore raw
In reply to Peter Lieven
Quoted 15 lines · Peter Lieven

Am 02.02.21 um 16:51 schrieb Eric Blake:

Quoted 8 lines · Peter Lieven
Quoted 1 line

Your commit message says 'what', but not 'why'. Generally, the one-line 'what' works well as the subject line, but you want the commit body to give an argument why your patch should be applied, rather than blank.

Here's the last time we tried to improve qemu-img dd:
https://lists.gnu.org/archive/html/qemu-devel/2018-08/msg02618.html

I was not aware of that story. My use case is that I want to be

able to "patch" an image that Qemu is able to handle by overwriting

Quoted 7 lines

certain sectors. And I especially do not want to "mount" that image

via qemu-nbd because I might not trust it. I totally want to avoid that the host

system tries to analyse that image in terms of scanning the bootsector, partprobe,

lvm etc. pp.

qemu will have FUSE exporting as of 6.0 (didn’t quite make it into 5.2), so you can do something like this:

$ qemu-storage-daemon \
    --blockdev node-name=export,driver=qcow2,\
file.driver=file,file.filename=image.qcow2 \
    --export fuse,id=fuse,node-name=export,mountpoint=image.qcow2

This exports the image on image.qcow2 (i.e., on itself) and so by accessing the image file you then get raw access to its contents (so you can use system tools like dd).

Doesn’t require root rights, and shouldn’t make the kernel scan anything, because it’s exported as just a regular file.

Max

#7 lore raw

Am 05.02.21 um 09:18 schrieb Max Reitz:

Quoted 26 lines · Peter Lieven
Quoted 15 lines

Am 02.02.21 um 16:51 schrieb Eric Blake:

Quoted 8 lines · Peter Lieven
Quoted 1 line

Your commit message says 'what', but not 'why'.  Generally, the one-line 'what' works well as the subject line, but you want the commit body to give an argument why your patch should be applied, rather than blank.

Here's the last time we tried to improve qemu-img dd:
https://lists.gnu.org/archive/html/qemu-devel/2018-08/msg02618.html

I was not aware of that story. My use case is that I want to be

able to "patch" an image that Qemu is able to handle by overwriting

Quoted 7 lines

certain sectors. And I especially do not want to "mount" that image

via qemu-nbd because I might not trust it. I totally want to avoid that the host

system tries to analyse that image in terms of scanning the bootsector, partprobe,

lvm etc. pp.

qemu will have FUSE exporting as of 6.0 (didn’t quite make it into 5.2), so you can do something like this:

Quoted 8 lines
$ qemu-storage-daemon \
    --blockdev node-name=export,driver=qcow2,\
file.driver=file,file.filename=image.qcow2 \
    --export fuse,id=fuse,node-name=export,mountpoint=image.qcow2

This exports the image on image.qcow2 (i.e., on itself) and so by accessing the image file you then get raw access to its contents (so you can use system tools like dd).

Doesn’t require root rights, and shouldn’t make the kernel scan anything, because it’s exported as just a regular file.

Okay, but that is still more housekeeping than just invoking a single command.

Would it be an option to extend qemu-io to write data at a certain offset which it reads from STDIN?

Peter

#8 lore raw
Quoted 32 lines · Peter Lieven

Am 05.02.21 um 09:18 schrieb Max Reitz:

Quoted 31 lines · Peter Lieven
Quoted 15 lines

Am 02.02.21 um 16:51 schrieb Eric Blake:

Quoted 8 lines · Peter Lieven
Quoted 1 line

Your commit message says 'what', but not 'why'.  Generally, the one-line 'what' works well as the subject line, but you want the commit body to give an argument why your patch should be applied, rather than blank.

Here's the last time we tried to improve qemu-img dd:
https://lists.gnu.org/archive/html/qemu-devel/2018-08/msg02618.html

I was not aware of that story. My use case is that I want to be

able to "patch" an image that Qemu is able to handle by overwriting

Quoted 7 lines

certain sectors. And I especially do not want to "mount" that image

via qemu-nbd because I might not trust it. I totally want to avoid that the host

system tries to analyse that image in terms of scanning the bootsector, partprobe,

lvm etc. pp.

qemu will have FUSE exporting as of 6.0 (didn’t quite make it into 5.2), so you can do something like this:

$ qemu-storage-daemon \
     --blockdev node-name=export,driver=qcow2,\
file.driver=file,file.filename=image.qcow2 \
     --export fuse,id=fuse,node-name=export,mountpoint=image.qcow2
Quoted 6 lines
Quoted 3 lines

This exports the image on image.qcow2 (i.e., on itself) and so by accessing the image file you then get raw access to its contents (so you can use system tools like dd).

Doesn’t require root rights, and shouldn’t make the kernel scan anything, because it’s exported as just a regular file.

Okay, but that is still more housekeeping than just invoking a single command.

Yes, but I personally see this as much better than copying all of dd’s functionality into qemu-img.

My personal complaint is only that it’s a pain in the ass to invoke QSD this way. It would be nice to have a script that does the same via

$ qemu-blk-fuse-export image.qcow2

Would probably be trivial to write, but well, first we gotta do it, and have justification to keep it as part of qemu...

And if that’s still too much housekeeping, we could even write a qemu-dd script that scans all file arguments for non-raw images, launches a QSD instance to present them as raw, and then invokes dd.

All much simpler and much more feature-complete than to add more dd functionality to qemu-img.

Quoted 1 line

Would it be an option to extend qemu-io to write data at a certain offset which it reads from STDIN?

You mean read the data from stdin? Isn’t that what “write -s <file>” does?

Max

#9 lore raw
Quoted 42 lines · Max Reitz
Quoted 32 lines · Peter Lieven

Am 05.02.21 um 09:18 schrieb Max Reitz:

Quoted 31 lines · Peter Lieven
Quoted 15 lines

Am 02.02.21 um 16:51 schrieb Eric Blake:

Quoted 8 lines · Peter Lieven
Quoted 1 line

Your commit message says 'what', but not 'why'.  Generally, the one-line 'what' works well as the subject line, but you want the commit body to give an argument why your patch should be applied, rather than blank.

Here's the last time we tried to improve qemu-img dd:
https://lists.gnu.org/archive/html/qemu-devel/2018-08/msg02618.html

I was not aware of that story. My use case is that I want to be

able to "patch" an image that Qemu is able to handle by overwriting

Quoted 7 lines

certain sectors. And I especially do not want to "mount" that image

via qemu-nbd because I might not trust it. I totally want to avoid that the host

system tries to analyse that image in terms of scanning the bootsector, partprobe,

lvm etc. pp.

qemu will have FUSE exporting as of 6.0 (didn’t quite make it into 5.2), so you can do something like this:

$ qemu-storage-daemon \
     --blockdev node-name=export,driver=qcow2,\
file.driver=file,file.filename=image.qcow2 \
     --export fuse,id=fuse,node-name=export,mountpoint=image.qcow2
Quoted 6 lines
Quoted 3 lines

This exports the image on image.qcow2 (i.e., on itself) and so by accessing the image file you then get raw access to its contents (so you can use system tools like dd).

Doesn’t require root rights, and shouldn’t make the kernel scan anything, because it’s exported as just a regular file.

Okay, but that is still more housekeeping than just invoking a single command.

Yes, but I personally see this as much better than copying all of dd’s functionality into qemu-img.

Quoted 7 lines

My personal complaint is only that it’s a pain in the ass to invoke QSD this way.  It would be nice to have a script that does the same via

$ qemu-blk-fuse-export image.qcow2

Would probably be trivial to write, but well, first we gotta do it, and have justification to keep it as part of qemu...

And if that’s still too much housekeeping, we could even write a qemu-dd script that scans all file arguments for non-raw images, launches a QSD instance to present them as raw, and then invokes dd.

Since today’s Day of Learning at Red Hat, I decided to have some fun writing a qemu-dd.py script.

Max

#!/usr/bin/python

import json
import os
import subprocess
import sys
from typing import Optional

images = {}

for arg in sys.argv[1:]:
if arg.startswith('if=') or arg.startswith('of='):
filename = arg[3:]

# Ignore non-existing files, the user probably wants to make # dd create them (as raw images)
if not os.path.exists(filename):
continue

qemu_img = subprocess.Popen(('qemu-img', 'info', filename),
stdout=subprocess.PIPE, universal_newlines=True)
output: str = qemu_img.communicate()[0]
if qemu_img.returncode != 0:
sys.exit(os.EX_NOINPUT)

fmt_line = next(line
for line in output.split('\n') if line.startswith('file format: '))
fmt = fmt_line.split(': ', 1)[1]

if fmt != 'raw':
images[filename] = fmt

qsd: Optional[subprocess.Popen] = None

if images:
args = ['qemu-storage-daemon',
'--chardev', 'stdio,id=monitor',
'--monitor', 'monitor']

for i, (image, fmt) in enumerate(images.items()):
args += ['--blockdev',
f'{fmt},node-name=export{i},' + f'file.driver=file,file.filename={image}', '--export', f'fuse,id=fuse{i},node-name=export{i},' + f'mountpoint={image},writable=on']

qsd = subprocess.Popen(args,
stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True)

assert qsd.stdin is not None
assert qsd.stdout is not None

# Do some QMP communication so we know for sure that the exports # are up

qmp_line = json.loads(qsd.stdout.readline())
if 'QMP' in qmp_line:
qsd.terminate()
qsd.wait()
sys.stderr.write('The QEMU storage daemon did not provide ' +
'a QMP monitor\n')
sys.exit(os.EX_SOFTWARE)

qsd.stdin.write('{ "execute": "qmp_capabilities" }') qsd.stdin.flush()

# qmp_capabilities response
if json.loads(qsd.stdout.readline()) != {'return': {}}:
qsd.terminate()
qsd.wait()
sys.stderr.write('Communication error with the QEMU storage daemon\n') sys.exit(os.EX_SOFTWARE)

while True:
qsd.stdin.write('{ "execute": "query-block-exports" }') qsd.stdin.flush()
exports = json.loads(qsd.stdout.readline())['return'] if exports is None or len(exports) > len(images):
qsd.terminate()
qsd.wait() sys.stderr.write('Communication error with the ' +
'QEMU storage daemon\n')
sys.exit(os.EX_SOFTWARE)

if len(exports) == len(images):
break

subprocess.Popen(['dd'] + sys.argv[1:]).wait()

if qsd is not None:

    qsd.terminate()
    qsd.wait()
#5 lore raw
In reply to Eric Blake
Quoted 20 lines · Eric Blake
Quoted 19 lines · Peter Lieven

Am 02.02.21 um 16:51 schrieb Eric Blake:

Quoted 8 lines · Peter Lieven
Quoted 1 line

Your commit message says 'what', but not 'why'. Generally, the one-line 'what' works well as the subject line, but you want the commit body to give an argument why your patch should be applied, rather than blank.

Here's the last time we tried to improve qemu-img dd:
https://lists.gnu.org/archive/html/qemu-devel/2018-08/msg02618.html

I was not aware of that story. My use case is that I want to be

able to "patch" an image that Qemu is able to handle by overwriting

certain sectors. And I especially do not want to "mount" that image

via qemu-nbd because I might not trust it. I totally want to avoid that the host

Quoted 8 lines
Quoted 3 lines

system tries to analyse that image in terms of scanning the bootsector, partprobe,

lvm etc. pp.

qemu-nbd does not have to mount an image (yes, one use of qemu-nbd is to use -c /dev/nbdX to get the kernel to mount it; but other uses are to expose the NBD image in user-space only with no kernel involvement, and therefore no system mount efforts).

I agree, there's nothing unsafe about qemu-nbd (provided you don't use the -c option).

Quoted 8 lines

Another thing you might try is libnbd, which now includes a utility nbdcopy. It should make it easier to overwrite a portion of an NBD image using only user-space actions. I'm not sure if Rich has got it doing partial file overwrites yet (.../me goes and compiles the latest git checkout... nope, still a TODO item to implement subsetting), but it may be possible to combine nbdkit's --filter=offset with the full NBD image in order to then easily point nbdcopy to only the subset you care about. Definitely some ideas worthy of implementation.

TBH I would use nbdsh. For example to overwrite the sector at 1M in a qcow2 image with "1"s:

  $ qemu-img create -f qcow2 test.qcow2 10M
  $ nbdsh -c 'h.connect_systemd_socket_activation(["qemu-nbd","-t","-f","qcow2","test.qcow2"])' \
          -c 'h.pwrite(b"1"*512, 1024*1024)'

and to show it was really overwritten:

  $ nbdcopy -- [ qemu-nbd -f qcow2 test.qcow2 ] - | hexdump -C
  00000000  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
  *
  00100000  31 31 31 31 31 31 31 31  31 31 31 31 31 31 31 31  |1111111111111111|
*
  00100200  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
*
  00a00000

Rich.

Signature · 5 lines
Richard Jones, Virtualization Group, Red Hat http://people.redhat.com/~rjones
Read my programming and virtualization blog: http://rwmj.wordpress.com
virt-df lists disk usage of guests without needing to install any
software inside the virtual machine.  Supports Linux and Windows.
http://people.redhat.com/~rjones/virt-df/