Skip to contentSkip to conversation

Apply COR-filter to the block-stream permanently

PATCH Applied Vladimir Sementsov-Ogievskiy 2 participants 28 messages Dec 16, 2020 – Jan 8, 2021 Block layer

lore
Share & tools
lore
#4 lore raw

PATCH v15 4/13 qapi: add filter-node-name to block-stream

In reply to Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

Provide the possibility to pass the 'filter-node-name' parameter to the block-stream job as it is done for the commit block job.

[vsementsov: comment indentation, s/Since: 5.2/Since: 6.0/]
---
 qapi/block-core.json           | 6 ++++++
 include/block/block_int.h      | 7 ++++++-
 block/monitor/block-hmp-cmds.c | 4 ++--
 block/stream.c                 | 4 +++-
 blockdev.c                     | 4 +++-
 5 files changed, 20 insertions(+), 5 deletions(-)
qapi/block-core.json +6 -0
index 04c5196e59..6050cf3c39 100644
@@ -2543,6 +2543,11 @@
 #            'stop' and 'enospc' can only be used if the block device
 #            supports io-status (see BlockInfo).  Since 1.3.
 #
+# @filter-node-name: the node name that should be assigned to the
+#                    filter driver that the stream job inserts into the graph
+#                    above @device. If this option is not given, a node name is
+#                    autogenerated. (Since: 6.0)
+#
 # @auto-finalize: When false, this job will wait in a PENDING state after it has
 #                 finished its work, waiting for @block-job-finalize before
 #                 making any block graph changes.
@@ -2573,6 +2578,7 @@
   'data': { '*job-id': 'str', 'device': 'str', '*base': 'str',
             '*base-node': 'str', '*backing-file': 'str', '*speed': 'int',
             '*on-error': 'BlockdevOnError',
+            '*filter-node-name': 'str',
             '*auto-finalize': 'bool', '*auto-dismiss': 'bool' } }
 
 ##
include/block/block_int.h +6 -1
index 1eeafc118c..6f778e2517 100644
@@ -1142,6 +1142,9 @@ int is_windows_drive(const char *filename);
  *                  See @BlockJobCreateFlags
  * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
  * @on_error: The action to take upon error.
+ * @filter_node_name: The node name that should be assigned to the filter
+ *                    driver that the commit job inserts into the graph above
+ *                    @bs. NULL means that a node name should be autogenerated.
  * @errp: Error object.
  *
  * Start a streaming operation on @bs.  Clusters that are unallocated
@@ -1154,7 +1157,9 @@ int is_windows_drive(const char *filename);
 void stream_start(const char *job_id, BlockDriverState *bs,
                   BlockDriverState *base, const char *backing_file_str,
                   int creation_flags, int64_t speed,
-                  BlockdevOnError on_error, Error **errp);
+                  BlockdevOnError on_error,
+                  const char *filter_node_name,
+                  Error **errp);
 
 /**
  * commit_start:
block/monitor/block-hmp-cmds.c +2 -2
index d15a2be827..e8a58f326e 100644
@@ -508,8 +508,8 @@ void hmp_block_stream(Monitor *mon, const QDict *qdict)
 
     qmp_block_stream(true, device, device, base != NULL, base, false, NULL,
                      false, NULL, qdict_haskey(qdict, "speed"), speed, true,
-                     BLOCKDEV_ON_ERROR_REPORT, false, false, false, false,
-                     &error);
+                     BLOCKDEV_ON_ERROR_REPORT, false, NULL, false, false, false,
+                     false, &error);
 
     hmp_handle_error(mon, error);
 }
block/stream.c +3 -1
index 236384f2f7..6e281c71ac 100644
@@ -221,7 +221,9 @@ static const BlockJobDriver stream_job_driver = {
 void stream_start(const char *job_id, BlockDriverState *bs,
                   BlockDriverState *base, const char *backing_file_str,
                   int creation_flags, int64_t speed,
-                  BlockdevOnError on_error, Error **errp)
+                  BlockdevOnError on_error,
+                  const char *filter_node_name,
+                  Error **errp)
 {
     StreamBlockJob *s;
     BlockDriverState *iter;
blockdev.c +3 -1
index 412354b4b6..c290cb1dca 100644
@@ -2501,6 +2501,7 @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device,
                       bool has_backing_file, const char *backing_file,
                       bool has_speed, int64_t speed,
                       bool has_on_error, BlockdevOnError on_error,
+                      bool has_filter_node_name, const char *filter_node_name,
                       bool has_auto_finalize, bool auto_finalize,
                       bool has_auto_dismiss, bool auto_dismiss,
                       Error **errp)
@@ -2583,7 +2584,8 @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device,
     }
 
     stream_start(has_job_id ? job_id : NULL, bs, base_bs, base_name,
-                 job_flags, has_speed ? speed : 0, on_error, &local_err);
+                 job_flags, has_speed ? speed : 0, on_error,
+                 filter_node_name, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         goto out;
Signature · 1 line
2.25.4
#0 lore raw

v15 cover Apply COR-filter to the block-stream permanently

Hi all!

Here is a new version of cor-filter in block-stream series. Main change is freezing the chain in cor-filter itself.

v15:
02: s/ = / = /
add Max's r-b
03: add Max's r-b
04: since: 6.0
indent comment
add Max's r-b
05: changed commit msg
wording
document the default
since: 6.0
use bdrv_find_node(), fix errp overwriting
freeze the chain
check bottom is not filter
ref bottom_bs
06: limit to qcow2 to not care
use qemu-img rebase -u -b ''
07: use assert instead of abort
add Max's r-b
08: add Max's r-b
09: changed commit msg (was "stream: skip filters when writing backing file name to QCOW2 header")
keep mostly same logic for the case when backing-file is specified, don't do bdrv_find_backing_image()
10: don't restrict backing-file for now
11: add Max's r-b
12: add Max's r-b
13: chain is now frozen in filter, so the logic changed around add/remove fitlter

Andrey Shinkevich (10):
copy-on-read: support preadv/pwritev_part functions
block: add API function to insert a node
copy-on-read: add filter drop function
qapi: add filter-node-name to block-stream
qapi: copy-on-read filter: add 'bottom' option
iotests: add #310 to test bottom node in COR driver
block: include supported_read_flags into BDS structure
copy-on-read: skip non-guest reads if no copy needed
stream: rework backing-file changing
block: apply COR-filter to block-stream jobs

Vladimir Sementsov-Ogievskiy (3):
qapi: block-stream: add "bottom" argument
iotests: 30: prepare to COR filter insertion by stream job block/stream: add s->target_bs

 qapi/block-core.json           |  38 ++++++-
 block/copy-on-read.h           |  32 ++++++
 include/block/block.h          |  10 +-
 include/block/block_int.h      |  12 ++-
 block.c                        |  25 +++++
 block/copy-on-read.c           | 184 +++++++++++++++++++++++++++++---
 block/io.c                     |  10 +-
 block/monitor/block-hmp-cmds.c |   7 +-
 block/stream.c                 | 185 ++++++++++++++++++++-------------
 blockdev.c                     |  69 +++++++++---
 tests/qemu-iotests/030         |  12 ++-
 tests/qemu-iotests/141.out     |   2 +-
 tests/qemu-iotests/245         |  20 ++--
 tests/qemu-iotests/310         | 116 +++++++++++++++++++++
 tests/qemu-iotests/310.out     |  15 +++
 tests/qemu-iotests/group       |   1 +
 16 files changed, 608 insertions(+), 130 deletions(-)
 create mode 100644 block/copy-on-read.h
 create mode 100755 tests/qemu-iotests/310
 create mode 100644 tests/qemu-iotests/310.out
Signature · 1 line
2.25.4
#2 lore raw

PATCH v15 2/13 block: add API function to insert a node

From: Andrey Shinkevich <[email protected]>

Provide API for insertion a node to backing chain.

---
 include/block/block.h |  2 ++
 block.c               | 25 +++++++++++++++++++++++++
 2 files changed, 27 insertions(+)
include/block/block.h +2 -0
index 5b81e33e94..8ea794959b 100644
@@ -360,6 +360,8 @@ void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top,
                  Error **errp);
 void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to,
                        Error **errp);
+BlockDriverState *bdrv_insert_node(BlockDriverState *bs, QDict *node_options,
+                                   int flags, Error **errp);
 
 int bdrv_parse_aio(const char *mode, int *flags);
 int bdrv_parse_cache_mode(const char *mode, int *flags, bool *writethrough);
block.c +25 -0
index 8f177504d4..8ed9c5eaf9 100644
@@ -4704,6 +4704,31 @@ static void bdrv_delete(BlockDriverState *bs)
     g_free(bs);
 }
 
+BlockDriverState *bdrv_insert_node(BlockDriverState *bs, QDict *node_options,
+                                   int flags, Error **errp)
+{
+    BlockDriverState *new_node_bs;
+    Error *local_err = NULL;
+
+    new_node_bs = bdrv_open(NULL, NULL, node_options, flags, errp);
+    if (new_node_bs == NULL) {
+        error_prepend(errp, "Could not create node: ");
+        return NULL;
+    }
+
+    bdrv_drained_begin(bs);
+    bdrv_replace_node(bs, new_node_bs, &local_err);
+    bdrv_drained_end(bs);
+
+    if (local_err) {
+        bdrv_unref(new_node_bs);
+        error_propagate(errp, local_err);
+        return NULL;
+    }
+
+    return new_node_bs;
+}
+
 /*
  * Run consistency checks on an image
  *
Signature · 1 line
2.25.4
#14 lore raw

PATCH v15 10/13 qapi: block-stream: add "bottom" argument

In reply to Vladimir Sementsov-Ogievskiy

The code already don't freeze base node and we try to make it prepared for the situation when base node is changed during the operation. In other words, block-stream doesn't own base node.

Let's introduce a new interface which should replace the current one, which will in better relations with the code. Specifying bottom node instead of base, and requiring it to be non-filter gives us the following benefits:

- drop difference between above_base and base_overlay, which will be renamed to just bottom, when old interface dropped

- clean way to work with parallel streams/commits on the same backing chain, which otherwise become a problem when we introduce a filter for stream job

- cleaner interface. Nobody will surprised the fact that base node may disappear during block-stream, when there is no word about "base" in the interface.

---
 qapi/block-core.json           | 12 ++++---
 include/block/block_int.h      |  1 +
 block/monitor/block-hmp-cmds.c |  3 +-
 block/stream.c                 | 50 +++++++++++++++++++---------
 blockdev.c                     | 59 ++++++++++++++++++++++++++++------
 5 files changed, 94 insertions(+), 31 deletions(-)
qapi/block-core.json +8 -4
index b8094a5ec7..cb0066fd5c 100644
@@ -2517,10 +2517,14 @@
 # @device: the device or node name of the top image
 #
 # @base: the common backing file name.
-#        It cannot be set if @base-node is also set.
+#        It cannot be set if @base-node or @bottom is also set.
 #
 # @base-node: the node name of the backing file.
-#             It cannot be set if @base is also set. (Since 2.8)
+#             It cannot be set if @base or @bottom is also set. (Since 2.8)
+#
+# @bottom: the last node in the chain that should be streamed into
+#          top. It cannot be set if @base or @base-node is also set.
+#          It cannot be filter node. (Since 6.0)
 #
 # @backing-file: The backing file string to write into the top
 #                image. This filename is not validated.
@@ -2576,8 +2580,8 @@
 ##
 { 'command': 'block-stream',
   'data': { '*job-id': 'str', 'device': 'str', '*base': 'str',
-            '*base-node': 'str', '*backing-file': 'str', '*speed': 'int',
-            '*on-error': 'BlockdevOnError',
+            '*base-node': 'str', '*backing-file': 'str', '*bottom': 'str',
+            '*speed': 'int', '*on-error': 'BlockdevOnError',
             '*filter-node-name': 'str',
             '*auto-finalize': 'bool', '*auto-dismiss': 'bool' } }
 
include/block/block_int.h +1 -0
index 1f56443440..4b8aa61fb4 100644
@@ -1160,6 +1160,7 @@ int is_windows_drive(const char *filename);
  */
 void stream_start(const char *job_id, BlockDriverState *bs,
                   BlockDriverState *base, const char *backing_file_str,
+                  BlockDriverState *bottom,
                   int creation_flags, int64_t speed,
                   BlockdevOnError on_error,
                   const char *filter_node_name,
block/monitor/block-hmp-cmds.c +2 -1
index e8a58f326e..afd75ab628 100644
@@ -507,7 +507,8 @@ void hmp_block_stream(Monitor *mon, const QDict *qdict)
     int64_t speed = qdict_get_try_int(qdict, "speed", 0);
 
     qmp_block_stream(true, device, device, base != NULL, base, false, NULL,
-                     false, NULL, qdict_haskey(qdict, "speed"), speed, true,
+                     false, NULL, false, NULL,
+                     qdict_haskey(qdict, "speed"), speed, true,
                      BLOCKDEV_ON_ERROR_REPORT, false, NULL, false, false, false,
                      false, &error);
 
block/stream.c +34 -16

65 lines not on this page · raw message

blockdev.c +49 -10

94 lines not on this page · raw message

Signature · 1 line
2.25.4
#6 lore raw

PATCH v15 5/13 qapi: copy-on-read filter: add 'bottom' option

In reply to Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

Add an option to limit copy-on-read operations to specified sub-chain of backing-chain, to make copy-on-read filter useful for block-stream job.

[vsementsov: change subject, modified to freeze the chain,
do some fixes]

---
 qapi/block-core.json | 20 ++++++++-
 block/copy-on-read.c | 98 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 115 insertions(+), 3 deletions(-)
qapi/block-core.json +19 -1
index 6050cf3c39..b8094a5ec7 100644
@@ -3942,6 +3942,24 @@
   'data': { 'throttle-group': 'str',
             'file' : 'BlockdevRef'
              } }
+
+##
+# @BlockdevOptionsCor:
+#
+# Driver specific block device options for the copy-on-read driver.
+#
+# @bottom: The name of a non-filter node (allocation-bearing layer) that
+#          limits the COR operations in the backing chain (inclusive), so
+#          that no data below this node will be copied by this filter.
+#          If option is absent, the limit is not applied, so that data
+#          from all backing layers may be copied.
+#
+# Since: 6.0
+##
+{ 'struct': 'BlockdevOptionsCor',
+  'base': 'BlockdevOptionsGenericFormat',
+  'data': { '*bottom': 'str' } }
+
 ##
 # @BlockdevOptions:
 #
@@ -3994,7 +4012,7 @@
       'bochs':      'BlockdevOptionsGenericFormat',
       'cloop':      'BlockdevOptionsGenericFormat',
       'compress':   'BlockdevOptionsGenericFormat',
-      'copy-on-read':'BlockdevOptionsGenericFormat',
+      'copy-on-read':'BlockdevOptionsCor',
       'dmg':        'BlockdevOptionsGenericFormat',
       'file':       'BlockdevOptionsFile',
       'ftp':        'BlockdevOptionsCurlFtp',
block/copy-on-read.c +96 -2

146 lines not on this page · raw message

Signature · 1 line
2.25.4
#10 lore raw

PATCH v15 9/13 stream: rework backing-file changing

In reply to Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

Stream in stream_prepare calls bdrv_change_backing_file() to change backing-file in the metadata of bs.

It may use either backing-file parameter given by user or just take filename of base on job start.

Backing file format is determined by base on job finish.

There are some problems with this design, we solve only two by this patch:

1. Consider scenario with backing-file unset. Current concept of stream supports changing of the base during the job (we don't freeze link to the base). So, we should not save base filename at job start,

- let's determine name of the base on job finish.

2. Using direct base to determine filename and format is not very good:
base node may be a filter, so its filename may be JSON, and format_name is not good for storing into qcow2 metadata as backing file format.

- let's use unfiltered_base

[vsementsov: change commit subject, change logic in stream_prepare]
---
 block/stream.c | 9 +++++----
 blockdev.c     | 8 +-------
 2 files changed, 6 insertions(+), 11 deletions(-)
block/stream.c +5 -4
index 6e281c71ac..6a525a5edf 100644
@@ -65,6 +65,7 @@ static int stream_prepare(Job *job)
     BlockDriverState *bs = blk_bs(bjob->blk);
     BlockDriverState *unfiltered_bs = bdrv_skip_filters(bs);
     BlockDriverState *base = bdrv_filter_or_cow_bs(s->above_base);
+    BlockDriverState *unfiltered_base = bdrv_skip_filters(base);
     Error *local_err = NULL;
     int ret = 0;
 
@@ -73,10 +74,10 @@ static int stream_prepare(Job *job)
 
     if (bdrv_cow_child(unfiltered_bs)) {
         const char *base_id = NULL, *base_fmt = NULL;
-        if (base) {
-            base_id = s->backing_file_str;
-            if (base->drv) {
-                base_fmt = base->drv->format_name;
+        if (unfiltered_base) {
+            base_id = s->backing_file_str ?: unfiltered_base->filename;
+            if (unfiltered_base->drv) {
+                base_fmt = unfiltered_base->drv->format_name;
             }
         }
         bdrv_set_backing_hd(unfiltered_bs, base, &local_err);
blockdev.c +1 -7
index c290cb1dca..b58f36fc31 100644
@@ -2510,7 +2510,6 @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device,
     BlockDriverState *base_bs = NULL;
     AioContext *aio_context;
     Error *local_err = NULL;
-    const char *base_name = NULL;
     int job_flags = JOB_DEFAULT;
 
     if (!has_on_error) {
@@ -2538,7 +2537,6 @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device,
             goto out;
         }
         assert(bdrv_get_aio_context(base_bs) == aio_context);
-        base_name = base;
     }
 
     if (has_base_node) {
@@ -2553,7 +2551,6 @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device,
         }
         assert(bdrv_get_aio_context(base_bs) == aio_context);
         bdrv_refresh_filename(base_bs);
-        base_name = base_bs->filename;
     }
 
     /* Check for op blockers in the whole chain between bs and base */
@@ -2573,9 +2570,6 @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device,
         goto out;
     }
 
-    /* backing_file string overrides base bs filename */
-    base_name = has_backing_file ? backing_file : base_name;
-
     if (has_auto_finalize && !auto_finalize) {
         job_flags |= JOB_MANUAL_FINALIZE;
     }
@@ -2583,7 +2577,7 @@ void qmp_block_stream(bool has_job_id, const char *job_id, const char *device,
         job_flags |= JOB_MANUAL_DISMISS;
     }
 
-    stream_start(has_job_id ? job_id : NULL, bs, base_bs, base_name,
+    stream_start(has_job_id ? job_id : NULL, bs, base_bs, backing_file,
                  job_flags, has_speed ? speed : 0, on_error,
                  filter_node_name, &local_err);
     if (local_err) {
Signature · 1 line
2.25.4
#1 lore raw

PATCH v15 1/13 copy-on-read: support preadv/pwritev_part functions

In reply to Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

Add support for the recently introduced functions
bdrv_co_preadv_part()
and
bdrv_co_pwritev_part()
to the COR-filter driver.

---
 block/copy-on-read.c | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)
block/copy-on-read.c +16 -12
index 2816e61afe..cb03e0f2d3 100644
@@ -74,21 +74,25 @@ static int64_t cor_getlength(BlockDriverState *bs)
 }
 
 
-static int coroutine_fn cor_co_preadv(BlockDriverState *bs,
-                                      uint64_t offset, uint64_t bytes,
-                                      QEMUIOVector *qiov, int flags)
+static int coroutine_fn cor_co_preadv_part(BlockDriverState *bs,
+                                           uint64_t offset, uint64_t bytes,
+                                           QEMUIOVector *qiov,
+                                           size_t qiov_offset,
+                                           int flags)
 {
-    return bdrv_co_preadv(bs->file, offset, bytes, qiov,
-                          flags | BDRV_REQ_COPY_ON_READ);
+    return bdrv_co_preadv_part(bs->file, offset, bytes, qiov, qiov_offset,
+                               flags | BDRV_REQ_COPY_ON_READ);
 }
 
 
-static int coroutine_fn cor_co_pwritev(BlockDriverState *bs,
-                                       uint64_t offset, uint64_t bytes,
-                                       QEMUIOVector *qiov, int flags)
+static int coroutine_fn cor_co_pwritev_part(BlockDriverState *bs,
+                                            uint64_t offset,
+                                            uint64_t bytes,
+                                            QEMUIOVector *qiov,
+                                            size_t qiov_offset, int flags)
 {
-
-    return bdrv_co_pwritev(bs->file, offset, bytes, qiov, flags);
+    return bdrv_co_pwritev_part(bs->file, offset, bytes, qiov, qiov_offset,
+                                flags);
 }
 
 
@@ -137,8 +141,8 @@ static BlockDriver bdrv_copy_on_read = {
 
     .bdrv_getlength                     = cor_getlength,
 
-    .bdrv_co_preadv                     = cor_co_preadv,
-    .bdrv_co_pwritev                    = cor_co_pwritev,
+    .bdrv_co_preadv_part                = cor_co_preadv_part,
+    .bdrv_co_pwritev_part               = cor_co_pwritev_part,
     .bdrv_co_pwrite_zeroes              = cor_co_pwrite_zeroes,
     .bdrv_co_pdiscard                   = cor_co_pdiscard,
     .bdrv_co_pwritev_compressed         = cor_co_pwritev_compressed,
Signature · 1 line
2.25.4
#7 lore raw

PATCH v15 6/13 iotests: add #310 to test bottom node in COR driver

In reply to Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

The test case #310 is similar to #216 by Max Reitz. The difference is that the test #310 involves a bottom node to the COR filter driver.

[vsementsov: detach backing to test reads from top, limit to qcow2]
---
 tests/qemu-iotests/310     | 116 +++++++++++++++++++++++++++++++++++++
 tests/qemu-iotests/310.out |  15 +++++
 tests/qemu-iotests/group   |   1 +
 3 files changed, 132 insertions(+)
 create mode 100755 tests/qemu-iotests/310
 create mode 100644 tests/qemu-iotests/310.out
tests/qemu-iotests/310 new file +116 -0
new file mode 100755
index 0000000000..a35e8e14f5
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+#
+# Copy-on-read tests using a COR filter with a bottom node
+#
+# Copyright (C) 2018 Red Hat, Inc.
+# Copyright (c) 2020 Virtuozzo International GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import iotests
+from iotests import log, qemu_img, qemu_io_silent
+
+# Need backing file support
+iotests.script_initialize(supported_fmts=['qcow2'],
+                          supported_platforms=['linux'])
+
+log('')
+log('=== Copy-on-read across nodes ===')
+log('')
+
+# This test is similar to the 216 one by Max Reitz <[email protected]>
+# The difference is that this test case involves a bottom node to the
+# COR filter driver.
+
+with iotests.FilePath('base.img') as base_img_path, \
+     iotests.FilePath('mid.img') as mid_img_path, \
+     iotests.FilePath('top.img') as top_img_path, \
+     iotests.VM() as vm:
+
+    log('--- Setting up images ---')
+    log('')
+
+    assert qemu_img('create', '-f', iotests.imgfmt, base_img_path, '64M') == 0
+    assert qemu_io_silent(base_img_path, '-c', 'write -P 1 0M 1M') == 0
+    assert qemu_io_silent(base_img_path, '-c', 'write -P 1 3M 1M') == 0
+    assert qemu_img('create', '-f', iotests.imgfmt, '-b', base_img_path,
+                    '-F', iotests.imgfmt, mid_img_path) == 0
+    assert qemu_io_silent(mid_img_path,  '-c', 'write -P 3 2M 1M') == 0
+    assert qemu_io_silent(mid_img_path,  '-c', 'write -P 3 4M 1M') == 0
+    assert qemu_img('create', '-f', iotests.imgfmt, '-b', mid_img_path,
+                    '-F', iotests.imgfmt, top_img_path) == 0
+    assert qemu_io_silent(top_img_path,  '-c', 'write -P 2 1M 1M') == 0
+
+#      0 1 2 3 4
+# top    2
+# mid      3   3
+# base 1     1
+
+    log('Done')
+
+    log('')
+    log('--- Doing COR ---')
+    log('')
+
+    vm.launch()
+
+    log(vm.qmp('blockdev-add',
+               node_name='node0',
+               driver='copy-on-read',
+               bottom='node2',
+               file={
+                   'driver': iotests.imgfmt,
+                   'file': {
+                       'driver': 'file',
+                       'filename': top_img_path
+                   },
+                   'backing': {
+                       'node-name': 'node2',
+                       'driver': iotests.imgfmt,
+                       'file': {
+                           'driver': 'file',
+                           'filename': mid_img_path
+                       },
+                       'backing': {
+                           'driver': iotests.imgfmt,
+                           'file': {
+                               'driver': 'file',
+                               'filename': base_img_path
+                           }
+                       },
+                   }
+               }))
+
+    # Trigger COR
+    log(vm.qmp('human-monitor-command',
+               command_line='qemu-io node0 "read 0 5M"'))
+
+    vm.shutdown()
+
+    log('')
+    log('--- Checking COR result ---')
+    log('')
+
+    # Detach backing to check that we can read the data from the top level now
+    assert qemu_img('rebase', '-u', '-b', '', '-f', iotests.imgfmt,
+                    top_img_path,) == 0
+
+    assert qemu_io_silent(top_img_path,  '-c', 'read -P 0 0 1M') == 0
+    assert qemu_io_silent(top_img_path,  '-c', 'read -P 2 1M 1M') == 0
+    assert qemu_io_silent(top_img_path,  '-c', 'read -P 3 2M 1M') == 0
+    assert qemu_io_silent(top_img_path,  '-c', 'read -P 0 3M 1M') == 0
+    assert qemu_io_silent(top_img_path,  '-c', 'read -P 3 4M 1M') == 0
+
+    log('Done')
tests/qemu-iotests/310.out new file +15 -0
new file mode 100644
index 0000000000..a70aa5cdae
@@ -0,0 +1,15 @@
+
+=== Copy-on-read across nodes ===
+
+--- Setting up images ---
+
+Done
+
+--- Doing COR ---
+
+{"return": {}}
+{"return": ""}
+
+--- Checking COR result ---
+
+Done
tests/qemu-iotests/group +1 -0
index 9a8394b4cd..9fa72cf442 100644
@@ -317,3 +317,4 @@
 307 rw quick export
 308 rw
 309 rw auto quick
+310 rw quick
Signature · 1 line
2.25.4
#19 lore raw

PATCH v15 11/13 iotests: 30: prepare to COR filter insertion by stream job

In reply to Vladimir Sementsov-Ogievskiy

test_stream_parallel run parallel stream jobs, intersecting so that top of one is base of another. It's OK now, but it would be a problem if insert the filter, as one job will want to use another job's filter as above_base node.

Correct thing to do is move to new interface: "bottom" argument instead of base. This guarantees that jobs don't intersect by their actions.

---
 tests/qemu-iotests/030 | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
tests/qemu-iotests/030 +3 -1
index dcb4b5d6a6..bd8cf9cff7 100755
@@ -245,7 +245,9 @@ class TestParallelOps(iotests.QMPTestCase):
             node_name = 'node%d' % i
             job_id = 'stream-%s' % node_name
             pending_jobs.append(job_id)
-            result = self.vm.qmp('block-stream', device=node_name, job_id=job_id, base=self.imgs[i-2], speed=1024)
+            result = self.vm.qmp('block-stream', device=node_name,
+                                 job_id=job_id, bottom=f'node{i-1}',
+                                 speed=1024)
             self.assert_qmp(result, 'return', {})
 
         for job in pending_jobs:
Signature · 1 line
2.25.4
#3 lore raw

PATCH v15 3/13 copy-on-read: add filter drop function

In reply to Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

Provide API for the COR-filter removal. Also, drop the filter child permissions for an inactive state when the filter node is being removed.
To insert the filter, the block generic layer function bdrv_insert_node() can be used.
The new function bdrv_cor_filter_drop() may be considered as an intermediate solution before the QEMU permission update system has overhauled. Then we are able to implement the API function bdrv_remove_node() on the block generic layer.

---
 block/copy-on-read.h | 32 +++++++++++++++++++++++++
 block/copy-on-read.c | 56 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+)
 create mode 100644 block/copy-on-read.h
block/copy-on-read.h new file +32 -0
new file mode 100644
index 0000000000..7bf405dccd
@@ -0,0 +1,32 @@
+/*
+ * Copy-on-read filter block driver
+ *
+ * The filter driver performs Copy-On-Read (COR) operations
+ *
+ * Copyright (c) 2018-2020 Virtuozzo International GmbH.
+ *
+ * Author:
+ *   Andrey Shinkevich <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef BLOCK_COPY_ON_READ
+#define BLOCK_COPY_ON_READ
+
+#include "block/block_int.h"
+
+void bdrv_cor_filter_drop(BlockDriverState *cor_filter_bs);
+
+#endif /* BLOCK_COPY_ON_READ */
block/copy-on-read.c +56 -0

91 lines not on this page · raw message

Signature · 1 line
2.25.4
#9 lore raw

PATCH v15 8/13 copy-on-read: skip non-guest reads if no copy needed

In reply to Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

If the flag BDRV_REQ_PREFETCH was set, skip idling read/write operations in COR-driver. It can be taken into account for the COR-algorithms optimization. That check is being made during the block stream job by the moment.

Add the BDRV_REQ_PREFETCH flag to the supported_read_flags of the COR-filter.

block: Modify the comment for the flag BDRV_REQ_PREFETCH as we are going to use it alone and pass it to the COR-filter driver for further processing.

---
 include/block/block.h |  8 +++++---
 block/copy-on-read.c  | 14 ++++++++++----
 2 files changed, 15 insertions(+), 7 deletions(-)
include/block/block.h +5 -3
index 8ea794959b..f652f31406 100644
@@ -81,9 +81,11 @@ typedef enum {
     BDRV_REQ_NO_FALLBACK        = 0x100,
 
     /*
-     * BDRV_REQ_PREFETCH may be used only together with BDRV_REQ_COPY_ON_READ
-     * on read request and means that caller doesn't really need data to be
-     * written to qiov parameter which may be NULL.
+     * BDRV_REQ_PREFETCH makes sense only in the context of copy-on-read
+     * (i.e., together with the BDRV_REQ_COPY_ON_READ flag or when a COR
+     * filter is involved), in which case it signals that the COR operation
+     * need not read the data into memory (qiov) but only ensure they are
+     * copied to the top layer (i.e., that COR operation is done).
      */
     BDRV_REQ_PREFETCH  = 0x200,
     /* Mask of valid flags */
block/copy-on-read.c +10 -4
index 71560984f6..9cad9e1b8c 100644
@@ -50,6 +50,8 @@ static int cor_open(BlockDriverState *bs, QDict *options, int flags,
         return -EINVAL;
     }
 
+    bs->supported_read_flags = BDRV_REQ_PREFETCH;
+
     bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED |
         (BDRV_REQ_FUA & bs->file->bs->supported_write_flags);
 
@@ -172,10 +174,14 @@ static int coroutine_fn cor_co_preadv_part(BlockDriverState *bs,
             }
         }
 
-        ret = bdrv_co_preadv_part(bs->file, offset, n, qiov, qiov_offset,
-                                  local_flags);
-        if (ret < 0) {
-            return ret;
+        /* Skip if neither read nor write are needed */
+        if ((local_flags & (BDRV_REQ_PREFETCH | BDRV_REQ_COPY_ON_READ)) !=
+            BDRV_REQ_PREFETCH) {
+            ret = bdrv_co_preadv_part(bs->file, offset, n, qiov, qiov_offset,
+                                      local_flags);
+            if (ret < 0) {
+                return ret;
+            }
         }
 
         offset += n;
Signature · 1 line
2.25.4
#20 lore raw

PATCH v15 12/13 block/stream: add s->target_bs

In reply to Vladimir Sementsov-Ogievskiy

Add a direct link to target bs for convenience and to simplify following commit which will insert COR filter above target bs.

This is a part of original commit written by Andrey.

---
 block/stream.c | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)
block/stream.c +10 -13
index 045d6bc76b..626dfa2b22 100644
@@ -33,6 +33,7 @@ typedef struct StreamBlockJob {
     BlockJob common;
     BlockDriverState *base_overlay; /* COW overlay (stream from this) */
     BlockDriverState *above_base;   /* Node directly above the base */
+    BlockDriverState *target_bs;
     BlockdevOnError on_error;
     char *backing_file_str;
     bool bs_read_only;
@@ -53,23 +54,20 @@ static void stream_abort(Job *job)
     StreamBlockJob *s = container_of(job, StreamBlockJob, common.job);
 
     if (s->chain_frozen) {
-        BlockJob *bjob = &s->common;
-        bdrv_unfreeze_backing_chain(blk_bs(bjob->blk), s->above_base);
+        bdrv_unfreeze_backing_chain(s->target_bs, s->above_base);
     }
 }
 
 static int stream_prepare(Job *job)
 {
     StreamBlockJob *s = container_of(job, StreamBlockJob, common.job);
-    BlockJob *bjob = &s->common;
-    BlockDriverState *bs = blk_bs(bjob->blk);
-    BlockDriverState *unfiltered_bs = bdrv_skip_filters(bs);
+    BlockDriverState *unfiltered_bs = bdrv_skip_filters(s->target_bs);
     BlockDriverState *base = bdrv_filter_or_cow_bs(s->above_base);
     BlockDriverState *unfiltered_base = bdrv_skip_filters(base);
     Error *local_err = NULL;
     int ret = 0;
 
-    bdrv_unfreeze_backing_chain(bs, s->above_base);
+    bdrv_unfreeze_backing_chain(s->target_bs, s->above_base);
     s->chain_frozen = false;
 
     if (bdrv_cow_child(unfiltered_bs)) {
@@ -95,13 +93,12 @@ static void stream_clean(Job *job)
 {
     StreamBlockJob *s = container_of(job, StreamBlockJob, common.job);
     BlockJob *bjob = &s->common;
-    BlockDriverState *bs = blk_bs(bjob->blk);
 
     /* Reopen the image back in read-only mode if necessary */
     if (s->bs_read_only) {
         /* Give up write permissions before making it read-only */
         blk_set_perm(bjob->blk, 0, BLK_PERM_ALL, &error_abort);
-        bdrv_reopen_set_read_only(bs, true, NULL);
+        bdrv_reopen_set_read_only(s->target_bs, true, NULL);
     }
 
     g_free(s->backing_file_str);
@@ -111,8 +108,7 @@ static int coroutine_fn stream_run(Job *job, Error **errp)
 {
     StreamBlockJob *s = container_of(job, StreamBlockJob, common.job);
     BlockBackend *blk = s->common.blk;
-    BlockDriverState *bs = blk_bs(blk);
-    BlockDriverState *unfiltered_bs = bdrv_skip_filters(bs);
+    BlockDriverState *unfiltered_bs = bdrv_skip_filters(s->target_bs);
     bool enable_cor = !bdrv_cow_child(s->base_overlay);
     int64_t len;
     int64_t offset = 0;
@@ -125,7 +121,7 @@ static int coroutine_fn stream_run(Job *job, Error **errp)
         return 0;
     }
 
-    len = bdrv_getlength(bs);
+    len = bdrv_getlength(s->target_bs);
     if (len < 0) {
         return len;
     }
@@ -137,7 +133,7 @@ static int coroutine_fn stream_run(Job *job, Error **errp)
      * account.
      */
     if (enable_cor) {
-        bdrv_enable_copy_on_read(bs);
+        bdrv_enable_copy_on_read(s->target_bs);
     }
 
     for ( ; offset < len; offset += n) {
@@ -199,7 +195,7 @@ static int coroutine_fn stream_run(Job *job, Error **errp)
     }
 
     if (enable_cor) {
-        bdrv_disable_copy_on_read(bs);
+        bdrv_disable_copy_on_read(s->target_bs);
     }
 
     /* Do not remove the backing file if an error was there but ignored. */
@@ -314,6 +310,7 @@ void stream_start(const char *job_id, BlockDriverState *bs,
     s->base_overlay = base_overlay;
     s->above_base = above_base;
     s->backing_file_str = g_strdup(backing_file_str);
+    s->target_bs = bs;
     s->bs_read_only = bs_read_only;
     s->chain_frozen = true;
 
Signature · 1 line
2.25.4
#8 lore raw

PATCH v15 7/13 block: include supported_read_flags into BDS structure

In reply to Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

Add the new member supported_read_flags to the BlockDriverState structure. It will control the flags set for copy-on-read operations. Make the block generic layer evaluate supported read flags before they go to a block driver.

---
 include/block/block_int.h |  4 ++++
 block/io.c                | 10 ++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)
include/block/block_int.h +4 -0
index 6f778e2517..1f56443440 100644
@@ -881,6 +881,10 @@ struct BlockDriverState {
     /* I/O Limits */
     BlockLimits bl;
 
+    /*
+     * Flags honored during pread
+     */
+    unsigned int supported_read_flags;
     /* Flags honored during pwrite (so far: BDRV_REQ_FUA,
      * BDRV_REQ_WRITE_UNCHANGED).
      * If a driver does not support BDRV_REQ_WRITE_UNCHANGED, those
block/io.c +8 -2
index 24205f5168..851fe53604 100644
@@ -1431,6 +1431,9 @@ static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
     if (flags & BDRV_REQ_COPY_ON_READ) {
         int64_t pnum;
 
+        /* The flag BDRV_REQ_COPY_ON_READ has reached its addressee */
+        flags &= ~BDRV_REQ_COPY_ON_READ;
+
         ret = bdrv_is_allocated(bs, offset, bytes, &pnum);
         if (ret < 0) {
             goto out;
@@ -1452,9 +1455,11 @@ static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
         goto out;
     }
 
+    assert(!(flags & ~bs->supported_read_flags));
+
     max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
     if (bytes <= max_bytes && bytes <= max_transfer) {
-        ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, 0);
+        ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, flags);
         goto out;
     }
 
@@ -1467,7 +1472,8 @@ static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
 
             ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
                                      num, qiov,
-                                     qiov_offset + bytes - bytes_remaining, 0);
+                                     qiov_offset + bytes - bytes_remaining,
+                                     flags);
             max_bytes -= num;
         } else {
             num = bytes_remaining;
Signature · 1 line
2.25.4
#21 lore raw

PATCH v15 13/13 block: apply COR-filter to block-stream jobs

In reply to Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

This patch completes the series with the COR-filter applied to block-stream operations.

Adding the filter makes it possible in future implement discarding copied regions in backing files during the block-stream job, to reduce the disk overuse (we need control on permissions).

Also, the filter now is smart enough to do copy-on-read with specified base, so we have benefit on guest reads even when doing block-stream of the part of the backing chain.

Several iotests are slightly modified due to filter insertion.

---
 block/stream.c             | 105 ++++++++++++++++++++++---------------
 tests/qemu-iotests/030     |   8 +--
 tests/qemu-iotests/141.out |   2 +-
 tests/qemu-iotests/245     |  20 ++++---
 4 files changed, 80 insertions(+), 55 deletions(-)
block/stream.c +62 -43
index 626dfa2b22..1fa742b0db 100644
@@ -17,8 +17,10 @@
 #include "block/blockjob_int.h"
 #include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
+#include "qapi/qmp/qdict.h"
 #include "qemu/ratelimit.h"
 #include "sysemu/block-backend.h"
+#include "block/copy-on-read.h"
 
 enum {
     /*
@@ -33,11 +35,11 @@ typedef struct StreamBlockJob {
     BlockJob common;
     BlockDriverState *base_overlay; /* COW overlay (stream from this) */
     BlockDriverState *above_base;   /* Node directly above the base */
+    BlockDriverState *cor_filter_bs;
     BlockDriverState *target_bs;
     BlockdevOnError on_error;
     char *backing_file_str;
     bool bs_read_only;
-    bool chain_frozen;
 } StreamBlockJob;
 
 static int coroutine_fn stream_populate(BlockBackend *blk,
@@ -45,17 +47,7 @@ static int coroutine_fn stream_populate(BlockBackend *blk,
 {
     assert(bytes < SIZE_MAX);
 
-    return blk_co_preadv(blk, offset, bytes, NULL,
-                         BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH);
-}
-
-static void stream_abort(Job *job)
-{
-    StreamBlockJob *s = container_of(job, StreamBlockJob, common.job);
-
-    if (s->chain_frozen) {
-        bdrv_unfreeze_backing_chain(s->target_bs, s->above_base);
-    }
+    return blk_co_preadv(blk, offset, bytes, NULL, BDRV_REQ_PREFETCH);
 }
 
 static int stream_prepare(Job *job)
@@ -67,8 +59,9 @@ static int stream_prepare(Job *job)
     Error *local_err = NULL;
     int ret = 0;
 
-    bdrv_unfreeze_backing_chain(s->target_bs, s->above_base);
-    s->chain_frozen = false;
+    /* We should drop filter at this point, as filter hold the backing chain */
+    bdrv_cor_filter_drop(s->cor_filter_bs);
+    s->cor_filter_bs = NULL;
 
     if (bdrv_cow_child(unfiltered_bs)) {
         const char *base_id = NULL, *base_fmt = NULL;
@@ -94,6 +87,11 @@ static void stream_clean(Job *job)
     StreamBlockJob *s = container_of(job, StreamBlockJob, common.job);
     BlockJob *bjob = &s->common;
 
+    if (s->cor_filter_bs) {
+        bdrv_cor_filter_drop(s->cor_filter_bs);
+        s->cor_filter_bs = NULL;
+    }
+
     /* Reopen the image back in read-only mode if necessary */
     if (s->bs_read_only) {
         /* Give up write permissions before making it read-only */
@@ -109,7 +107,6 @@ static int coroutine_fn stream_run(Job *job, Error **errp)
     StreamBlockJob *s = container_of(job, StreamBlockJob, common.job);
     BlockBackend *blk = s->common.blk;
     BlockDriverState *unfiltered_bs = bdrv_skip_filters(s->target_bs);
-    bool enable_cor = !bdrv_cow_child(s->base_overlay);
     int64_t len;
     int64_t offset = 0;
     uint64_t delay_ns = 0;
@@ -127,15 +124,6 @@ static int coroutine_fn stream_run(Job *job, Error **errp)
     }
     job_progress_set_remaining(&s->common.job, len);
 
-    /* Turn on copy-on-read for the whole block device so that guest read
-     * requests help us make progress.  Only do this when copying the entire
-     * backing chain since the copy-on-read operation does not take base into
-     * account.
-     */
-    if (enable_cor) {
-        bdrv_enable_copy_on_read(s->target_bs);
-    }
-
     for ( ; offset < len; offset += n) {
         bool copy;
         int ret;
@@ -194,10 +182,6 @@ static int coroutine_fn stream_run(Job *job, Error **errp)
         }
     }
 
-    if (enable_cor) {
-        bdrv_disable_copy_on_read(s->target_bs);
-    }
-
     /* Do not remove the backing file if an error was there but ignored. */
     return error;
 }
@@ -209,7 +193,6 @@ static const BlockJobDriver stream_job_driver = {
         .free          = block_job_free,
         .run           = stream_run,
         .prepare       = stream_prepare,
-        .abort         = stream_abort,
         .clean         = stream_clean,
         .user_resume   = block_job_user_resume,
     },
@@ -228,7 +211,9 @@ void stream_start(const char *job_id, BlockDriverState *bs,
     bool bs_read_only;
     int basic_flags = BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED;
     BlockDriverState *base_overlay;
+    BlockDriverState *cor_filter_bs = NULL;
     BlockDriverState *above_base;
+    QDict *opts;
 
     assert(!(base && bottom));
     assert(!(backing_file_str && bottom));
@@ -266,30 +251,62 @@ void stream_start(const char *job_id, BlockDriverState *bs,
         }
     }
 
-    if (bdrv_freeze_backing_chain(bs, above_base, errp) < 0) {
-        return;
-    }
-
     /* Make sure that the image is opened in read-write mode */
     bs_read_only = bdrv_is_read_only(bs);
     if (bs_read_only) {
-        if (bdrv_reopen_set_read_only(bs, false, errp) != 0) {
-            bs_read_only = false;
-            goto fail;
+        int ret;
+        /* Hold the chain during reopen */
+        if (bdrv_freeze_backing_chain(bs, above_base, errp) < 0) {
+            return;
+        }
+
+        ret = bdrv_reopen_set_read_only(bs, false, errp);
+
+        /* failure, or cor-filter will hold the chain */
+        bdrv_unfreeze_backing_chain(bs, above_base);
+
+        if (ret < 0) {
+            return;
         }
     }
 
-    /* Prevent concurrent jobs trying to modify the graph structure here, we
-     * already have our own plans. Also don't allow resize as the image size is
-     * queried only at the job start and then cached. */
-    s = block_job_create(job_id, &stream_job_driver, NULL, bs,
-                         basic_flags | BLK_PERM_GRAPH_MOD,
+    opts = qdict_new();
+
+    qdict_put_str(opts, "driver", "copy-on-read");
+    qdict_put_str(opts, "file", bdrv_get_node_name(bs));
+    /* Pass the base_overlay node name as 'bottom' to COR driver */
+    qdict_put_str(opts, "bottom", base_overlay->node_name);
+    if (filter_node_name) {
+        qdict_put_str(opts, "node-name", filter_node_name);
+    }
+
+    cor_filter_bs = bdrv_insert_node(bs, opts, BDRV_O_RDWR, errp);
+    if (!cor_filter_bs) {
+        goto fail;
+    }
+
+    if (!filter_node_name) {
+        cor_filter_bs->implicit = true;
+    }
+
+    s = block_job_create(job_id, &stream_job_driver, NULL, cor_filter_bs,
+                         BLK_PERM_CONSISTENT_READ,
                          basic_flags | BLK_PERM_WRITE,
                          speed, creation_flags, NULL, NULL, errp);
     if (!s) {
         goto fail;
     }
 
+    /*
+     * Prevent concurrent jobs trying to modify the graph structure here, we
+     * already have our own plans. Also don't allow resize as the image size is
+     * queried only at the job start and then cached.
+     */
+    if (block_job_add_bdrv(&s->common, "active node", bs, 0,
+                           basic_flags | BLK_PERM_WRITE, &error_abort)) {
+        goto fail;
+    }
+
     /* Block all intermediate nodes between bs and base, because they will
      * disappear from the chain after this operation. The streaming job reads
      * every block only once, assuming that it doesn't change, so forbid writes
@@ -310,9 +327,9 @@ void stream_start(const char *job_id, BlockDriverState *bs,
     s->base_overlay = base_overlay;
     s->above_base = above_base;
     s->backing_file_str = g_strdup(backing_file_str);
+    s->cor_filter_bs = cor_filter_bs;
     s->target_bs = bs;
     s->bs_read_only = bs_read_only;
-    s->chain_frozen = true;
 
     s->on_error = on_error;
     trace_stream_start(bs, base, s);
@@ -320,8 +337,10 @@ void stream_start(const char *job_id, BlockDriverState *bs,
     return;
 
 fail:
+    if (cor_filter_bs) {
+        bdrv_cor_filter_drop(cor_filter_bs);
+    }
     if (bs_read_only) {
         bdrv_reopen_set_read_only(bs, true, NULL);
     }
-    bdrv_unfreeze_backing_chain(bs, above_base);
 }
tests/qemu-iotests/030 +5 -3
index bd8cf9cff7..c576d55d07 100755
@@ -278,12 +278,14 @@ class TestParallelOps(iotests.QMPTestCase):
         self.assert_no_active_block_jobs()
 
         # Set a speed limit to make sure that this job blocks the rest
-        result = self.vm.qmp('block-stream', device='node4', job_id='stream-node4', base=self.imgs[1], speed=1024*1024)
+        result = self.vm.qmp('block-stream', device='node4',
+                             job_id='stream-node4', base=self.imgs[1],
+                             filter_node_name='stream-filter', speed=1024*1024)
         self.assert_qmp(result, 'return', {})
 
         result = self.vm.qmp('block-stream', device='node5', job_id='stream-node5', base=self.imgs[2])
         self.assert_qmp(result, 'error/desc',
-            "Node 'node4' is busy: block device is in use by block job: stream")
+            "Node 'stream-filter' is busy: block device is in use by block job: stream")
 
         result = self.vm.qmp('block-stream', device='node3', job_id='stream-node3', base=self.imgs[2])
         self.assert_qmp(result, 'error/desc',
@@ -296,7 +298,7 @@ class TestParallelOps(iotests.QMPTestCase):
         # block-commit should also fail if it touches nodes used by the stream job
         result = self.vm.qmp('block-commit', device='drive0', base=self.imgs[4], job_id='commit-node4')
         self.assert_qmp(result, 'error/desc',
-            "Node 'node4' is busy: block device is in use by block job: stream")
+            "Node 'stream-filter' is busy: block device is in use by block job: stream")
 
         result = self.vm.qmp('block-commit', device='drive0', base=self.imgs[1], top=self.imgs[3], job_id='commit-node1')
         self.assert_qmp(result, 'error/desc',
tests/qemu-iotests/141.out +1 -1
index 08e0aecd65..028a16f365 100644
@@ -99,7 +99,7 @@ wrote 1048576/1048576 bytes at offset 0
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job0"}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job0"}}
 {'execute': 'blockdev-del', 'arguments': {'node-name': 'drv0'}}
-{"error": {"class": "GenericError", "desc": "Node drv0 is in use"}}
+{"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: stream"}}
 {'execute': 'block-job-cancel', 'arguments': {'device': 'job0'}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "aborting", "id": "job0"}}
tests/qemu-iotests/245 +12 -8
index e60c8326d3..432e837e6c 100755
@@ -892,20 +892,24 @@ class TestBlockdevReopen(iotests.QMPTestCase):
 
         # hd1 <- hd0
         result = self.vm.qmp('block-stream', conv_keys = True, job_id = 'stream0',
-                             device = 'hd1', auto_finalize = False)
+                             device = 'hd1', filter_node_name='cor',
+                             auto_finalize = False)
         self.assert_qmp(result, 'return', {})
 
-        # We can't reopen with the original options because that would
-        # make hd1 read-only and block-stream requires it to be read-write
-        # (Which error message appears depends on whether the stream job is
-        # already done with copying at this point.)
+        # We can't reopen with the original options because there is a filter
+        # inserted by stream job above hd1.
         self.reopen(opts, {},
-            ["Can't set node 'hd1' to r/o with copy-on-read enabled",
-             "Cannot make block node read-only, there is a writer on it"])
+                    "Cannot change the option 'backing.backing.file.node-name'")
+
+        # We can't reopen hd1 to read-only, as block-stream requires it to be
+        # read-write
+        self.reopen(opts['backing'], {'read-only': True},
+                    "Cannot make block node read-only, there is a writer on it")
 
         # We can't remove hd2 while the stream job is ongoing
         opts['backing']['backing'] = None
-        self.reopen(opts, {'backing.read-only': False}, "Cannot change 'backing' link from 'hd1' to 'hd2'")
+        self.reopen(opts['backing'], {'read-only': False},
+                    "Cannot change 'backing' link from 'hd1' to 'hd2'")
 
         # We can detach hd1 from hd0 because it doesn't affect the stream job
         opts['backing'] = None
Signature · 1 line
2.25.4
#5 lore raw

Re: [PATCH v15 04/13] qapi: add filter-node-name to block-stream

In reply to Vladimir Sementsov-Ogievskiy
Quoted 16 lines · Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

Provide the possibility to pass the 'filter-node-name' parameter to the block-stream job as it is done for the commit block job.

[vsementsov: comment indentation, s/Since: 5.2/Since: 6.0/]
---
  qapi/block-core.json           | 6 ++++++
  include/block/block_int.h      | 7 ++++++-
  block/monitor/block-hmp-cmds.c | 4 ++--
  block/stream.c                 | 4 +++-
  blockdev.c                     | 4 +++-
  5 files changed, 20 insertions(+), 5 deletions(-)

[...]

include/block/block_int.h int is_windows_drive(const char *filename); · line 1146

Quoted 10 lines
include/block/block_int.h +2 -0
index 1eeafc118c..6f778e2517 100644
@@ -1142,6 +1142,9 @@ int is_windows_drive(const char *filename);
   *                  See @BlockJobCreateFlags
   * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
   * @on_error: The action to take upon error.
+ * @filter_node_name: The node name that should be assigned to the filter
+ *                    driver that the commit job inserts into the graph above

Only now noticed s/commit/stream/

(I’ll fix it)

Max

Quoted 4 lines
include/block/block_int.h +1 -0
+ *                    @bs. NULL means that a node name should be autogenerated.
   * @errp: Error object.
   *
   * Start a streaming operation on @bs.  Clusters that are unallocated
Reviewed-by on v15 9/13
#11 lore raw

Re: [PATCH v15 09/13] stream: rework backing-file changing

In reply to Vladimir Sementsov-Ogievskiy
Quoted 37 lines · Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

Stream in stream_prepare calls bdrv_change_backing_file() to change backing-file in the metadata of bs.

It may use either backing-file parameter given by user or just take filename of base on job start.

Backing file format is determined by base on job finish.

There are some problems with this design, we solve only two by this patch:

1. Consider scenario with backing-file unset. Current concept of stream supports changing of the base during the job (we don't freeze link to the base). So, we should not save base filename at job start,

- let's determine name of the base on job finish.

2. Using direct base to determine filename and format is not very good:
base node may be a filter, so its filename may be JSON, and format_name is not good for storing into qcow2 metadata as backing file format.

- let's use unfiltered_base

[vsementsov: change commit subject, change logic in stream_prepare]
---
  block/stream.c | 9 +++++----
  blockdev.c     | 8 +-------
  2 files changed, 6 insertions(+), 11 deletions(-)
block/stream.c +0 -0
index 6e281c71ac..6a525a5edf 100644

[...]

Quoted 5 lines
block/stream.c +0 -1
@@ -73,10 +74,10 @@ static int stream_prepare(Job *job)
 
      if (bdrv_cow_child(unfiltered_bs)) {
          const char *base_id = NULL, *base_fmt = NULL;
-        if (base) {

block/stream.c static int stream_prepare(Job *job) · line 83

Quoted 10 lines
block/stream.c +4 -3
-            base_id = s->backing_file_str;
-            if (base->drv) {
-                base_fmt = base->drv->format_name;
+        if (unfiltered_base) {
+            base_id = s->backing_file_str ?: unfiltered_base->filename;
+            if (unfiltered_base->drv) {
+                base_fmt = unfiltered_base->drv->format_name;
              }
          }
          bdrv_set_backing_hd(unfiltered_bs, base, &local_err);

I think I preferred the v14 behavior of not setting a backing file format if backing_file_str is nowhere to be found in the current backing chain. (I just noticed, I had a typo in my reply to v14, though; the “continuing on with setting a backing_fmt” should have read “continuing on *without* setting a backing_fmt”...)

Anyway, this is still an improvement on the pre-patch behavior, so:

(And as we discussed, the best would be for the user to specify a backing format through a yet-to-be-added option.)

#15 lore raw

Re: [PATCH v15 10/13] qapi: block-stream: add "bottom" argument

In reply to Vladimir Sementsov-Ogievskiy
Quoted 37 lines · Vladimir Sementsov-Ogievskiy

The code already don't freeze base node and we try to make it prepared for the situation when base node is changed during the operation. In other words, block-stream doesn't own base node.

Let's introduce a new interface which should replace the current one, which will in better relations with the code. Specifying bottom node instead of base, and requiring it to be non-filter gives us the following benefits:

- drop difference between above_base and base_overlay, which will be renamed to just bottom, when old interface dropped

- clean way to work with parallel streams/commits on the same backing chain, which otherwise become a problem when we introduce a filter for stream job

- cleaner interface. Nobody will surprised the fact that base node may disappear during block-stream, when there is no word about "base" in the interface.

---
  qapi/block-core.json           | 12 ++++---
  include/block/block_int.h      |  1 +
  block/monitor/block-hmp-cmds.c |  3 +-
  block/stream.c                 | 50 +++++++++++++++++++---------
  blockdev.c                     | 59 ++++++++++++++++++++++++++++------
  5 files changed, 94 insertions(+), 31 deletions(-)
qapi/block-core.json +0 -0
index b8094a5ec7..cb0066fd5c 100644
@@ -2517,10 +2517,14 @@
  # @device: the device or node name of the top image
  #
  # @base: the common backing file name.

qapi/block-core.json line 2527

Quoted 10 lines
qapi/block-core.json +6 -2
-#        It cannot be set if @base-node is also set.
+#        It cannot be set if @base-node or @bottom is also set.
  #
  # @base-node: the node name of the backing file.
-#             It cannot be set if @base is also set. (Since 2.8)
+#             It cannot be set if @base or @bottom is also set. (Since 2.8)
+#
+# @bottom: the last node in the chain that should be streamed into
+#          top. It cannot be set if @base or @base-node is also set.
+#          It cannot be filter node. (Since 6.0)

As far as I can make out, one of the results of our discussion on v14 was that when using backing-file + bottom, we want to require the user to specify backing-fmt as well. Now, backing-fmt isn’t present yet. Doesn’t that mean we have to make bottom + backing-file an error until we have backing-fmt (like it was in v14)?

Or do you consider the change to patch 9 sufficient to make at least the case work for which backing-file was purportedly introduced, i.e. FD passing? (Patch 9’s new version will just take the format of the base post-streaming, which would be most likely a correct guess when using FD passing.)

(I.e., now with patch 9 being more liberal in guessing, perhaps you decided to no longer make backing-fmt mandatory after all.)

Max

#22 lore raw

Re: [PATCH v15 13/13] block: apply COR-filter to block-stream jobs

In reply to Vladimir Sementsov-Ogievskiy
Quoted 28 lines · Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

This patch completes the series with the COR-filter applied to block-stream operations.

Adding the filter makes it possible in future implement discarding copied regions in backing files during the block-stream job, to reduce the disk overuse (we need control on permissions).

Also, the filter now is smart enough to do copy-on-read with specified base, so we have benefit on guest reads even when doing block-stream of the part of the backing chain.

Several iotests are slightly modified due to filter insertion.

---
  block/stream.c             | 105 ++++++++++++++++++++++---------------
  tests/qemu-iotests/030     |   8 +--
  tests/qemu-iotests/141.out |   2 +-
  tests/qemu-iotests/245     |  20 ++++---
  4 files changed, 80 insertions(+), 55 deletions(-)
block/stream.c +0 -0
index 626dfa2b22..1fa742b0db 100644

[...]

Quoted 1 line
block/stream.c +0 -0
@@ -266,30 +251,62 @@ void stream_start(const char *job_id, BlockDriverState *bs,

[...]

Quoted 8 lines
      /* Make sure that the image is opened in read-write mode */
      bs_read_only = bdrv_is_read_only(bs);
      if (bs_read_only) {
block/stream.c +2 -3
-        if (bdrv_reopen_set_read_only(bs, false, errp) != 0) {
-            bs_read_only = false;
-            goto fail;
+        int ret;
+        /* Hold the chain during reopen */

block/stream.c void stream_start(const char *job_id, BlockDriverState *bs, · line 268

Quoted 10 lines
block/stream.c +10 -0
+        if (bdrv_freeze_backing_chain(bs, above_base, errp) < 0) {
+            return;
+        }
+
+        ret = bdrv_reopen_set_read_only(bs, false, errp);
+
+        /* failure, or cor-filter will hold the chain */
+        bdrv_unfreeze_backing_chain(bs, above_base);
+
+        if (ret < 0) {

Shouldn’t we keep the “bs_read_only = false;” here?

(All the rest of this patch looks good.)

Max

Quoted 3 lines
+            return;
          }
      }
#12 lore raw

Re: [PATCH v15 09/13] stream: rework backing-file changing

In reply to Max Reitz
Quoted 58 lines · Max Reitz
Quoted 37 lines · Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

Stream in stream_prepare calls bdrv_change_backing_file() to change backing-file in the metadata of bs.

It may use either backing-file parameter given by user or just take filename of base on job start.

Backing file format is determined by base on job finish.

There are some problems with this design, we solve only two by this patch:

1. Consider scenario with backing-file unset. Current concept of stream supports changing of the base during the job (we don't freeze link to the base). So, we should not save base filename at job start,

   - let's determine name of the base on job finish.

2. Using direct base to determine filename and format is not very good:
base node may be a filter, so its filename may be JSON, and format_name is not good for storing into qcow2 metadata as backing file format.

   - let's use unfiltered_base

   [vsementsov: change commit subject, change logic in stream_prepare]

---
  block/stream.c | 9 +++++----
  blockdev.c     | 8 +-------

  2 files changed, 6 insertions(+), 11 deletions(-)
block/stream.c +0 -0
index 6e281c71ac..6a525a5edf 100644

[...]

Quoted 4 lines
block/stream.c +0 -1
@@ -73,10 +74,10 @@ static int stream_prepare(Job *job)
       if (bdrv_cow_child(unfiltered_bs)) {
           const char *base_id = NULL, *base_fmt = NULL;
-        if (base) {

block/stream.c static int stream_prepare(Job *job) · line 82

Quoted 10 lines
block/stream.c +4 -3
-            base_id = s->backing_file_str;
-            if (base->drv) {
-                base_fmt = base->drv->format_name;
+        if (unfiltered_base) {
+            base_id = s->backing_file_str ?: unfiltered_base->filename;
+            if (unfiltered_base->drv) {
+                base_fmt = unfiltered_base->drv->format_name;
               }
           }
           bdrv_set_backing_hd(unfiltered_bs, base, &local_err);

I think I preferred the v14 behavior of not setting a backing file format if backing_file_str is nowhere to be found in the current backing chain.  (I just noticed, I had a typo in my reply to v14, though; the “continuing on with setting a backing_fmt” should have read “continuing on *without* setting a backing_fmt”...)

Quoted 6 lines

Anyway, this is still an improvement on the pre-patch behavior, so:

(And as we discussed, the best would be for the user to specify a backing format through a yet-to-be-added option.)

We discussed that the original aim of backing_file_str arg is something like fd-passing, when qemu doesn't know real name. In that way what was done in v14 is a degradation: we'll never find such name in a backing chain. And acutally, using format of backing file is a correct thing.

So, as I understand now:

We set backing file to the node which is the new backing-bs (maybe, skipping some filters). Nobody should set backing in qcow2 metadata to something absolutely different. So, using format_name of backing bs (skipping filters) is a correct thing.

We want to support cases when qemu doens't know real file-names. So, trying to check filename, or search it in a backing chain is wrong idea..

Hmm, or when we search backing name, we really track what was written in backing_file field of some qcow2 image in a chain, so it should be something correct? Hmm, then may be v14 was OK.. But in this case again, user should not pass backing_file, but we can just use backing_file field of cow-parent of the base at job start.. Oh. Anyway, we can adjust it later in a separate series.

Signature · 2 lines
Best regards,
Vladimir
#16 lore raw

Re: [PATCH v15 10/13] qapi: block-stream: add "bottom" argument

In reply to Max Reitz
Quoted 42 lines · Max Reitz
Quoted 41 lines · Vladimir Sementsov-Ogievskiy

The code already don't freeze base node and we try to make it prepared for the situation when base node is changed during the operation. In other words, block-stream doesn't own base node.

Let's introduce a new interface which should replace the current one, which will in better relations with the code. Specifying bottom node instead of base, and requiring it to be non-filter gives us the following benefits:

  - drop difference between above_base and base_overlay, which will be renamed to just bottom, when old interface dropped

  - clean way to work with parallel streams/commits on the same backing chain, which otherwise become a problem when we introduce a filter for stream job

  - cleaner interface. Nobody will surprised the fact that base node may disappear during block-stream, when there is no word about "base" in the interface.

---
  qapi/block-core.json           | 12 ++++--- include/block/block_int.h      |  1 + block/monitor/block-hmp-cmds.c |  3 +-
  block/stream.c                 | 50 +++++++++++++++++++--------- blockdev.c                     | 59 ++++++++++++++++++++++++++++------

  5 files changed, 94 insertions(+), 31 deletions(-)
qapi/block-core.json +1 -1
index b8094a5ec7..cb0066fd5c 100644
@@ -2517,10 +2517,14 @@
   # @device: the device or node name of the top image
   #
   # @base: the common backing file name.
-#        It cannot be set if @base-node is also set.
+#        It cannot be set if @base-node or @bottom is also set.
   #
   # @base-node: the node name of the backing file.
Quoted 8 lines

qapi/block-core.json line 2527

Quoted 6 lines
qapi/block-core.json +5 -1
-#             It cannot be set if @base is also set. (Since 2.8)
+#             It cannot be set if @base or @bottom is also set. (Since 2.8)
+#
+# @bottom: the last node in the chain that should be streamed into
+#          top. It cannot be set if @base or @base-node is also set.
+#          It cannot be filter node. (Since 6.0)

As far as I can make out, one of the results of our discussion on v14 was that when using backing-file + bottom, we want to require the user to specify backing-fmt as well.  Now, backing-fmt isn’t present yet. Doesn’t that mean we have to make bottom + backing-file an error until we have backing-fmt (like it was in v14)?

See my answer on 09. I just have some doubts around backing-fmt and decided to keep it as is.

I don't think that we really need backing-fmt. We shouldn't have use-cases when backing-fmt is set to something another than final base node. Therefore, using format_name of final base node is a correct thing to do. So, I don't see the reason now for introducing new option.

Quoted 2 lines

Or do you consider the change to patch 9 sufficient to make at least the case work for which backing-file was purportedly introduced, i.e. FD passing?  (Patch 9’s new version will just take the format of the base post-streaming, which would be most likely a correct guess when using FD passing.)

Yes, I decided just to keep logic around backing-fmt as is. Hmm, definitely I should have described my decision in cover letter, for you not being surprised.

Quoted 5 lines

(I.e., now with patch 9 being more liberal in guessing, perhaps you decided to no longer make backing-fmt mandatory after all.)

Max

Signature · 2 lines
Best regards,
Vladimir
#23 lore raw

Re: [PATCH v15 13/13] block: apply COR-filter to block-stream jobs

In reply to Max Reitz
Quoted 48 lines · Max Reitz
Quoted 28 lines · Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

This patch completes the series with the COR-filter applied to block-stream operations.

Adding the filter makes it possible in future implement discarding copied regions in backing files during the block-stream job, to reduce the disk overuse (we need control on permissions).

Also, the filter now is smart enough to do copy-on-read with specified base, so we have benefit on guest reads even when doing block-stream of the part of the backing chain.

Several iotests are slightly modified due to filter insertion.

---
  block/stream.c             | 105 ++++++++++++++++++++++--------------- tests/qemu-iotests/030     |   8 +-- tests/qemu-iotests/141.out |   2 +-
  tests/qemu-iotests/245     |  20 ++++---

  4 files changed, 80 insertions(+), 55 deletions(-)
block/stream.c +0 -0
index 626dfa2b22..1fa742b0db 100644

[...]

Quoted 1 line
block/stream.c +0 -0
@@ -266,30 +251,62 @@ void stream_start(const char *job_id, BlockDriverState *bs,

[...]

Quoted 12 lines

      /* Make sure that the image is opened in read-write mode */ bs_read_only = bdrv_is_read_only(bs);
      if (bs_read_only) {
-        if (bdrv_reopen_set_read_only(bs, false, errp) != 0) { -            bs_read_only = false;
-            goto fail;
+        int ret;
+        /* Hold the chain during reopen */
+        if (bdrv_freeze_backing_chain(bs, above_base, errp) < 0) { +            return;
+        }
+

Quoted 9 lines

block/stream.c void stream_start(const char *job_id, BlockDriverState *bs, · line 268

Quoted 6 lines

+        ret = bdrv_reopen_set_read_only(bs, false, errp);

block/stream.c +5 -0
+
+        /* failure, or cor-filter will hold the chain */
+        bdrv_unfreeze_backing_chain(bs, above_base);
+
+        if (ret < 0) {

Shouldn’t we keep the “bs_read_only = false;” here?

No, as we don't goto fail. (pre-patch, we goto fail here, and don't want fail: code path to reopend back to RW (as reopening to RO is failed anyway (and we hope it's transactional enough)))

Quoted 8 lines

(All the rest of this patch looks good.)

Max

Quoted 3 lines

+            return;
          }
      }

Signature · 2 lines
Best regards,
Vladimir
#17 lore raw

Re: [PATCH v15 10/13] qapi: block-stream: add "bottom" argument

In reply to Vladimir Sementsov-Ogievskiy
Quoted 1 line · Vladimir Sementsov-Ogievskiy

We shouldn't have use-cases when backing-fmt is set to something another than final base node.

I mean, we shouldn't have use-cases when backing-file is [...]

Signature · 2 lines
Best regards,
Vladimir
#13 lore raw

Re: [PATCH v15 09/13] stream: rework backing-file changing

In reply to Vladimir Sementsov-Ogievskiy
Quoted 59 lines · Vladimir Sementsov-Ogievskiy
Quoted 58 lines · Max Reitz
Quoted 37 lines · Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

Stream in stream_prepare calls bdrv_change_backing_file() to change backing-file in the metadata of bs.

It may use either backing-file parameter given by user or just take filename of base on job start.

Backing file format is determined by base on job finish.

There are some problems with this design, we solve only two by this patch:

1. Consider scenario with backing-file unset. Current concept of stream supports changing of the base during the job (we don't freeze link to the base). So, we should not save base filename at job start,

   - let's determine name of the base on job finish.

2. Using direct base to determine filename and format is not very good:
base node may be a filter, so its filename may be JSON, and format_name is not good for storing into qcow2 metadata as backing file format.

   - let's use unfiltered_base

   [vsementsov: change commit subject, change logic in stream_prepare]

---
  block/stream.c | 9 +++++----
  blockdev.c     | 8 +-------

  2 files changed, 6 insertions(+), 11 deletions(-)
block/stream.c +0 -0
index 6e281c71ac..6a525a5edf 100644

[...]

Quoted 4 lines
block/stream.c +0 -1
@@ -73,10 +74,10 @@ static int stream_prepare(Job *job)
       if (bdrv_cow_child(unfiltered_bs)) {
           const char *base_id = NULL, *base_fmt = NULL;
-        if (base) {

block/stream.c static int stream_prepare(Job *job) · line 82

Quoted 10 lines
block/stream.c +4 -3
-            base_id = s->backing_file_str;
-            if (base->drv) {
-                base_fmt = base->drv->format_name;
+        if (unfiltered_base) {
+            base_id = s->backing_file_str ?: unfiltered_base->filename;
+            if (unfiltered_base->drv) {
+                base_fmt = unfiltered_base->drv->format_name;
               }
           }
           bdrv_set_backing_hd(unfiltered_bs, base, &local_err);

I think I preferred the v14 behavior of not setting a backing file format if backing_file_str is nowhere to be found in the current backing chain.  (I just noticed, I had a typo in my reply to v14, though; the “continuing on with setting a backing_fmt” should have read “continuing on *without* setting a backing_fmt”...)

Quoted 8 lines
Quoted 6 lines

Anyway, this is still an improvement on the pre-patch behavior, so:

(And as we discussed, the best would be for the user to specify a backing format through a yet-to-be-added option.)

We discussed that the original aim of backing_file_str arg is something like fd-passing, when qemu doesn't know real name. In that way what was done in v14 is a degradation: we'll never find such name in a backing chain. And acutally, using format of backing file is a correct thing.

It was my understanding that this was an example use case; other users might want to use backing-file for something else entirely. (I imagined using e.g. a completely different file in a different format.)

OTOH, considering that “something else entirely” simply doesn’t work (because the driver has to be something from the backing chain), my imagination was just too wild. If anyone should ever want to follow up on it, I expect them to complain, and we’ll worry about it then.

Quoted 7 lines

So, as I understand now:

We set backing file to the node which is the new backing-bs (maybe, skipping some filters). Nobody should set backing in qcow2 metadata to something absolutely different. So, using format_name of backing bs (skipping filters) is a correct thing.

We want to support cases when qemu doens't know real file-names. So, trying to check filename, or search it in a backing chain is wrong idea..

Hmm, or when we search backing name, we really track what was written in backing_file field of some qcow2 image in a chain, so it should be something correct?

If it does something else than what people want it to do, they’ll complain. :)

(It isn’t like this patch is breaking anything that would work right now.)

So I agree that we don’t need backing-fmt now. (Or maybe ever.)

Max

Max

Reviewed-by
#18 lore raw

Re: [PATCH v15 10/13] qapi: block-stream: add "bottom" argument

In reply to Vladimir Sementsov-Ogievskiy
Quoted 53 lines · Vladimir Sementsov-Ogievskiy
Quoted 43 lines · Max Reitz
Quoted 42 lines · Vladimir Sementsov-Ogievskiy

The code already don't freeze base node and we try to make it prepared for the situation when base node is changed during the operation. In other words, block-stream doesn't own base node.

Let's introduce a new interface which should replace the current one, which will in better relations with the code. Specifying bottom node instead of base, and requiring it to be non-filter gives us the following benefits:

  - drop difference between above_base and base_overlay, which will be renamed to just bottom, when old interface dropped

  - clean way to work with parallel streams/commits on the same backing chain, which otherwise become a problem when we introduce a filter for stream job

  - cleaner interface. Nobody will surprised the fact that base node may disappear during block-stream, when there is no word about "base" in the interface.

---
  qapi/block-core.json           | 12 ++++--- include/block/block_int.h      |  1 + block/monitor/block-hmp-cmds.c |  3 +-
  block/stream.c                 | 50 +++++++++++++++++++--------- blockdev.c                     | 59 ++++++++++++++++++++++++++++------

  5 files changed, 94 insertions(+), 31 deletions(-)
qapi/block-core.json +1 -2
index b8094a5ec7..cb0066fd5c 100644
@@ -2517,10 +2517,14 @@
   # @device: the device or node name of the top image
   #
   # @base: the common backing file name.
-#        It cannot be set if @base-node is also set.
+#        It cannot be set if @base-node or @bottom is also set.
   #
   # @base-node: the node name of the backing file.
-#             It cannot be set if @base is also set. (Since 2.8)
Quoted 8 lines

qapi/block-core.json line 2527

Quoted 6 lines
qapi/block-core.json +5 -0
+#             It cannot be set if @base or @bottom is also set. 
 (Since 2.8)
+#
+# @bottom: the last node in the chain that should be streamed into
+#          top. It cannot be set if @base or @base-node is also set.
+#          It cannot be filter node. (Since 6.0)

As far as I can make out, one of the results of our discussion on v14 was that when using backing-file + bottom, we want to require the user to specify backing-fmt as well.  Now, backing-fmt isn’t present yet. Doesn’t that mean we have to make bottom + backing-file an error until we have backing-fmt (like it was in v14)?

Quoted 3 lines

See my answer on 09. I just have some doubts around backing-fmt and decided to keep it as is.

I don't think that we really need backing-fmt. We shouldn't have use-cases when backing-fmt is set to something another than final base node. Therefore, using format_name of final base node is a correct thing to do. So, I don't see the reason now for introducing new option.

Yup, yup, all good.

Reviewed-by
#24 lore raw

Re: [PATCH v15 13/13] block: apply COR-filter to block-stream jobs

In reply to Vladimir Sementsov-Ogievskiy
Quoted 60 lines · Vladimir Sementsov-Ogievskiy
Quoted 49 lines · Max Reitz
Quoted 28 lines · Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

This patch completes the series with the COR-filter applied to block-stream operations.

Adding the filter makes it possible in future implement discarding copied regions in backing files during the block-stream job, to reduce the disk overuse (we need control on permissions).

Also, the filter now is smart enough to do copy-on-read with specified base, so we have benefit on guest reads even when doing block-stream of the part of the backing chain.

Several iotests are slightly modified due to filter insertion.

---
  block/stream.c             | 105 ++++++++++++++++++++++--------------- tests/qemu-iotests/030     |   8 +-- tests/qemu-iotests/141.out |   2 +-
  tests/qemu-iotests/245     |  20 ++++---

  4 files changed, 80 insertions(+), 55 deletions(-)
block/stream.c +0 -0
index 626dfa2b22..1fa742b0db 100644

[...]

Quoted 2 lines
block/stream.c +0 -0
@@ -266,30 +251,62 @@ void stream_start(const char *job_id, 
 BlockDriverState *bs,

[...]

Quoted 12 lines

      /* Make sure that the image is opened in read-write mode */ bs_read_only = bdrv_is_read_only(bs);
      if (bs_read_only) {
-        if (bdrv_reopen_set_read_only(bs, false, errp) != 0) { -            bs_read_only = false;
-            goto fail;
+        int ret;
+        /* Hold the chain during reopen */
+        if (bdrv_freeze_backing_chain(bs, above_base, errp) < 0) { +            return;
+        }
+

Quoted 9 lines

block/stream.c void stream_start(const char *job_id, BlockDriverState *bs, · line 268

Quoted 6 lines

+        ret = bdrv_reopen_set_read_only(bs, false, errp);

block/stream.c +5 -0
+
+        /* failure, or cor-filter will hold the chain */
+        bdrv_unfreeze_backing_chain(bs, above_base);
+
+        if (ret < 0) {

Shouldn’t we keep the “bs_read_only = false;” here?

Quoted 1 line

No, as we don't goto fail.

Ah, right, then it won’t do anything.

Quoted 1 line

(pre-patch, we goto fail here, and don't want fail: code path to reopend back to RW (as reopening to RO is failed anyway (and we hope it's transactional enough)))

That’s why we had bs_read_only = false; pre-patch, so the reopen back to RW is skipped.

And with this patch, we don’t need anything else from the “fail” path (freezing is done by the filter, and the filter doesn’t exist yet), so it’s correct to condense the “bs_read_only = false; goto fail;” into a plain “return”.

#25 lore raw

Re: [PATCH v15 13/13] block: apply COR-filter to block-stream jobs

In reply to Vladimir Sementsov-Ogievskiy
Quoted 23 lines · Vladimir Sementsov-Ogievskiy

From: Andrey Shinkevich <[email protected]>

This patch completes the series with the COR-filter applied to block-stream operations.

Adding the filter makes it possible in future implement discarding copied regions in backing files during the block-stream job, to reduce the disk overuse (we need control on permissions).

Also, the filter now is smart enough to do copy-on-read with specified base, so we have benefit on guest reads even when doing block-stream of the part of the backing chain.

Several iotests are slightly modified due to filter insertion.

---
  block/stream.c             | 105 ++++++++++++++++++++++---------------
  tests/qemu-iotests/030     |   8 +--
  tests/qemu-iotests/141.out |   2 +-
  tests/qemu-iotests/245     |  20 ++++---
  4 files changed, 80 insertions(+), 55 deletions(-)

[...]

tests/qemu-iotests/141.out wrote 1048576/1048576 bytes at offset 0 · line 105

Quoted 13 lines
tests/qemu-iotests/141.out +1 -1
index 08e0aecd65..028a16f365 100644
@@ -99,7 +99,7 @@ wrote 1048576/1048576 bytes at offset 0
  {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "created", "id": "job0"}}
  {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "running", "id": "job0"}}
  {'execute': 'blockdev-del', 'arguments': {'node-name': 'drv0'}}
-{"error": {"class": "GenericError", "desc": "Node drv0 is in use"}}
+{"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: stream"}}
  {'execute': 'block-job-cancel', 'arguments': {'device': 'job0'}}
  {"return": {}}
  {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "JOB_STATUS_CHANGE", "data": {"status": "aborting", "id": "job0"}}

Amusing side note: This context matches two places in 141.out. With 0e720781282 (which requires two contextual whitespace tweaks), the file gets longer, so the line number doesn’t match anymore. git then tries to apply the context to the first match (which is closer to line 99), which is wrong.

First time I had something like that happen, actually.

Max