Support larger NBD_CMD_WRITE_ZEROES

The NBD spec is clarifying [1] that a server may want to advertise different limits for READ/WRITE (in our case, 32M) than for TRIM/ZERO (in our case, nearly 4G). Add the constants and name lookups for new NBD_INFO_ fields used during handshake to convey this additional information. Note that the NBD spec already requires servers to ignore unknown requests from the client, and for clients to ignore unknown gratuitous responses sent from the server.

[1] https://lists.debian.org/nbd/2018/03/msg00048.html

Signed-off-by Eric Blake <[email protected]>

---

The given URL for the NBD spec was v3; it will change to be a v4 version of that patch in part to point back to this qemu commit as a proof of implementation.

---
  include/block/nbd.h | 4 +++-
  nbd/common.c        | 4 ++++
  2 files changed, 7 insertions(+), 1 deletion(-)

include/block/nbd.h +3 -1

index fcdcd545023..cbf51628f78 100644
@@ -1,5 +1,5 @@
  /*
- *  Copyright (C) 2016-2017 Red Hat, Inc.
+ *  Copyright (C) 2016-2018 Red Hat, Inc.
   *  Copyright (C) 2005  Anthony Liguori <[email protected]>
   *
   *  Network Block Device

@@ -180,6 +180,8 @@ typedef struct NBDExtent {
  #define NBD_INFO_NAME           1
  #define NBD_INFO_DESCRIPTION    2
  #define NBD_INFO_BLOCK_SIZE     3
+#define NBD_INFO_TRIM_SIZE      4
+#define NBD_INFO_ZERO_SIZE      5
 
  /* Request flags, sent from client to server during transmission phase */
  #define NBD_CMD_FLAG_FUA        (1 << 0) /* 'force unit access' during write */

nbd/common.c +4 -0

index 8c95c1d606e..840c91d5ca4 100644
@@ -129,6 +129,10 @@ const char *nbd_info_lookup(uint16_t info)
          return "description";
      case NBD_INFO_BLOCK_SIZE:
          return "block size";
+    case NBD_INFO_TRIM_SIZE:
+        return "trim size";
+    case NBD_INFO_ZERO_SIZE:
+        return "zero size";
      default:
          return "<unknown>";
      }

The next patch will ask the server for more items of NBD_INFO. However, the server is free to respond with INFO items in a different order than what we request, so performing any sanity checks about constraints that occur between multiple INFO items must be done after all items have been received. Make it easier to see that such checks are last by sinking the final processing after the while loop, rather than embedded in the NBD_REP_ACK processing that occurs textually within the loop before other INFO processing.

Signed-off-by Eric Blake <[email protected]>

---
  nbd/client.c | 19 +++++++++++--------
  1 file changed, 11 insertions(+), 8 deletions(-)

nbd/client.c +11 -8

index 232ff4f46da..0abb195b856 100644
@@ -1,5 +1,5 @@
  /*
- *  Copyright (C) 2016-2017 Red Hat, Inc.
+ *  Copyright (C) 2016-2018 Red Hat, Inc.
   *  Copyright (C) 2005  Anthony Liguori <[email protected]>
   *
   *  Network Block Device Client Side

@@ -365,17 +365,12 @@ static int nbd_opt_go(QIOChannel *ioc, const char *wantname,
 
          if (reply.type == NBD_REP_ACK) {
              /* Server is done sending info and moved into transmission
-               phase, but make sure it sent flags */
+               phase */
              if (len) {
                  error_setg(errp, "server sent invalid NBD_REP_ACK");
                  return -1;
              }
-            if (!info->flags) {
-                error_setg(errp, "broken server omitted NBD_INFO_EXPORT");
-                return -1;
-            }
-            trace_nbd_opt_go_success();
-            return 1;
+            break;
          }
          if (reply.type != NBD_REP_INFO) {
              error_setg(errp, "unexpected reply type %" PRIu32

@@ -482,6 +477,14 @@ static int nbd_opt_go(QIOChannel *ioc, const char *wantname,
              break;
          }
      }
+
+    /* Sanity check that server's responses make sense */
+    if (!info->flags) {
+        error_setg(errp, "broken server omitted NBD_INFO_EXPORT");
+        return -1;
+    }
+    trace_nbd_opt_go_success();
+    return 1;
  }
 
  /* Return -1 on failure, 0 if wantname is an available export. */

The NBD spec is clarifying [1] that a server may want to advertise different limits for READ/WRITE (in our case, 32M) than for TRIM/ZERO (in our case, nearly 4G). Implement the client side support for these alternate limits, by always requesting the new information (a compliant server must ignore the request if it is unknown), and by validating anything reported by the server before populating the block layer limits.

[1] https://lists.debian.org/nbd/2018/03/msg00048.html

Signed-off-by Eric Blake <[email protected]>

---

The given URL for the NBD spec was v3; it will change to be a v4 version of that patch in part to point back to this qemu commit as a proof of implementation.

---
  include/block/nbd.h |   4 ++
  block/nbd.c         |  15 ++++++-
  nbd/client.c        | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++--
  nbd/trace-events    |   2 +
  4 files changed, 131 insertions(+), 6 deletions(-)

include/block/nbd.h +4 -0

index cbf51628f78..90ddef32bb3 100644
@@ -270,6 +270,10 @@ struct NBDExportInfo {
      uint32_t min_block;
      uint32_t opt_block;
      uint32_t max_block;
+    uint32_t min_trim;
+    uint32_t max_trim;
+    uint32_t min_zero;
+    uint32_t max_zero;
 
      uint32_t meta_base_allocation_id;
  };

block/nbd.c +13 -2

index 1e2b3ba2d3b..823d10b251d 100644
@@ -478,8 +478,19 @@ static void nbd_refresh_limits(BlockDriverState *bs, Error **errp)
      uint32_t max = MIN_NON_ZERO(NBD_MAX_BUFFER_SIZE, s->info.max_block);
 
      bs->bl.request_alignment = min ? min : BDRV_SECTOR_SIZE;
-    bs->bl.max_pdiscard = max;
-    bs->bl.max_pwrite_zeroes = max;
+    if (s->info.max_trim) {
+        bs->bl.max_pdiscard = MIN(s->info.max_trim, BDRV_REQUEST_MAX_BYTES);
+    } else {
+        bs->bl.max_pdiscard = max;
+    }
+    bs->bl.pdiscard_alignment = s->info.min_trim;
+    if (s->info.max_zero) {
+        bs->bl.max_pwrite_zeroes = MIN(s->info.max_zero,
+                                       BDRV_REQUEST_MAX_BYTES);
+    } else {
+        bs->bl.max_pwrite_zeroes = max;
+    }
+    bs->bl.pwrite_zeroes_alignment = s->info.min_zero;
      bs->bl.max_transfer = max;
 
      if (s->info.opt_block &&

nbd/client.c +0 -0

index 0abb195b856..f1364747ba1 100644

nbd/client.c +1 -2

      stl_be_p(buf, len);
      memcpy(buf + 4, wantname, len);
-    /* At most one request, everything else up to server */
-    stw_be_p(buf + 4 + len, info->request_sizes);
+    /* Either 0 or 3 requests, everything else up to server */

nbd/client.c +20 -0

      g_free(buf);
      if (error < 0) {

@@ -467,6 +469,72 @@ static int nbd_opt_go(QIOChannel *ioc, const char *wantname,
                                               info->max_block);
              break;
 
+        case NBD_INFO_TRIM_SIZE:
+            if (len != sizeof(info->min_trim) * 2) {
+                error_setg(errp, "remaining trim info len %" PRIu32
+                           " is unexpected size", len);
+                nbd_send_opt_abort(ioc);
+                return -1;
+            }
+            if (nbd_read(ioc, &info->min_trim, sizeof(info->min_trim),
+                         errp) < 0) {
+                error_prepend(errp, "failed to read info minimum trim size: ");
+                nbd_send_opt_abort(ioc);
+                return -1;
+            }
+            be32_to_cpus(&info->min_trim);
+            if (nbd_read(ioc, &info->max_trim, sizeof(info->max_trim),
+                         errp) < 0) {
+                error_prepend(errp,
+                              "failed to read info maximum trim size: ");
+                nbd_send_opt_abort(ioc);
+                return -1;

nbd/client.c +26 -0

+            trace_nbd_opt_go_info_trim_size(info->min_trim, info->max_trim);
+            break;
+
+        case NBD_INFO_ZERO_SIZE:
+            if (len != sizeof(info->min_zero) * 2) {
+                error_setg(errp, "remaining zero info len %" PRIu32
+                           " is unexpected size", len);
+                nbd_send_opt_abort(ioc);
+                return -1;
+            }
+            if (nbd_read(ioc, &info->min_zero, sizeof(info->min_zero),
+                         errp) < 0) {
+                error_prepend(errp, "failed to read info minimum zero size: ");
+                nbd_send_opt_abort(ioc);
+                return -1;
+            }
+            be32_to_cpus(&info->min_zero);
+            if (nbd_read(ioc, &info->max_zero, sizeof(info->max_zero),
+                         errp) < 0) {
+                error_prepend(errp,
+                              "failed to read info maximum zero size: ");
+                nbd_send_opt_abort(ioc);
+                return -1;
+            }
+            be32_to_cpus(&info->max_zero);
+            if (!info->min_zero || !info->max_zero ||

nbd/client.c +40 -0

          default:
              trace_nbd_opt_go_info_unknown(type, nbd_info_lookup(type));
              if (nbd_drop(ioc, len, errp) < 0) {

@@ -483,6 +551,46 @@ static int nbd_opt_go(QIOChannel *ioc, const char *wantname,
          error_setg(errp, "broken server omitted NBD_INFO_EXPORT");
          return -1;
      }
+    if (info->min_trim) {
+        if (!info->min_block) {
+            error_setg(errp, "broken server sent INFO_TRIM_SIZE without"
+                       " INFO_BLOCK_SIZE");
+            return -1;
+        }
+        if (info->min_trim < info->opt_block) {
+            error_setg(errp, "broken server sent INFO_TRIM_SIZE with"
+                       " minimum trim %" PRIu32 " less than preferred block %"
+                       PRIu32, info->min_trim, info->opt_block);
+            return -1;
+        }
+        if (info->max_trim < info->max_block) {
+            error_setg(errp, "broken server sent INFO_TRIM_SIZE with"
+                       " maximum trim %" PRIu32 " less than maximum block %"
+                       PRIu32, info->max_trim, info->max_block);
+            return -1;
+        }
+        info->max_trim = QEMU_ALIGN_DOWN(info->max_trim, info->min_block);
+    }
+    if (info->min_zero) {
+        if (!info->min_block) {
+            error_setg(errp, "broken server sent INFO_ZERO_SIZE without"
+                       " INFO_BLOCK_SIZE");
+            return -1;
+        }
+        if (info->min_zero < info->opt_block) {
+            error_setg(errp, "broken server sent INFO_ZERO_SIZE with"
+                       " minimum zero %" PRIu32 " less than preferred block %"
+                       PRIu32, info->min_zero, info->opt_block);
+            return -1;
+        }
+        if (info->max_zero < info->max_block) {
+            error_setg(errp, "broken server sent INFO_ZERO_SIZE with"
+                       " maximum zero %" PRIu32 " less than maximum block %"
+                       PRIu32, info->max_zero, info->max_block);
+            return -1;
+        }
+        info->max_zero = QEMU_ALIGN_DOWN(info->max_zero, info->min_block);
+    }
      trace_nbd_opt_go_success();
      return 1;
  }

nbd/trace-events +2 -0

index dee081e7758..ddb9d0a2b3e 100644
@@ -6,6 +6,8 @@ nbd_opt_go_start(const char *name) "Attempting NBD_OPT_GO for export '%s'"
  nbd_opt_go_success(void) "Export is good to go"
  nbd_opt_go_info_unknown(int info, const char *name) "Ignoring unknown info %d (%s)"
  nbd_opt_go_info_block_size(uint32_t minimum, uint32_t preferred, uint32_t maximum) "Block sizes are 0x%" PRIx32 ", 0x%" PRIx32 ", 0x%" PRIx32
+nbd_opt_go_info_trim_size(uint32_t minimum, uint32_t maximum) "Trim sizes are 0x%" PRIx32 ", 0x%" PRIx32
+nbd_opt_go_info_zero_size(uint32_t minimum, uint32_t maximum) "Zero sizes are 0x%" PRIx32 ", 0x%" PRIx32
  nbd_receive_query_exports_start(const char *wantname) "Querying export list for '%s'"
  nbd_receive_query_exports_success(const char *wantname) "Found desired export name '%s'"
  nbd_receive_starttls_new_client(void) "Setting up TLS"

Quoted 18 lines · Eric Blake

The NBD spec is clarifying [1] that a server may want to advertise different limits for READ/WRITE (in our case, 32M) than for TRIM/ZERO (in our case, nearly 4G).  Implement the client side support for these alternate limits, by always requesting the new information (a compliant server must ignore the request if it is unknown), and by validating anything reported by the server before populating the block layer limits.

[1] https://lists.debian.org/nbd/2018/03/msg00048.html

Signed-off-by Eric Blake <[email protected]>

+++ b/nbd/client.c

nbd/client.c +0 -0

@@ -337,16 +337,18 @@ static int nbd_opt_go(QIOChannel *ioc, const 
 char *wantname,
       info->flags = 0;
 

nbd/client.c static int nbd_opt_go(QIOChannel *ioc, const char *wantname, · line 340

Quoted 3 lines

nbd/client.c +1 -1

       trace_nbd_opt_go_start(wantname);
-    buf = g_malloc(4 + len + 2 + 2 * info->request_sizes + 1);
+    buf = g_malloc(4 + len + 2 + 3 * 2 * info->request_sizes + 1);

Hmm, what "+1" means?

Quoted 16 lines

nbd/client.c +5 -3

       stl_be_p(buf, len);
       memcpy(buf + 4, wantname, len);
-    /* At most one request, everything else up to server */
-    stw_be_p(buf + 4 + len, info->request_sizes);
+    /* Either 0 or 3 requests, everything else up to server */
+    stw_be_p(buf + 4 + len, 3 * info->request_sizes);
       if (info->request_sizes) {
           stw_be_p(buf + 4 + len + 2, NBD_INFO_BLOCK_SIZE);
+        stw_be_p(buf + 4 + len + 2 + 2, NBD_INFO_TRIM_SIZE);
+        stw_be_p(buf + 4 + len + 2 + 2 + 2, NBD_INFO_ZERO_SIZE);
       }
       error = nbd_send_option_request(ioc, NBD_OPT_GO,
-                                    4 + len + 2 + 2 * 
 info->request_sizes,
+                                    4 + len + 2 + 3 * 2 * 
 info->request_sizes,

Quoted 1 line

                                      buf, errp);

magic chunk) Change looks correct. Is it worth introducing a buf_len variable, to not retype it?

Quoted 11 lines

+            be32_to_cpus(&info->max_trim);
+            if (!info->min_trim || !info->max_trim ||
+                (info->max_trim != UINT32_MAX &&
+                 !QEMU_IS_ALIGNED(info->max_trim, info->min_trim))) { +                error_setg(errp, "server trim sizes %" PRIu32 "/%" PRIu32
+                           " are not valid", info->min_trim, info->max_trim);
+                nbd_send_opt_abort(ioc);
+                return -1;
+            }

Didn't you forget to check that min_trim is a power of two?

hmm, so, now nbd_opt_go() is full of very-very similar code parts, which may worth some refactoring now or in future.

Are we going to add similar limits for BLOCK_STATUS? and for CACHE? I have an idea: why not to make an universal option for it in protocol? Something like

option INFO_CMD_SIZE:
  uint16_t command
  uint32_t min_block
  uint32_t max_block

and server can send several such options. Most of semantics for these additional limits are common, so it will simplify both documentation and realization..

I'll copy this to nbd thread and it is better to discuss it in it.

Quoted 19 lines · Vladimir Sementsov-Ogievskiy

Quoted 18 lines · Eric Blake

The NBD spec is clarifying [1] that a server may want to advertise different limits for READ/WRITE (in our case, 32M) than for TRIM/ZERO (in our case, nearly 4G).  Implement the client side support for these alternate limits, by always requesting the new information (a compliant server must ignore the request if it is unknown), and by validating anything reported by the server before populating the block layer limits.

[1] https://lists.debian.org/nbd/2018/03/msg00048.html

Signed-off-by Eric Blake <[email protected]>

+++ b/nbd/client.c

nbd/client.c +0 -0

@@ -337,16 +337,18 @@ static int nbd_opt_go(QIOChannel *ioc, const 
 char *wantname,
       info->flags = 0;
 

Quoted 5 lines

nbd/client.c static int nbd_opt_go(QIOChannel *ioc, const char *wantname, · line 340

Quoted 3 lines

nbd/client.c +1 -1

       trace_nbd_opt_go_start(wantname);
-    buf = g_malloc(4 + len + 2 + 2 * info->request_sizes + 1);
+    buf = g_malloc(4 + len + 2 + 3 * 2 * info->request_sizes + 1);

Hmm, what "+1" means?

Doesn't appear to be necessary, but a little slop never hurts. Better might be struct-ifying things rather than open-coding offsets, or even using a vectored approach rather than requiring a single buffer.  But if useful, that's refactoring that is independent of this patch, while this is just demonstrating the bare minimum implementation of the new feature.

Quoted 18 lines

Quoted 16 lines

nbd/client.c +5 -3

       stl_be_p(buf, len);
       memcpy(buf + 4, wantname, len);
-    /* At most one request, everything else up to server */
-    stw_be_p(buf + 4 + len, info->request_sizes);
+    /* Either 0 or 3 requests, everything else up to server */
+    stw_be_p(buf + 4 + len, 3 * info->request_sizes);
       if (info->request_sizes) {
           stw_be_p(buf + 4 + len + 2, NBD_INFO_BLOCK_SIZE);
+        stw_be_p(buf + 4 + len + 2 + 2, NBD_INFO_TRIM_SIZE);
+        stw_be_p(buf + 4 + len + 2 + 2 + 2, NBD_INFO_ZERO_SIZE);
       }
       error = nbd_send_option_request(ioc, NBD_OPT_GO,
-                                    4 + len + 2 + 2 * 
 info->request_sizes,
+                                    4 + len + 2 + 3 * 2 * 
 info->request_sizes,

Quoted 1 line

                                      buf, errp);

Quoted 2 lines

magic chunk) Change looks correct. Is it worth introducing a buf_len variable, to not retype it?

And increment as we go?  Yeah, it might make things more legible.

Quoted 12 lines

Quoted 11 lines

+ be32_to_cpus(&info->max_trim);
+            if (!info->min_trim || !info->max_trim ||
+                (info->max_trim != UINT32_MAX &&
+                 !QEMU_IS_ALIGNED(info->max_trim, info->min_trim))) { +                error_setg(errp, "server trim sizes %" PRIu32 "/%" PRIu32
+                           " are not valid", info->min_trim, info->max_trim);
+                nbd_send_opt_abort(ioc);
+                return -1;
+            }

Quoted 1 line

Didn't you forget to check that min_trim is a power of two?

Nope. The NBD spec wording specifically uses "The minimum trim size SHOULD be a power of 2, and MUST be at least as large as the preferred block size advertised in NBD_INFO_BLOCK_SIZE", in part because there ARE existing hardware iscsi devices that have an advertised preferred/maximum trim size of exactly 15 megabytes (you can request smaller trims 1M at a time, and the device then caches things to eventually report unmapped slices at a 15M boundary).  For more information on this odd hardware, look in the qemu logs for the Dell Equallogic device, such as commit 3482b9bc.

Since the NBD 'min trim' is advisory, and merely telling the client the ideal alignments to use for a trim most likely to have an effect (on Equallogic, a 1M trim by itself might not trim anything unless neighboring areas are also trimmed, while an aligned 15M trim is immediately observable), an NBD server wrapping such an iscsi device may prefer to mirror the hardware's preferred alignment of 15M, rather than inventing a minimum alignment of 1M, in what it advertises in NBD_INFO_TRIM_SIZE.

And maybe I should tweak the NBD spec addition to call things "preferred trim/zero" rather than "min trim/zero", if that makes things any easier to grasp on first read, and better matches the iscsi concept.

Quoted 12 lines

hmm, so, now nbd_opt_go() is full of very-very similar code parts, which may worth some refactoring now or in future.

Are we going to add similar limits for BLOCK_STATUS? and for CACHE? I have an idea: why not to make an universal option for it in protocol? Something like

option INFO_CMD_SIZE:
   uint16_t command
   uint32_t min_block
   uint32_t max_block

Quoted 4 lines

and server can send several such options. Most of semantics for these additional limits are common, so it will simplify both documentation and realization..

I'll copy this to nbd thread and it is better to discuss it in it.

Yep, I've followed up on that thread, but will post a v2 of this proposal along those lines.

The NBD spec is clarifying [1] that a server may want to advertise different limits for READ/WRITE (in our case, 32M) than for TRIM/ZERO (in our case, nearly 4G). Implement the client side support for these alternate limits, by always requesting the new information (a compliant server must ignore the request if it is unknown), and by validating anything reported by the server before populating the block layer limits.

[1] https://lists.debian.org/nbd/2018/03/msg00048.html

Signed-off-by Eric Blake <[email protected]>

---

The given URL for the NBD spec was v3; it will change to be a v4 version of that patch in part to point back to this qemu commit as a proof of implementation.

---
  include/block/nbd.h |   4 ++
  block/nbd.c         |  15 ++++++-
  nbd/client.c        | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++--
  nbd/trace-events    |   2 +
  4 files changed, 131 insertions(+), 6 deletions(-)

include/block/nbd.h +4 -0

index cbf51628f78..90ddef32bb3 100644
@@ -270,6 +270,10 @@ struct NBDExportInfo {
      uint32_t min_block;
      uint32_t opt_block;
      uint32_t max_block;
+    uint32_t min_trim;
+    uint32_t max_trim;
+    uint32_t min_zero;
+    uint32_t max_zero;
 
      uint32_t meta_base_allocation_id;
  };

block/nbd.c +4 -2

index 1e2b3ba2d3b..823d10b251d 100644
@@ -478,8 +478,19 @@ static void nbd_refresh_limits(BlockDriverState *bs, Error **errp)
      uint32_t max = MIN_NON_ZERO(NBD_MAX_BUFFER_SIZE, s->info.max_block);
 
      bs->bl.request_alignment = min ? min : BDRV_SECTOR_SIZE;
-    bs->bl.max_pdiscard = max;
-    bs->bl.max_pwrite_zeroes = max;
+    if (s->info.max_trim) {
+        bs->bl.max_pdiscard = MIN(s->info.max_trim, BDRV_REQUEST_MAX_BYTES);
+    } else {
+        bs->bl.max_pdiscard = max;

Quoted 7 lines · Eric Blake

The NBD spec is clarifying [1] that a server may want to advertise different limits for READ/WRITE (in our case, 32M) than for TRIM/ZERO (in our case, nearly 4G).  Implement the client side support for these alternate limits, by always requesting the new information (a compliant server must ignore the request if it is unknown), and by validating anything reported by the server before populating the block layer limits.

Quoted 10 lines

      bs->bl.request_alignment = min ? min : BDRV_SECTOR_SIZE; -    bs->bl.max_pdiscard = max;
-    bs->bl.max_pwrite_zeroes = max;
+    if (s->info.max_trim) {
+        bs->bl.max_pdiscard = MIN(s->info.max_trim, BDRV_REQUEST_MAX_BYTES);
+    } else {
+        bs->bl.max_pdiscard = max;
+    }
+    bs->bl.pdiscard_alignment = s->info.min_trim;

Quoted 8 lines

+    if (s->info.max_zero) {
+        bs->bl.max_pwrite_zeroes = MIN(s->info.max_zero, +                                       BDRV_REQUEST_MAX_BYTES); +    } else {
+        bs->bl.max_pwrite_zeroes = max;
+    }
+    bs->bl.pwrite_zeroes_alignment = s->info.min_zero; bs->bl.max_transfer = max;

Should not max_transfer be updated too? Looks like it limits all requests, is it right?