Skip to contentSkip to conversation

nbd: trace long NBD operations

PATCH Superseded Denis V. Lunev 3 participants 5 messages May 27, 2022 Block layer

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 nbd: trace long NBD operations

At the moment there are 2 sources of lengthy operations if configured:
* open connection, which could retry inside and
* reconnect of already opened connection
These operations could be quite lengthy and cumbersome to catch thus it would be quite natural to add trace points for them.

This patch is based on the original downstream work made by Vladimir.

CC Eric Blake <[email protected]>
CC Vladimir Sementsov-Ogievskiy <[email protected]>
CC Kevin Wolf <[email protected]>
CC Hanna Reitz <[email protected]>
CC Paolo Bonzini <[email protected]>
---
 block/nbd.c             | 11 ++++++++---
 block/trace-events      |  2 ++
 nbd/client-connection.c |  2 ++
 nbd/trace-events        |  3 +++
 4 files changed, 15 insertions(+), 3 deletions(-)
block/nbd.c +8 -3
index 6085ab1d2c..f1a473d36b 100644
@@ -371,6 +371,7 @@ static bool nbd_client_connecting(BDRVNBDState *s)
 /* Called with s->requests_lock taken.  */
 static coroutine_fn void nbd_reconnect_attempt(BDRVNBDState *s)
 {
+    int ret;
     bool blocking = s->state == NBD_CLIENT_CONNECTING_WAIT;
 
     /*
@@ -380,6 +381,8 @@ static coroutine_fn void nbd_reconnect_attempt(BDRVNBDState *s)
     assert(nbd_client_connecting(s));
     assert(s->in_flight == 1);
 
+    trace_nbd_reconnect_attempt(s->bs->in_flight);
+
     if (blocking && !s->reconnect_delay_timer) {
         /*
          * It's the first reconnect attempt after switching to
@@ -401,7 +404,7 @@ static coroutine_fn void nbd_reconnect_attempt(BDRVNBDState *s)
     }
 
     qemu_mutex_unlock(&s->requests_lock);
-    nbd_co_do_establish_connection(s->bs, blocking, NULL);
+    ret = nbd_co_do_establish_connection(s->bs, blocking, NULL);
     qemu_mutex_lock(&s->requests_lock);
 
     /*
@@ -410,6 +413,8 @@ static coroutine_fn void nbd_reconnect_attempt(BDRVNBDState *s)
      * this I/O request (so draining removes all timers).
      */
     reconnect_delay_timer_del(s);
+
+    trace_nbd_reconnect_attempt_result(ret, s->bs->in_flight);
 }
 
 static coroutine_fn int nbd_receive_replies(BDRVNBDState *s, uint64_t handle)
@@ -1856,8 +1861,8 @@ static int nbd_process_options(BlockDriverState *bs, QDict *options,
         goto error;
     }
 
-    s->reconnect_delay = qemu_opt_get_number(opts, "reconnect-delay", 0);
-    s->open_timeout = qemu_opt_get_number(opts, "open-timeout", 0);
+    s->reconnect_delay = qemu_opt_get_number(opts, "reconnect-delay", 300);
+    s->open_timeout = qemu_opt_get_number(opts, "open-timeout", 300);
 
     ret = 0;
 
block/trace-events +2 -0
index 549090d453..caab699c22 100644
@@ -172,6 +172,8 @@ nbd_read_reply_entry_fail(int ret, const char *err) "ret = %d, err: %s"
 nbd_co_request_fail(uint64_t from, uint32_t len, uint64_t handle, uint16_t flags, uint16_t type, const char *name, int ret, const char *err) "Request failed { .from = %" PRIu64", .len = %" PRIu32 ", .handle = %" PRIu64 ", .flags = 0x%" PRIx16 ", .type = %" PRIu16 " (%s) } ret = %d, err: %s"
 nbd_client_handshake(const char *export_name) "export '%s'"
 nbd_client_handshake_success(const char *export_name) "export '%s'"
+nbd_reconnect_attempt(int in_flight) "in_flight %d"
+nbd_reconnect_attempt_result(int ret, int in_flight) "ret %d in_flight %d"
 
 # ssh.c
 ssh_restart_coroutine(void *co) "co=%p"
nbd/client-connection.c +2 -0
index 2a632931c3..a5ee82e804 100644
@@ -23,6 +23,7 @@
  */
 
 #include "qemu/osdep.h"
+#include "trace.h"
 
 #include "block/nbd.h"
 
@@ -210,6 +211,7 @@ static void *connect_thread_func(void *opaque)
             object_unref(OBJECT(conn->sioc));
             conn->sioc = NULL;
             if (conn->do_retry && !conn->detached) {
+                trace_nbd_connect_iteration(timeout);
                 qemu_mutex_unlock(&conn->mutex);
 
                 sleep(timeout);
nbd/trace-events +3 -0
index c4919a2dd5..bdadfdc82d 100644
@@ -73,3 +73,6 @@ nbd_co_receive_request_decode_type(uint64_t handle, uint16_t type, const char *n
 nbd_co_receive_request_payload_received(uint64_t handle, uint32_t len) "Payload received: handle = %" PRIu64 ", len = %" PRIu32
 nbd_co_receive_align_compliance(const char *op, uint64_t from, uint32_t len, uint32_t align) "client sent non-compliant unaligned %s request: from=0x%" PRIx64 ", len=0x%" PRIx32 ", align=0x%" PRIx32
 nbd_trip(void) "Reading request"
+
+# client-connection.c
+nbd_connect_iteration(int in_flight) "timeout %d"
Signature · 1 line
2.32.0
#1 lore raw
Quoted 52 lines · Denis V. Lunev

At the moment there are 2 sources of lengthy operations if configured:
* open connection, which could retry inside and
* reconnect of already opened connection
These operations could be quite lengthy and cumbersome to catch thus it would be quite natural to add trace points for them.

This patch is based on the original downstream work made by Vladimir.

CC Eric Blake <[email protected]>
CC Vladimir Sementsov-Ogievskiy <[email protected]>
CC Kevin Wolf <[email protected]>
CC Hanna Reitz <[email protected]>
CC Paolo Bonzini <[email protected]>
---
  block/nbd.c             | 11 ++++++++---
  block/trace-events      |  2 ++
  nbd/client-connection.c |  2 ++
  nbd/trace-events        |  3 +++
  4 files changed, 15 insertions(+), 3 deletions(-)
block/nbd.c +4 -1
index 6085ab1d2c..f1a473d36b 100644
@@ -371,6 +371,7 @@ static bool nbd_client_connecting(BDRVNBDState *s)
  /* Called with s->requests_lock taken.  */
  static coroutine_fn void nbd_reconnect_attempt(BDRVNBDState *s)
  {
+    int ret;
      bool blocking = s->state == NBD_CLIENT_CONNECTING_WAIT;
 
      /*
@@ -380,6 +381,8 @@ static coroutine_fn void nbd_reconnect_attempt(BDRVNBDState *s)
      assert(nbd_client_connecting(s));
      assert(s->in_flight == 1);
 
+    trace_nbd_reconnect_attempt(s->bs->in_flight);
+
      if (blocking && !s->reconnect_delay_timer) {
          /*
           * It's the first reconnect attempt after switching to
@@ -401,7 +404,7 @@ static coroutine_fn void nbd_reconnect_attempt(BDRVNBDState *s)
      }
 
      qemu_mutex_unlock(&s->requests_lock);
-    nbd_co_do_establish_connection(s->bs, blocking, NULL);
+    ret = nbd_co_do_establish_connection(s->bs, blocking, NULL);
      qemu_mutex_lock(&s->requests_lock);
 
      /*

block/nbd.c static coroutine_fn void nbd_reconnect_attempt(BDRVNBDState *s) · line 417

Quoted 5 lines
block/nbd.c +2 -0
@@ -410,6 +413,8 @@ static coroutine_fn void nbd_reconnect_attempt(BDRVNBDState *s)
       * this I/O request (so draining removes all timers).
       */
      reconnect_delay_timer_del(s);
+
+    trace_nbd_reconnect_attempt_result(ret, s->bs->in_flight);

May be better trace exactly after nbd_co_do_establish_connection(). Doesn't really matter, just simpler code.

block/nbd.c static int nbd_process_options(BlockDriverState *bs, QDict *options, · line 1865

Quoted 11 lines
block/nbd.c +2 -2
  }
 
  static coroutine_fn int nbd_receive_replies(BDRVNBDState *s, uint64_t handle)
@@ -1856,8 +1861,8 @@ static int nbd_process_options(BlockDriverState *bs, QDict *options,
          goto error;
      }
 
-    s->reconnect_delay = qemu_opt_get_number(opts, "reconnect-delay", 0);
-    s->open_timeout = qemu_opt_get_number(opts, "open-timeout", 0);
+    s->reconnect_delay = qemu_opt_get_number(opts, "reconnect-delay", 300);
+    s->open_timeout = qemu_opt_get_number(opts, "open-timeout", 300);

That's changing defaults. Should not be in this patch. And I don't think we can simply change upstream default of open-timeout, as it breaks habitual behavior.

block/trace-events nbd_read_reply_entry_fail(int ret, const char *err) "ret = %d, err: %s" · line 176

Quoted 13 lines
block/nbd.c +0 -0
      ret = 0;
 
block/trace-events +2 -0
index 549090d453..caab699c22 100644
@@ -172,6 +172,8 @@ nbd_read_reply_entry_fail(int ret, const char *err) "ret = %d, err: %s"
  nbd_co_request_fail(uint64_t from, uint32_t len, uint64_t handle, uint16_t flags, uint16_t type, const char *name, int ret, const char *err) "Request failed { .from = %" PRIu64", .len = %" PRIu32 ", .handle = %" PRIu64 ", .flags = 0x%" PRIx16 ", .type = %" PRIu16 " (%s) } ret = %d, err: %s"
  nbd_client_handshake(const char *export_name) "export '%s'"
  nbd_client_handshake_success(const char *export_name) "export '%s'"
+nbd_reconnect_attempt(int in_flight) "in_flight %d"
+nbd_reconnect_attempt_result(int ret, int in_flight) "ret %d in_flight %d"

bs->in_flight is "unsigned int", so a bit better would be use "unsigned int" and "%u" here

Quoted 16 lines
block/trace-events +0 -0
  # ssh.c
  ssh_restart_coroutine(void *co) "co=%p"
nbd/client-connection.c +1 -0
index 2a632931c3..a5ee82e804 100644
@@ -23,6 +23,7 @@
   */
 
  #include "qemu/osdep.h"
+#include "trace.h"
 
  #include "block/nbd.h"
 

nbd/client-connection.c static void *connect_thread_func(void *opaque) · line 214

Quoted 4 lines
nbd/client-connection.c +1 -0
@@ -210,6 +211,7 @@ static void *connect_thread_func(void *opaque)
              object_unref(OBJECT(conn->sioc));
              conn->sioc = NULL;
              if (conn->do_retry && !conn->detached) {
+                trace_nbd_connect_iteration(timeout);

Here we are going to sleep a bit, before next reconnect attempt. I'd call the trace-point "trace_nbd_connect_thread_sleep" or something like this to be more intuitive.

Quoted 14 lines
nbd/client-connection.c +0 -0
                  qemu_mutex_unlock(&conn->mutex);
 
                  sleep(timeout);
nbd/trace-events +3 -0
index c4919a2dd5..bdadfdc82d 100644
@@ -73,3 +73,6 @@ nbd_co_receive_request_decode_type(uint64_t handle, uint16_t type, const char *n
  nbd_co_receive_request_payload_received(uint64_t handle, uint32_t len) "Payload received: handle = %" PRIu64 ", len = %" PRIu32
  nbd_co_receive_align_compliance(const char *op, uint64_t from, uint32_t len, uint32_t align) "client sent non-compliant unaligned %s request: from=0x%" PRIx64 ", len=0x%" PRIx32 ", align=0x%" PRIx32
  nbd_trip(void) "Reading request"
+
+# client-connection.c
+nbd_connect_iteration(int in_flight) "timeout %d"
Signature · 2 lines
Best regards,
Vladimir
#2 lore raw
Quoted 51 lines · Vladimir Sementsov-Ogievskiy
Quoted 50 lines · Denis V. Lunev

At the moment there are 2 sources of lengthy operations if configured:
* open connection, which could retry inside and
* reconnect of already opened connection
These operations could be quite lengthy and cumbersome to catch thus it would be quite natural to add trace points for them.

This patch is based on the original downstream work made by Vladimir.

CC Eric Blake <[email protected]>
CC Vladimir Sementsov-Ogievskiy <[email protected]>
CC Kevin Wolf <[email protected]>
CC Hanna Reitz <[email protected]>
CC Paolo Bonzini <[email protected]>

---
  block/nbd.c             | 11 ++++++++---
  block/trace-events      |  2 ++
  nbd/client-connection.c |  2 ++
  nbd/trace-events        |  3 +++

  4 files changed, 15 insertions(+), 3 deletions(-)
block/nbd.c +2 -0
index 6085ab1d2c..f1a473d36b 100644
@@ -371,6 +371,7 @@ static bool nbd_client_connecting(BDRVNBDState *s)
   /* Called with s->requests_lock taken.  */
   static coroutine_fn void nbd_reconnect_attempt(BDRVNBDState *s)
   {
+    int ret;
       bool blocking = s->state == NBD_CLIENT_CONNECTING_WAIT;
         /*
@@ -380,6 +381,8 @@ static coroutine_fn void 
 nbd_reconnect_attempt(BDRVNBDState *s)
       assert(nbd_client_connecting(s));
       assert(s->in_flight == 1);
   +    trace_nbd_reconnect_attempt(s->bs->in_flight);
+
       if (blocking && !s->reconnect_delay_timer) {
           /*

           * It's the first reconnect attempt after switching to

block/nbd.c +1 -1
@@ -401,7 +404,7 @@ static coroutine_fn void 
 nbd_reconnect_attempt(BDRVNBDState *s)
       }
         qemu_mutex_unlock(&s->requests_lock);
-    nbd_co_do_establish_connection(s->bs, blocking, NULL);
+    ret = nbd_co_do_establish_connection(s->bs, blocking, NULL);
       qemu_mutex_lock(&s->requests_lock);
         /*
@@ -410,6 +413,8 @@ static coroutine_fn void 
Quoted 9 lines

block/nbd.c static coroutine_fn void nbd_reconnect_attempt(BDRVNBDState *s) · line 417

Quoted 6 lines
block/nbd.c +2 -0
 nbd_reconnect_attempt(BDRVNBDState *s)
        * this I/O request (so draining removes all timers).
        */
       reconnect_delay_timer_del(s);
+
+    trace_nbd_reconnect_attempt_result(ret, s->bs->in_flight);

May be better trace exactly after nbd_co_do_establish_connection(). Doesn't really matter, just simpler code.

ya, I'll change this

Quoted 5 lines
Quoted 5 lines
block/nbd.c +0 -0
   }
     static coroutine_fn int nbd_receive_replies(BDRVNBDState *s, uint64_t handle)
@@ -1856,8 +1861,8 @@ static int nbd_process_options(BlockDriverState 
 *bs, QDict *options,
           goto error;
Quoted 9 lines

block/nbd.c static int nbd_process_options(BlockDriverState *bs, QDict *options, · line 1865

Quoted 6 lines
block/nbd.c +2 -1
       }
   -    s->reconnect_delay = qemu_opt_get_number(opts, "reconnect-delay", 0);
-    s->open_timeout = qemu_opt_get_number(opts, "open-timeout", 0);
+    s->reconnect_delay = qemu_opt_get_number(opts, 
 "reconnect-delay", 300);
+    s->open_timeout = qemu_opt_get_number(opts, "open-timeout", 300);

That's changing defaults. Should not be in this patch. And I don't think we can simply change upstream default of open-timeout, as it breaks habitual behavior.

whoops :( used for testing, cut before sending once but somehow this goes out anyway.

Thanks for pointing this.

Quoted 7 lines
Quoted 7 lines
block/nbd.c +0 -0
         ret = 0;

  diff --git a/block/trace-events b/block/trace-events index 549090d453..caab699c22 100644

block/trace-events +0 -0
@@ -172,6 +172,8 @@ nbd_read_reply_entry_fail(int ret, const char 
 *err) "ret = %d, err: %s"
Quoted 9 lines
Quoted 6 lines

  nbd_co_request_fail(uint64_t from, uint32_t len, uint64_t handle, uint16_t flags, uint16_t type, const char *name, int ret, const char *err) "Request failed { .from = %" PRIu64", .len = %" PRIu32 ", .handle = %" PRIu64 ", .flags = 0x%" PRIx16 ", .type = %" PRIu16 " (%s) } ret = %d, err: %s" nbd_client_handshake(const char *export_name) "export '%s'"
  nbd_client_handshake_success(const char *export_name) "export '%s'"

block/trace-events +2 -0
+nbd_reconnect_attempt(int in_flight) "in_flight %d"
+nbd_reconnect_attempt_result(int ret, int in_flight) "ret %d 

in_flight %d"

bs->in_flight is "unsigned int", so a bit better would be use "unsigned int" and "%u" here

noted

Quoted 10 lines
Quoted 10 lines
    # ssh.c
  ssh_restart_coroutine(void *co) "co=%p"
nbd/client-connection.c +1 -0
index 2a632931c3..a5ee82e804 100644
@@ -23,6 +23,7 @@
    */
     #include "qemu/osdep.h"
+#include "trace.h"
Quoted 9 lines
Quoted 6 lines
nbd/client-connection.c +0 -0
     #include "block/nbd.h"
   @@ -210,6 +211,7 @@ static void *connect_thread_func(void *opaque)
               object_unref(OBJECT(conn->sioc));
               conn->sioc = NULL;

              if (conn->do_retry && !conn->detached) { +                trace_nbd_connect_iteration(timeout);

Here we are going to sleep a bit, before next reconnect attempt. I'd call the trace-point "trace_nbd_connect_thread_sleep" or something like this to be more intuitive.

ok

Quoted 16 lines
Quoted 14 lines

qemu_mutex_unlock(&conn->mutex);
                    sleep(timeout);

nbd/trace-events +0 -0
index c4919a2dd5..bdadfdc82d 100644
@@ -73,3 +73,6 @@ nbd_co_receive_request_decode_type(uint64_t handle, 

uint16_t type, const char *n
  nbd_co_receive_request_payload_received(uint64_t handle, uint32_t len) "Payload received: handle = %" PRIu64 ", len = %" PRIu32
  nbd_co_receive_align_compliance(const char *op, uint64_t from, uint32_t len, uint32_t align) "client sent non-compliant unaligned %s request: from=0x%" PRIx64 ", len=0x%" PRIx32 ", align=0x%" PRIx32 nbd_trip(void) "Reading request"

nbd/trace-events +3 -0
+
+# client-connection.c
+nbd_connect_iteration(int in_flight) "timeout %d"
#3 lore raw
In reply to Denis V. Lunev
Quoted 22 lines · Denis V. Lunev

+++ b/nbd/client-connection.c

nbd/client-connection.c +2 -0
@@ -23,6 +23,7 @@
   */
 
  #include "qemu/osdep.h"
+#include "trace.h"
 
  #include "block/nbd.h"
 
@@ -210,6 +211,7 @@ static void *connect_thread_func(void *opaque)
              object_unref(OBJECT(conn->sioc));
              conn->sioc = NULL;
              if (conn->do_retry && !conn->detached) {
+                trace_nbd_connect_iteration(timeout);
                  qemu_mutex_unlock(&conn->mutex);
 
                  sleep(timeout);
nbd/trace-events +0 -0
index c4919a2dd5..bdadfdc82d 100644

nbd/trace-events nbd_co_receive_request_decode_type(uint64_t handle, uint16_t type, const char *n · line 78

Quoted 6 lines
nbd/trace-events +3 -0
@@ -73,3 +73,6 @@ nbd_co_receive_request_decode_type(uint64_t handle, uint16_t type, const char *n
  nbd_co_receive_request_payload_received(uint64_t handle, uint32_t len) "Payload received: handle = %" PRIu64 ", len = %" PRIu32
  nbd_co_receive_align_compliance(const char *op, uint64_t from, uint32_t len, uint32_t align) "client sent non-compliant unaligned %s request: from=0x%" PRIx64 ", len=0x%" PRIx32 ", align=0x%" PRIx32
  nbd_trip(void) "Reading request"
+
+# client-connection.c
+nbd_connect_iteration(int in_flight) "timeout %d"

timeout is uint64_t, so, it should be "uint64_t timeout" here and %" PRIu64

Signature · 2 lines
Best regards,
Vladimir
#4 lore raw
Quoted 20 lines · Vladimir Sementsov-Ogievskiy
Quoted 19 lines · Denis V. Lunev

+++ b/nbd/client-connection.c

nbd/client-connection.c +1 -0
@@ -23,6 +23,7 @@
    */
     #include "qemu/osdep.h"
+#include "trace.h"
     #include "block/nbd.h"
   @@ -210,6 +211,7 @@ static void *connect_thread_func(void *opaque)
               object_unref(OBJECT(conn->sioc));
               conn->sioc = NULL;

              if (conn->do_retry && !conn->detached) { +                trace_nbd_connect_iteration(timeout); qemu_mutex_unlock(&conn->mutex); sleep(timeout);

nbd/trace-events +0 -0
index c4919a2dd5..bdadfdc82d 100644
@@ -73,3 +73,6 @@ nbd_co_receive_request_decode_type(uint64_t handle, 

uint16_t type, const char *n

Quoted 9 lines

nbd/trace-events nbd_co_receive_request_decode_type(uint64_t handle, uint16_t type, const char *n · line 78

Quoted 6 lines

  nbd_co_receive_request_payload_received(uint64_t handle, uint32_t len) "Payload received: handle = %" PRIu64 ", len = %" PRIu32
  nbd_co_receive_align_compliance(const char *op, uint64_t from, uint32_t len, uint32_t align) "client sent non-compliant unaligned %s request: from=0x%" PRIx64 ", len=0x%" PRIx32 ", align=0x%" PRIx32 nbd_trip(void) "Reading request"

nbd/trace-events +3 -0
+
+# client-connection.c
+nbd_connect_iteration(int in_flight) "timeout %d"

timeout is uint64_t, so, it should be "uint64_t timeout" here and %" PRIu64

Thanks! will change