Skip to contentSkip to conversation

RBD: Add support readv,writev for rbd

RFC Superseded [email protected] 7 participants 13 messages Jan 21, 2017 – Feb 16, 2017 Block layer

v3 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#0 lore raw

RFC v3 1/1 RBD: Add support readv,writev for rbd

From: tianqing <[email protected]>

Rbd can do readv and writev directly, so wo do not need to transform iov to buf or vice versa any more.

---
 block/rbd.c | 47 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 41 insertions(+), 6 deletions(-)
block/rbd.c +41 -6
index a57b3e3..4add4dd 100644
@@ -73,7 +73,12 @@ typedef struct RBDAIOCB {
     BlockAIOCB common;
     int64_t ret;
     QEMUIOVector *qiov;
+/* Note:
+ * The LIBRBD_SUPPORTS_IOVEC is defined in librbd.h.
+ */
+#ifndef LIBRBD_SUPPORTS_IOVEC
     char *bounce;
+#endif
     RBDAIOCmd cmd;
     int error;
     struct BDRVRBDState *s;
@@ -83,7 +88,9 @@ typedef struct RADOSCB {
     RBDAIOCB *acb;
     struct BDRVRBDState *s;
     int64_t size;
+#ifndef LIBRBD_SUPPORTS_IOVEC
     char *buf;
+#endif
     int64_t ret;
 } RADOSCB;
 
@@ -426,11 +433,21 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
         }
     } else {
         if (r < 0) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
             memset(rcb->buf, 0, rcb->size);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov, 0, 0, acb->qiov->size);
+#endif
             acb->ret = r;
             acb->error = 1;
         } else if (r < rcb->size) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
             memset(rcb->buf + r, 0, rcb->size - r);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov,
+                       r, 0, acb->qiov->size - r);
+#endif
+
             if (!acb->error) {
                 acb->ret = rcb->size;
             }
@@ -441,10 +458,12 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
 
     g_free(rcb);
 
+#ifndef LIBRBD_SUPPORTS_IOVEC
     if (acb->cmd == RBD_AIO_READ) {
         qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
     }
     qemu_vfree(acb->bounce);
+#endif
     acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret));
 
     qemu_aio_unref(acb);
@@ -655,8 +674,10 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
     RBDAIOCB *acb;
     RADOSCB *rcb = NULL;
     rbd_completion_t c;
-    char *buf;
     int r;
+#ifndef LIBRBD_SUPPORTS_IOVEC
+    char * buf = NULL; 
+#endif
 
     BDRVRBDState *s = bs->opaque;
 
@@ -664,6 +685,8 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
     acb->cmd = cmd;
     acb->qiov = qiov;
     assert(!qiov || qiov->size == size);
+#ifndef LIBRBD_SUPPORTS_IOVEC
+    
     if (cmd == RBD_AIO_DISCARD || cmd == RBD_AIO_FLUSH) {
         acb->bounce = NULL;
     } else {
@@ -672,19 +695,21 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
             goto failed;
         }
     }
-    acb->ret = 0;
-    acb->error = 0;
-    acb->s = s;
-
     if (cmd == RBD_AIO_WRITE) {
         qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
     }
-
     buf = acb->bounce;
+#endif
+    acb->ret = 0;
+    acb->error = 0;
+    acb->s = s;
 
     rcb = g_new(RADOSCB, 1);
+
     rcb->acb = acb;
+#ifndef LIBRBD_SUPPORTS_IOVEC
     rcb->buf = buf;
+#endif
     rcb->s = acb->s;
     rcb->size = size;
     r = rbd_aio_create_completion(rcb, (rbd_callback_t) rbd_finish_aiocb, &c);
@@ -694,10 +719,18 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
 
     switch (cmd) {
     case RBD_AIO_WRITE:
+#ifndef LIBRBD_SUPPORTS_IOVEC
         r = rbd_aio_write(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_writev(s->image, qiov->iov, qiov->niov, off, c);
+#endif
         break;
     case RBD_AIO_READ:
+#ifndef LIBRBD_SUPPORTS_IOVEC
         r = rbd_aio_read(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_readv(s->image, qiov->iov, qiov->niov, off, c);
+#endif
         break;
     case RBD_AIO_DISCARD:
         r = rbd_aio_discard_wrapper(s->image, off, size, c);
@@ -719,7 +752,9 @@ failed_completion:
     rbd_aio_release(c);
 failed:
     g_free(rcb);
+#ifndef LIBRBD_SUPPORTS_IOVEC
     qemu_vfree(acb->bounce);
+#endif
     qemu_aio_unref(acb);
     return NULL;
 }
Signature · 1 line
1.8.3.1
#1 lore raw
Your series seems to have some coding style problems. See output below for more information:

Hi,

Your series seems to have some coding style problems. See output below for more information:

Subject: [Qemu-devel] [RFC v3] RBD: Add support readv,writev for rbd
Message-id: [email protected]
Type: series

=== TEST SCRIPT BEGIN ===
#!/bin/bash

BASE=base
n=1
total=$(git log --oneline $BASE.. | wc -l)
failed=0

# Useful git options
git config --local diff.renamelimit 0
git config --local diff.renames True

commits="$(git log --format=%H --reverse $BASE..)" for c in $commits; do

    echo "Checking PATCH $n/$total: $(git log -n 1 --format=%s $c)..."
    if ! git show $c --format=email | ./scripts/checkpatch.pl --mailback -; then
        failed=1
        echo
    fi
    n=$((n+1))

done

exit $failed
=== TEST SCRIPT END ===

Updating 3c8cf5a9c21ff8782164d1def7f44bd888713384
Switched to a new branch 'test'
f826583 RBD: Add support readv,writev for rbd

=== OUTPUT BEGIN ===
Checking PATCH 1/1: RBD: Add support readv,writev for rbd...
ERROR: trailing whitespace
#81: FILE: block/rbd.c:679:
+ char * buf = NULL; $

ERROR: "foo * bar" should be "foo *bar"
#81: FILE: block/rbd.c:679:
+ char * buf = NULL;

ERROR: trailing whitespace
#91: FILE: block/rbd.c:689:
+ $

total: 3 errors, 0 warnings, 126 lines checked

Your patch has style problems, please review. If any of these errors are false positives report them to the maintainer, see CHECKPATCH in MAINTAINERS.

=== OUTPUT END ===

Test command exited with code: 1

---
Email generated automatically by Patchew [http://patchew.org/].
Please send your feedback to [email protected]

#2 lore raw

RFC v4 1/1 RBD: Add support readv,writev for rbd

From: tianqing <[email protected]>

Rbd can do readv and writev directly, so wo do not need to transform iov to buf or vice versa any more.

---
 block/rbd.c | 47 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 41 insertions(+), 6 deletions(-)
block/rbd.c +41 -6
index a57b3e3..0cbd9e3 100644
@@ -73,7 +73,12 @@ typedef struct RBDAIOCB {
     BlockAIOCB common;
     int64_t ret;
     QEMUIOVector *qiov;
+/* Note:
+ * The LIBRBD_SUPPORTS_IOVEC is defined in librbd.h.
+ */
+#ifndef LIBRBD_SUPPORTS_IOVEC
     char *bounce;
+#endif
     RBDAIOCmd cmd;
     int error;
     struct BDRVRBDState *s;
@@ -83,7 +88,9 @@ typedef struct RADOSCB {
     RBDAIOCB *acb;
     struct BDRVRBDState *s;
     int64_t size;
+#ifndef LIBRBD_SUPPORTS_IOVEC
     char *buf;
+#endif
     int64_t ret;
 } RADOSCB;
 
@@ -426,11 +433,21 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
         }
     } else {
         if (r < 0) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
             memset(rcb->buf, 0, rcb->size);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov, 0, 0, acb->qiov->size);
+#endif
             acb->ret = r;
             acb->error = 1;
         } else if (r < rcb->size) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
             memset(rcb->buf + r, 0, rcb->size - r);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov,
+                       r, 0, acb->qiov->size - r);
+#endif
+
             if (!acb->error) {
                 acb->ret = rcb->size;
             }
@@ -441,10 +458,12 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
 
     g_free(rcb);
 
+#ifndef LIBRBD_SUPPORTS_IOVEC
     if (acb->cmd == RBD_AIO_READ) {
         qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
     }
     qemu_vfree(acb->bounce);
+#endif
     acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret));
 
     qemu_aio_unref(acb);
@@ -655,8 +674,10 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
     RBDAIOCB *acb;
     RADOSCB *rcb = NULL;
     rbd_completion_t c;
-    char *buf;
     int r;
+#ifndef LIBRBD_SUPPORTS_IOVEC
+    char *buf = NULL;
+#endif
 
     BDRVRBDState *s = bs->opaque;
 
@@ -664,6 +685,8 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
     acb->cmd = cmd;
     acb->qiov = qiov;
     assert(!qiov || qiov->size == size);
+#ifndef LIBRBD_SUPPORTS_IOVEC
+
     if (cmd == RBD_AIO_DISCARD || cmd == RBD_AIO_FLUSH) {
         acb->bounce = NULL;
     } else {
@@ -672,19 +695,21 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
             goto failed;
         }
     }
-    acb->ret = 0;
-    acb->error = 0;
-    acb->s = s;
-
     if (cmd == RBD_AIO_WRITE) {
         qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
     }
-
     buf = acb->bounce;
+#endif
+    acb->ret = 0;
+    acb->error = 0;
+    acb->s = s;
 
     rcb = g_new(RADOSCB, 1);
+
     rcb->acb = acb;
+#ifndef LIBRBD_SUPPORTS_IOVEC
     rcb->buf = buf;
+#endif
     rcb->s = acb->s;
     rcb->size = size;
     r = rbd_aio_create_completion(rcb, (rbd_callback_t) rbd_finish_aiocb, &c);
@@ -694,10 +719,18 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
 
     switch (cmd) {
     case RBD_AIO_WRITE:
+#ifndef LIBRBD_SUPPORTS_IOVEC
         r = rbd_aio_write(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_writev(s->image, qiov->iov, qiov->niov, off, c);
+#endif
         break;
     case RBD_AIO_READ:
+#ifndef LIBRBD_SUPPORTS_IOVEC
         r = rbd_aio_read(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_readv(s->image, qiov->iov, qiov->niov, off, c);
+#endif
         break;
     case RBD_AIO_DISCARD:
         r = rbd_aio_discard_wrapper(s->image, off, size, c);
@@ -719,7 +752,9 @@ failed_completion:
     rbd_aio_release(c);
 failed:
     g_free(rcb);
+#ifndef LIBRBD_SUPPORTS_IOVEC
     qemu_vfree(acb->bounce);
+#endif
     qemu_aio_unref(acb);
     return NULL;
 }
Signature · 1 line
2.10.2
#3 lore raw

RFC v4 1/1 RBD: Add support readv,writev for rbd

From: tianqing <[email protected]>

Rbd can do readv and writev directly, so wo do not need to transform iov to buf or vice versa any more.

---
 block/rbd.c | 47 +++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 41 insertions(+), 6 deletions(-)
block/rbd.c +41 -6
index a57b3e3..0cbd9e3 100644
@@ -73,7 +73,12 @@ typedef struct RBDAIOCB {
     BlockAIOCB common;
     int64_t ret;
     QEMUIOVector *qiov;
+/* Note:
+ * The LIBRBD_SUPPORTS_IOVEC is defined in librbd.h.
+ */
+#ifndef LIBRBD_SUPPORTS_IOVEC
     char *bounce;
+#endif
     RBDAIOCmd cmd;
     int error;
     struct BDRVRBDState *s;
@@ -83,7 +88,9 @@ typedef struct RADOSCB {
     RBDAIOCB *acb;
     struct BDRVRBDState *s;
     int64_t size;
+#ifndef LIBRBD_SUPPORTS_IOVEC
     char *buf;
+#endif
     int64_t ret;
 } RADOSCB;
 
@@ -426,11 +433,21 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
         }
     } else {
         if (r < 0) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
             memset(rcb->buf, 0, rcb->size);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov, 0, 0, acb->qiov->size);
+#endif
             acb->ret = r;
             acb->error = 1;
         } else if (r < rcb->size) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
             memset(rcb->buf + r, 0, rcb->size - r);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov,
+                       r, 0, acb->qiov->size - r);
+#endif
+
             if (!acb->error) {
                 acb->ret = rcb->size;
             }
@@ -441,10 +458,12 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
 
     g_free(rcb);
 
+#ifndef LIBRBD_SUPPORTS_IOVEC
     if (acb->cmd == RBD_AIO_READ) {
         qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
     }
     qemu_vfree(acb->bounce);
+#endif
     acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret));
 
     qemu_aio_unref(acb);
@@ -655,8 +674,10 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
     RBDAIOCB *acb;
     RADOSCB *rcb = NULL;
     rbd_completion_t c;
-    char *buf;
     int r;
+#ifndef LIBRBD_SUPPORTS_IOVEC
+    char *buf = NULL;
+#endif
 
     BDRVRBDState *s = bs->opaque;
 
@@ -664,6 +685,8 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
     acb->cmd = cmd;
     acb->qiov = qiov;
     assert(!qiov || qiov->size == size);
+#ifndef LIBRBD_SUPPORTS_IOVEC
+
     if (cmd == RBD_AIO_DISCARD || cmd == RBD_AIO_FLUSH) {
         acb->bounce = NULL;
     } else {
@@ -672,19 +695,21 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
             goto failed;
         }
     }
-    acb->ret = 0;
-    acb->error = 0;
-    acb->s = s;
-
     if (cmd == RBD_AIO_WRITE) {
         qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
     }
-
     buf = acb->bounce;
+#endif
+    acb->ret = 0;
+    acb->error = 0;
+    acb->s = s;
 
     rcb = g_new(RADOSCB, 1);
+
     rcb->acb = acb;
+#ifndef LIBRBD_SUPPORTS_IOVEC
     rcb->buf = buf;
+#endif
     rcb->s = acb->s;
     rcb->size = size;
     r = rbd_aio_create_completion(rcb, (rbd_callback_t) rbd_finish_aiocb, &c);
@@ -694,10 +719,18 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
 
     switch (cmd) {
     case RBD_AIO_WRITE:
+#ifndef LIBRBD_SUPPORTS_IOVEC
         r = rbd_aio_write(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_writev(s->image, qiov->iov, qiov->niov, off, c);
+#endif
         break;
     case RBD_AIO_READ:
+#ifndef LIBRBD_SUPPORTS_IOVEC
         r = rbd_aio_read(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_readv(s->image, qiov->iov, qiov->niov, off, c);
+#endif
         break;
     case RBD_AIO_DISCARD:
         r = rbd_aio_discard_wrapper(s->image, off, size, c);
@@ -719,7 +752,9 @@ failed_completion:
     rbd_aio_release(c);
 failed:
     g_free(rcb);
+#ifndef LIBRBD_SUPPORTS_IOVEC
     qemu_vfree(acb->bounce);
+#endif
     qemu_aio_unref(acb);
     return NULL;
 }
Signature · 1 line
2.10.2
#4 lore raw

RFC v5 1/1 RBD: Add support readv,writev for rbd

From: tianqing <[email protected]>

Rbd can do readv and writev directly, so wo do not need to transform iov to buf or vice versa any more.

---
 block/rbd.c | 49 ++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 42 insertions(+), 7 deletions(-)
block/rbd.c +42 -7
index a57b3e3..75ae1d6 100644
@@ -47,7 +47,7 @@
  */
 
 /* rbd_aio_discard added in 0.1.2 */
-#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(0, 1, 2)
+#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(12, 0, 0)
 #define LIBRBD_SUPPORTS_DISCARD
 #else
 #undef LIBRBD_SUPPORTS_DISCARD
@@ -73,7 +73,12 @@ typedef struct RBDAIOCB {
     BlockAIOCB common;
     int64_t ret;
     QEMUIOVector *qiov;
+/* Note:
+ * The LIBRBD_SUPPORTS_IOVEC is defined in librbd.h.
+ */
+#ifndef LIBRBD_SUPPORTS_IOVEC
     char *bounce;
+#endif
     RBDAIOCmd cmd;
     int error;
     struct BDRVRBDState *s;
@@ -83,7 +88,9 @@ typedef struct RADOSCB {
     RBDAIOCB *acb;
     struct BDRVRBDState *s;
     int64_t size;
+#ifndef LIBRBD_SUPPORTS_IOVEC
     char *buf;
+#endif
     int64_t ret;
 } RADOSCB;
 
@@ -426,11 +433,21 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
         }
     } else {
         if (r < 0) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
             memset(rcb->buf, 0, rcb->size);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov, 0, 0, acb->qiov->size);
+#endif
             acb->ret = r;
             acb->error = 1;
         } else if (r < rcb->size) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
             memset(rcb->buf + r, 0, rcb->size - r);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov,
+                       r, 0, acb->qiov->size - r);
+#endif
+
             if (!acb->error) {
                 acb->ret = rcb->size;
             }
@@ -441,10 +458,12 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
 
     g_free(rcb);
 
+#ifndef LIBRBD_SUPPORTS_IOVEC
     if (acb->cmd == RBD_AIO_READ) {
         qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
     }
     qemu_vfree(acb->bounce);
+#endif
     acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret));
 
     qemu_aio_unref(acb);
@@ -655,8 +674,10 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
     RBDAIOCB *acb;
     RADOSCB *rcb = NULL;
     rbd_completion_t c;
-    char *buf;
     int r;
+#ifndef LIBRBD_SUPPORTS_IOVEC
+    char *buf = NULL;
+#endif
 
     BDRVRBDState *s = bs->opaque;
 
@@ -664,6 +685,8 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
     acb->cmd = cmd;
     acb->qiov = qiov;
     assert(!qiov || qiov->size == size);
+#ifndef LIBRBD_SUPPORTS_IOVEC
+
     if (cmd == RBD_AIO_DISCARD || cmd == RBD_AIO_FLUSH) {
         acb->bounce = NULL;
     } else {
@@ -672,19 +695,21 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
             goto failed;
         }
     }
-    acb->ret = 0;
-    acb->error = 0;
-    acb->s = s;
-
     if (cmd == RBD_AIO_WRITE) {
         qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
     }
-
     buf = acb->bounce;
+#endif
+    acb->ret = 0;
+    acb->error = 0;
+    acb->s = s;
 
     rcb = g_new(RADOSCB, 1);
+
     rcb->acb = acb;
+#ifndef LIBRBD_SUPPORTS_IOVEC
     rcb->buf = buf;
+#endif
     rcb->s = acb->s;
     rcb->size = size;
     r = rbd_aio_create_completion(rcb, (rbd_callback_t) rbd_finish_aiocb, &c);
@@ -694,10 +719,18 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
 
     switch (cmd) {
     case RBD_AIO_WRITE:
+#ifndef LIBRBD_SUPPORTS_IOVEC
         r = rbd_aio_write(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_writev(s->image, qiov->iov, qiov->niov, off, c);
+#endif
         break;
     case RBD_AIO_READ:
+#ifndef LIBRBD_SUPPORTS_IOVEC
         r = rbd_aio_read(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_readv(s->image, qiov->iov, qiov->niov, off, c);
+#endif
         break;
     case RBD_AIO_DISCARD:
         r = rbd_aio_discard_wrapper(s->image, off, size, c);
@@ -719,7 +752,9 @@ failed_completion:
     rbd_aio_release(c);
 failed:
     g_free(rcb);
+#ifndef LIBRBD_SUPPORTS_IOVEC
     qemu_vfree(acb->bounce);
+#endif
     qemu_aio_unref(acb);
     return NULL;
 }
Signature · 1 line
2.10.2
#5 lore raw

Tianqing,

Do we have any performance data for this patch? Thanks.

Tiger

Quoted 167 lines · 在 2017年2月16日,下午5:00,[email protected] 写道:

From: tianqing <[email protected]>

Rbd can do readv and writev directly, so wo do not need to transform iov to buf or vice versa any more.

---
block/rbd.c | 49 ++++++++++++++++++++++++++++++++++++++++++-------

1 file changed, 42 insertions(+), 7 deletions(-)
block/rbd.c +40 -7
index a57b3e3..75ae1d6 100644
@@ -47,7 +47,7 @@
 */
 
 /* rbd_aio_discard added in 0.1.2 */
-#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(0, 1, 2)
+#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(12, 0, 0)
 #define LIBRBD_SUPPORTS_DISCARD
 #else
 #undef LIBRBD_SUPPORTS_DISCARD
@@ -73,7 +73,12 @@ typedef struct RBDAIOCB {
    BlockAIOCB common;
    int64_t ret;
    QEMUIOVector *qiov;
+/* Note:
+ * The LIBRBD_SUPPORTS_IOVEC is defined in librbd.h.
+ */
+#ifndef LIBRBD_SUPPORTS_IOVEC
    char *bounce;
+#endif
    RBDAIOCmd cmd;
    int error;
    struct BDRVRBDState *s;
@@ -83,7 +88,9 @@ typedef struct RADOSCB {
    RBDAIOCB *acb;
    struct BDRVRBDState *s;
    int64_t size;
+#ifndef LIBRBD_SUPPORTS_IOVEC
    char *buf;
+#endif
    int64_t ret;
 } RADOSCB;
 
@@ -426,11 +433,21 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
        }
    } else {
        if (r < 0) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
            memset(rcb->buf, 0, rcb->size);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov, 0, 0, acb->qiov->size);
+#endif
            acb->ret = r;
            acb->error = 1;
        } else if (r < rcb->size) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
            memset(rcb->buf + r, 0, rcb->size - r);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov,
+                       r, 0, acb->qiov->size - r);
+#endif
+
            if (!acb->error) {
                acb->ret = rcb->size;
            }
@@ -441,10 +458,12 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
 
    g_free(rcb);
 
+#ifndef LIBRBD_SUPPORTS_IOVEC
    if (acb->cmd == RBD_AIO_READ) {
        qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
    }
    qemu_vfree(acb->bounce);
+#endif
    acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret));
 
    qemu_aio_unref(acb);
@@ -655,8 +674,10 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
    RBDAIOCB *acb;
    RADOSCB *rcb = NULL;
    rbd_completion_t c;
-    char *buf;
    int r;
+#ifndef LIBRBD_SUPPORTS_IOVEC
+    char *buf = NULL;
+#endif
 
    BDRVRBDState *s = bs->opaque;
 
@@ -664,6 +685,8 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
    acb->cmd = cmd;
    acb->qiov = qiov;
    assert(!qiov || qiov->size == size);
+#ifndef LIBRBD_SUPPORTS_IOVEC
+
    if (cmd == RBD_AIO_DISCARD || cmd == RBD_AIO_FLUSH) {
        acb->bounce = NULL;
    } else {
@@ -672,19 +695,21 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
            goto failed;
        }
    }
-    acb->ret = 0;
-    acb->error = 0;
-    acb->s = s;
-
    if (cmd == RBD_AIO_WRITE) {
        qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
    }
-
    buf = acb->bounce;
+#endif
+    acb->ret = 0;
+    acb->error = 0;
+    acb->s = s;
 
    rcb = g_new(RADOSCB, 1);
+
    rcb->acb = acb;
+#ifndef LIBRBD_SUPPORTS_IOVEC
    rcb->buf = buf;
+#endif
    rcb->s = acb->s;
    rcb->size = size;
    r = rbd_aio_create_completion(rcb, (rbd_callback_t) rbd_finish_aiocb, &c);
@@ -694,10 +719,18 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
 
    switch (cmd) {
    case RBD_AIO_WRITE:
+#ifndef LIBRBD_SUPPORTS_IOVEC
        r = rbd_aio_write(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_writev(s->image, qiov->iov, qiov->niov, off, c);
+#endif
        break;
    case RBD_AIO_READ:
+#ifndef LIBRBD_SUPPORTS_IOVEC
        r = rbd_aio_read(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_readv(s->image, qiov->iov, qiov->niov, off, c);
+#endif
        break;
    case RBD_AIO_DISCARD:
        r = rbd_aio_discard_wrapper(s->image, off, size, c);
@@ -719,7 +752,9 @@ failed_completion:
    rbd_aio_release(c);

failed:

    g_free(rcb);
block/rbd.c +2 -0
+#ifndef LIBRBD_SUPPORTS_IOVEC
    qemu_vfree(acb->bounce);
+#endif
    qemu_aio_unref(acb);
    return NULL;

}

Signature · 6 lines
2.10.2

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
author
#6 lore raw

No yet. I just test on one qemu-kvm vm. It works fine. The performance may need more time.
Any one can test on this patch if you do fast....

2017-02-16 20:07 GMT+08:00 Tiger Hu <[email protected]>:

Quoted 173 lines

Tianqing,

Do we have any performance data for this patch? Thanks.

Tiger

Quoted 167 lines · 在 2017年2月16日,下午5:00,[email protected] 写道:

From: tianqing <[email protected]>

Rbd can do readv and writev directly, so wo do not need to transform iov to buf or vice versa any more.

---
block/rbd.c | 49 ++++++++++++++++++++++++++++++++++++++++++-------

1 file changed, 42 insertions(+), 7 deletions(-)
block/rbd.c +40 -7
index a57b3e3..75ae1d6 100644
@@ -47,7 +47,7 @@
 */
 
 /* rbd_aio_discard added in 0.1.2 */
-#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(0, 1, 2)
+#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(12, 0, 0)
 #define LIBRBD_SUPPORTS_DISCARD
 #else
 #undef LIBRBD_SUPPORTS_DISCARD
@@ -73,7 +73,12 @@ typedef struct RBDAIOCB {
    BlockAIOCB common;
    int64_t ret;
    QEMUIOVector *qiov;
+/* Note:
+ * The LIBRBD_SUPPORTS_IOVEC is defined in librbd.h.
+ */
+#ifndef LIBRBD_SUPPORTS_IOVEC
    char *bounce;
+#endif
    RBDAIOCmd cmd;
    int error;
    struct BDRVRBDState *s;
@@ -83,7 +88,9 @@ typedef struct RADOSCB {
    RBDAIOCB *acb;
    struct BDRVRBDState *s;
    int64_t size;
+#ifndef LIBRBD_SUPPORTS_IOVEC
    char *buf;
+#endif
    int64_t ret;
 } RADOSCB;
 
@@ -426,11 +433,21 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
        }
    } else {
        if (r < 0) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
            memset(rcb->buf, 0, rcb->size);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov, 0, 0, acb->qiov->size);
+#endif
            acb->ret = r;
            acb->error = 1;
        } else if (r < rcb->size) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
            memset(rcb->buf + r, 0, rcb->size - r);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov,
+                       r, 0, acb->qiov->size - r);
+#endif
+
            if (!acb->error) {
                acb->ret = rcb->size;
            }
@@ -441,10 +458,12 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
 
    g_free(rcb);
 
+#ifndef LIBRBD_SUPPORTS_IOVEC
    if (acb->cmd == RBD_AIO_READ) {
        qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
    }
    qemu_vfree(acb->bounce);
+#endif
    acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret));
 
    qemu_aio_unref(acb);
@@ -655,8 +674,10 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
    RBDAIOCB *acb;
    RADOSCB *rcb = NULL;
    rbd_completion_t c;
-    char *buf;
    int r;
+#ifndef LIBRBD_SUPPORTS_IOVEC
+    char *buf = NULL;
+#endif
 
    BDRVRBDState *s = bs->opaque;
 
@@ -664,6 +685,8 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
    acb->cmd = cmd;
    acb->qiov = qiov;
    assert(!qiov || qiov->size == size);
+#ifndef LIBRBD_SUPPORTS_IOVEC
+
    if (cmd == RBD_AIO_DISCARD || cmd == RBD_AIO_FLUSH) {
        acb->bounce = NULL;
    } else {
@@ -672,19 +695,21 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
            goto failed;
        }
    }
-    acb->ret = 0;
-    acb->error = 0;
-    acb->s = s;
-
    if (cmd == RBD_AIO_WRITE) {
        qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
    }
-
    buf = acb->bounce;
+#endif
+    acb->ret = 0;
+    acb->error = 0;
+    acb->s = s;
 
    rcb = g_new(RADOSCB, 1);
+
    rcb->acb = acb;
+#ifndef LIBRBD_SUPPORTS_IOVEC
    rcb->buf = buf;
+#endif
    rcb->s = acb->s;
    rcb->size = size;
    r = rbd_aio_create_completion(rcb, (rbd_callback_t) rbd_finish_aiocb, &c);
@@ -694,10 +719,18 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
 
    switch (cmd) {
    case RBD_AIO_WRITE:
+#ifndef LIBRBD_SUPPORTS_IOVEC
        r = rbd_aio_write(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_writev(s->image, qiov->iov, qiov->niov, off, c);
+#endif
        break;
    case RBD_AIO_READ:
+#ifndef LIBRBD_SUPPORTS_IOVEC
        r = rbd_aio_read(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_readv(s->image, qiov->iov, qiov->niov, off, c);
+#endif
        break;
    case RBD_AIO_DISCARD:
        r = rbd_aio_discard_wrapper(s->image, off, size, c);
@@ -719,7 +752,9 @@ failed_completion:
    rbd_aio_release(c);

failed:

    g_free(rcb);
block/rbd.c +2 -0
+#ifndef LIBRBD_SUPPORTS_IOVEC
    qemu_vfree(acb->bounce);
+#endif
    qemu_aio_unref(acb);
    return NULL;

}
--
2.10.2

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Signature · 1 line
谦谦君子
#7 lore raw
Quoted 3 lines
Quoted 3 lines

No yet. I just test on one qemu-kvm vm. It works fine. The performance may need more time.
Any one can test on this patch if you do fast....

Hi, I would like to bench it with small 4k read/write.

On the ceph side,do we need this PR ? :
https://github.com/ceph/ceph/pull/13447

----- Mail original -----
De: "jazeltq" <[email protected]>
À: "Tiger Hu" <[email protected]>
Cc: "Josh Durgin" <[email protected]>, "Jeff Cody" <[email protected]>, "dillaman" <[email protected]>, "Kevin Wolf" <[email protected]>, [email protected], [email protected], "qemu-devel" <[email protected]>, "ceph-devel" <[email protected]>, "tianqing" <[email protected]> Envoyé: Jeudi 16 Février 2017 15:03:52
Objet: Re: [RFC v5] RBD: Add support readv,writev for rbd

No yet. I just test on one qemu-kvm vm. It works fine. The performance may need more time.
Any one can test on this patch if you do fast....

2017-02-16 20:07 GMT+08:00 Tiger Hu <[email protected]>:

Quoted 173 lines

Tianqing,

Do we have any performance data for this patch? Thanks.

Tiger

Quoted 167 lines · 在 2017年2月16日,下午5:00,[email protected] 写道:

From: tianqing <[email protected]>

Rbd can do readv and writev directly, so wo do not need to transform iov to buf or vice versa any more.

---
block/rbd.c | 49 ++++++++++++++++++++++++++++++++++++++++++-------

1 file changed, 42 insertions(+), 7 deletions(-) 
block/rbd.c +32 -7
index a57b3e3..75ae1d6 100644 
@@ -47,7 +47,7 @@ 
 */ 
 
 /* rbd_aio_discard added in 0.1.2 */ 
-#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(0, 1, 2) 
+#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(12, 0, 0) 
 #define LIBRBD_SUPPORTS_DISCARD 
 #else 
 #undef LIBRBD_SUPPORTS_DISCARD 
@@ -73,7 +73,12 @@ typedef struct RBDAIOCB { 
 BlockAIOCB common; 
 int64_t ret; 
 QEMUIOVector *qiov; 
+/* Note: 
+ * The LIBRBD_SUPPORTS_IOVEC is defined in librbd.h. 
+ */ 
+#ifndef LIBRBD_SUPPORTS_IOVEC 
 char *bounce; 
+#endif 
 RBDAIOCmd cmd; 
 int error; 
 struct BDRVRBDState *s; 
@@ -83,7 +88,9 @@ typedef struct RADOSCB { 
 RBDAIOCB *acb; 
 struct BDRVRBDState *s; 
 int64_t size; 
+#ifndef LIBRBD_SUPPORTS_IOVEC 
 char *buf; 
+#endif 
 int64_t ret; 
 } RADOSCB; 
 
@@ -426,11 +433,21 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb) 
 } 
 } else { 
 if (r < 0) { 
+#ifndef LIBRBD_SUPPORTS_IOVEC 
 memset(rcb->buf, 0, rcb->size); 
+#else 
+ iov_memset(acb->qiov->iov, acb->qiov->niov, 0, 0, acb->qiov->size); 
+#endif 
 acb->ret = r; 
 acb->error = 1; 
 } else if (r < rcb->size) { 
+#ifndef LIBRBD_SUPPORTS_IOVEC 
 memset(rcb->buf + r, 0, rcb->size - r); 
+#else 
+ iov_memset(acb->qiov->iov, acb->qiov->niov, 
+ r, 0, acb->qiov->size - r); 
+#endif 
+ 
 if (!acb->error) { 
 acb->ret = rcb->size; 
 } 
@@ -441,10 +458,12 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb) 
 
 g_free(rcb); 
 
+#ifndef LIBRBD_SUPPORTS_IOVEC 
 if (acb->cmd == RBD_AIO_READ) { 
 qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size); 
 } 
 qemu_vfree(acb->bounce); 
+#endif 
 acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret)); 
 
 qemu_aio_unref(acb); 
@@ -655,8 +674,10 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs, 
 RBDAIOCB *acb; 
 RADOSCB *rcb = NULL; 
 rbd_completion_t c; 
- char *buf; 
 int r; 
+#ifndef LIBRBD_SUPPORTS_IOVEC 
+ char *buf = NULL; 
+#endif 
 
 BDRVRBDState *s = bs->opaque; 
 
@@ -664,6 +685,8 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs, 
 acb->cmd = cmd; 
 acb->qiov = qiov; 
 assert(!qiov || qiov->size == size); 
+#ifndef LIBRBD_SUPPORTS_IOVEC 
+ 
 if (cmd == RBD_AIO_DISCARD || cmd == RBD_AIO_FLUSH) { 
 acb->bounce = NULL; 
 } else { 
@@ -672,19 +695,21 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs, 
 goto failed; 
 } 
 } 
- acb->ret = 0; 
- acb->error = 0; 
- acb->s = s; 
- 
 if (cmd == RBD_AIO_WRITE) { 
 qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size); 
 } 
- 
 buf = acb->bounce; 
+#endif 
+ acb->ret = 0; 
+ acb->error = 0; 
+ acb->s = s; 
 
 rcb = g_new(RADOSCB, 1); 
+ 
 rcb->acb = acb; 
+#ifndef LIBRBD_SUPPORTS_IOVEC 
 rcb->buf = buf; 
+#endif 
 rcb->s = acb->s; 
 rcb->size = size; 
 r = rbd_aio_create_completion(rcb, (rbd_callback_t) rbd_finish_aiocb, &c); 
@@ -694,10 +719,18 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs, 
 
 switch (cmd) { 

case RBD_AIO_WRITE:
+#ifndef LIBRBD_SUPPORTS_IOVEC
r = rbd_aio_write(s->image, off, size, buf, c);

block/rbd.c +3 -0
+#else 
+ r = rbd_aio_writev(s->image, qiov->iov, qiov->niov, off, c); 
+#endif 

break;
case RBD_AIO_READ:
+#ifndef LIBRBD_SUPPORTS_IOVEC
r = rbd_aio_read(s->image, off, size, buf, c);

block/rbd.c +3 -0
+#else 
+ r = rbd_aio_readv(s->image, qiov->iov, qiov->niov, off, c); 
+#endif 

break;
case RBD_AIO_DISCARD:
r = rbd_aio_discard_wrapper(s->image, off, size, c);

block/rbd.c +0 -0
@@ -719,7 +752,9 @@ failed_completion: 
 rbd_aio_release(c); 

failed:
g_free(rcb);
+#ifndef LIBRBD_SUPPORTS_IOVEC
qemu_vfree(acb->bounce);
+#endif
qemu_aio_unref(acb);
return NULL;
}

Signature · 6 lines
2.10.2 

-- 
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in 
the body of a message to [email protected] 
More majordomo info at http://vger.kernel.org/majordomo-info.html 
Signature · 5 lines
谦谦君子 
-- 
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in 
the body of a message to [email protected] 
More majordomo info at http://vger.kernel.org/majordomo-info.html 
#8 lore raw
Quoted 4 lines · Alexandre DERUMIER

Hi, I would like to bench it with small 4k read/write.

On the ceph side,do we need this PR ? :
https://github.com/ceph/ceph/pull/13447

Yes, that is the correct PR for the client-side librbd changes. You should be able to test it against Hammer/Jewel-release clusters.

Signature · 1 line
Jason
#9 lore raw
In reply to [email protected]
Quoted 15 lines · On Thu, Feb 16, 2017 at 4:00 AM, <[email protected]> wrote:

From: tianqing <[email protected]>

Rbd can do readv and writev directly, so wo do not need to transform iov to buf or vice versa any more.

---
 block/rbd.c | 49 ++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 42 insertions(+), 7 deletions(-)
block/rbd.c +0 -0
index a57b3e3..75ae1d6 100644

block/rbd.c line 53

Quoted 8 lines
block/rbd.c +1 -1
@@ -47,7 +47,7 @@
  */
 
 /* rbd_aio_discard added in 0.1.2 */
-#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(0, 1, 2)
+#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(12, 0, 0)
 #define LIBRBD_SUPPORTS_DISCARD
 #else
 #undef LIBRBD_SUPPORTS_DISCARD

Do not change this -- discard support is available in very old versions of librbd not just the future Luminous release.

Quoted 138 lines
block/rbd.c +41 -7
@@ -73,7 +73,12 @@ typedef struct RBDAIOCB {
     BlockAIOCB common;
     int64_t ret;
     QEMUIOVector *qiov;
+/* Note:
+ * The LIBRBD_SUPPORTS_IOVEC is defined in librbd.h.
+ */
+#ifndef LIBRBD_SUPPORTS_IOVEC
     char *bounce;
+#endif
     RBDAIOCmd cmd;
     int error;
     struct BDRVRBDState *s;
@@ -83,7 +88,9 @@ typedef struct RADOSCB {
     RBDAIOCB *acb;
     struct BDRVRBDState *s;
     int64_t size;
+#ifndef LIBRBD_SUPPORTS_IOVEC
     char *buf;
+#endif
     int64_t ret;
 } RADOSCB;
 
@@ -426,11 +433,21 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
         }
     } else {
         if (r < 0) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
             memset(rcb->buf, 0, rcb->size);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov, 0, 0, acb->qiov->size);
+#endif
             acb->ret = r;
             acb->error = 1;
         } else if (r < rcb->size) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
             memset(rcb->buf + r, 0, rcb->size - r);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov,
+                       r, 0, acb->qiov->size - r);
+#endif
+
             if (!acb->error) {
                 acb->ret = rcb->size;
             }
@@ -441,10 +458,12 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
 
     g_free(rcb);
 
+#ifndef LIBRBD_SUPPORTS_IOVEC
     if (acb->cmd == RBD_AIO_READ) {
         qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
     }
     qemu_vfree(acb->bounce);
+#endif
     acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret));
 
     qemu_aio_unref(acb);
@@ -655,8 +674,10 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
     RBDAIOCB *acb;
     RADOSCB *rcb = NULL;
     rbd_completion_t c;
-    char *buf;
     int r;
+#ifndef LIBRBD_SUPPORTS_IOVEC
+    char *buf = NULL;
+#endif
 
     BDRVRBDState *s = bs->opaque;
 
@@ -664,6 +685,8 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
     acb->cmd = cmd;
     acb->qiov = qiov;
     assert(!qiov || qiov->size == size);
+#ifndef LIBRBD_SUPPORTS_IOVEC
+
     if (cmd == RBD_AIO_DISCARD || cmd == RBD_AIO_FLUSH) {
         acb->bounce = NULL;
     } else {
@@ -672,19 +695,21 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
             goto failed;
         }
     }
-    acb->ret = 0;
-    acb->error = 0;
-    acb->s = s;
-
     if (cmd == RBD_AIO_WRITE) {
         qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
     }
-
     buf = acb->bounce;
+#endif
+    acb->ret = 0;
+    acb->error = 0;
+    acb->s = s;
 
     rcb = g_new(RADOSCB, 1);
+
     rcb->acb = acb;
+#ifndef LIBRBD_SUPPORTS_IOVEC
     rcb->buf = buf;
+#endif
     rcb->s = acb->s;
     rcb->size = size;
     r = rbd_aio_create_completion(rcb, (rbd_callback_t) rbd_finish_aiocb, &c);
@@ -694,10 +719,18 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
 
     switch (cmd) {
     case RBD_AIO_WRITE:
+#ifndef LIBRBD_SUPPORTS_IOVEC
         r = rbd_aio_write(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_writev(s->image, qiov->iov, qiov->niov, off, c);
+#endif
         break;
     case RBD_AIO_READ:
+#ifndef LIBRBD_SUPPORTS_IOVEC
         r = rbd_aio_read(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_readv(s->image, qiov->iov, qiov->niov, off, c);
+#endif
         break;
     case RBD_AIO_DISCARD:
         r = rbd_aio_discard_wrapper(s->image, off, size, c);
@@ -719,7 +752,9 @@ failed_completion:
     rbd_aio_release(c);
 failed:
     g_free(rcb);
+#ifndef LIBRBD_SUPPORTS_IOVEC
     qemu_vfree(acb->bounce);
+#endif
     qemu_aio_unref(acb);
     return NULL;
 }
--

2.10.2

Signature · 1 line
Jason
author
#10 lore raw

2017-02-16 22:14 GMT+08:00 Jason Dillaman <[email protected]>:

Quoted 18 lines · On Thu, Feb 16, 2017 at 4:00 AM, <[email protected]> wrote:
Quoted 17 lines

From: tianqing <[email protected]>

Rbd can do readv and writev directly, so wo do not need to transform iov to buf or vice versa any more.

---
 block/rbd.c | 49 ++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 42 insertions(+), 7 deletions(-)
block/rbd.c +0 -0
index a57b3e3..75ae1d6 100644
@@ -47,7 +47,7 @@
  */
 
Quoted 9 lines

block/rbd.c line 53

Quoted 6 lines
block/rbd.c +1 -1
 /* rbd_aio_discard added in 0.1.2 */
-#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(0, 1, 2)
+#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(12, 0, 0)
 #define LIBRBD_SUPPORTS_DISCARD
 #else
 #undef LIBRBD_SUPPORTS_DISCARD

Do not change this -- discard support is available in very old versions of librbd not just the future Luminous release.

Ok. Thanks a lot.

Quoted 144 lines
Quoted 137 lines
block/rbd.c +41 -7
@@ -73,7 +73,12 @@ typedef struct RBDAIOCB {
     BlockAIOCB common;
     int64_t ret;
     QEMUIOVector *qiov;
+/* Note:
+ * The LIBRBD_SUPPORTS_IOVEC is defined in librbd.h.
+ */
+#ifndef LIBRBD_SUPPORTS_IOVEC
     char *bounce;
+#endif
     RBDAIOCmd cmd;
     int error;
     struct BDRVRBDState *s;
@@ -83,7 +88,9 @@ typedef struct RADOSCB {
     RBDAIOCB *acb;
     struct BDRVRBDState *s;
     int64_t size;
+#ifndef LIBRBD_SUPPORTS_IOVEC
     char *buf;
+#endif
     int64_t ret;
 } RADOSCB;
 
@@ -426,11 +433,21 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
         }
     } else {
         if (r < 0) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
             memset(rcb->buf, 0, rcb->size);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov, 0, 0, acb->qiov->size);
+#endif
             acb->ret = r;
             acb->error = 1;
         } else if (r < rcb->size) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
             memset(rcb->buf + r, 0, rcb->size - r);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov,
+                       r, 0, acb->qiov->size - r);
+#endif
+
             if (!acb->error) {
                 acb->ret = rcb->size;
             }
@@ -441,10 +458,12 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
 
     g_free(rcb);
 
+#ifndef LIBRBD_SUPPORTS_IOVEC
     if (acb->cmd == RBD_AIO_READ) {
         qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
     }
     qemu_vfree(acb->bounce);
+#endif
     acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret));
 
     qemu_aio_unref(acb);
@@ -655,8 +674,10 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
     RBDAIOCB *acb;
     RADOSCB *rcb = NULL;
     rbd_completion_t c;
-    char *buf;
     int r;
+#ifndef LIBRBD_SUPPORTS_IOVEC
+    char *buf = NULL;
+#endif
 
     BDRVRBDState *s = bs->opaque;
 
@@ -664,6 +685,8 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
     acb->cmd = cmd;
     acb->qiov = qiov;
     assert(!qiov || qiov->size == size);
+#ifndef LIBRBD_SUPPORTS_IOVEC
+
     if (cmd == RBD_AIO_DISCARD || cmd == RBD_AIO_FLUSH) {
         acb->bounce = NULL;
     } else {
@@ -672,19 +695,21 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
             goto failed;
         }
     }
-    acb->ret = 0;
-    acb->error = 0;
-    acb->s = s;
-
     if (cmd == RBD_AIO_WRITE) {
         qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
     }
-
     buf = acb->bounce;
+#endif
+    acb->ret = 0;
+    acb->error = 0;
+    acb->s = s;
 
     rcb = g_new(RADOSCB, 1);
+
     rcb->acb = acb;
+#ifndef LIBRBD_SUPPORTS_IOVEC
     rcb->buf = buf;
+#endif
     rcb->s = acb->s;
     rcb->size = size;
     r = rbd_aio_create_completion(rcb, (rbd_callback_t) rbd_finish_aiocb, &c);
@@ -694,10 +719,18 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
 
     switch (cmd) {
     case RBD_AIO_WRITE:
+#ifndef LIBRBD_SUPPORTS_IOVEC
         r = rbd_aio_write(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_writev(s->image, qiov->iov, qiov->niov, off, c);
+#endif
         break;
     case RBD_AIO_READ:
+#ifndef LIBRBD_SUPPORTS_IOVEC
         r = rbd_aio_read(s->image, off, size, buf, c);
+#else
+        r = rbd_aio_readv(s->image, qiov->iov, qiov->niov, off, c);
+#endif
         break;
     case RBD_AIO_DISCARD:
         r = rbd_aio_discard_wrapper(s->image, off, size, c);
@@ -719,7 +752,9 @@ failed_completion:
     rbd_aio_release(c);
 failed:
     g_free(rcb);
+#ifndef LIBRBD_SUPPORTS_IOVEC
     qemu_vfree(acb->bounce);
+#endif
     qemu_aio_unref(acb);
     return NULL;
 }
--
Quoted 2 lines

2.10.2

--
Jason

Signature · 1 line
谦谦君子
#11 lore raw
In reply to [email protected]
Quoted 4 lines · On 02/16/2017 03:00 AM, [email protected] wrote:

From: tianqing <[email protected]>

Rbd can do readv and writev directly, so wo do not need to transform iov to buf or vice versa any more.

In general, we prefer new revisions of a patch series to be sent as a new top-level thread, rather than buried in-reply-to the earlier revision, as nesting can mess up some of the automated tooling that tests whether a patch will introduce problems.

Quoted 11 lines
---
 block/rbd.c | 49 ++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 42 insertions(+), 7 deletions(-)
block/rbd.c +0 -0
index a57b3e3..75ae1d6 100644

block/rbd.c line 50

Quoted 5 lines
block/rbd.c +1 -1
@@ -47,7 +47,7 @@
  */
 
 /* rbd_aio_discard added in 0.1.2 */
-#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(0, 1, 2)
+#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(12, 0, 0)

Does the comment need updating, too?

Signature · 2 lines
Eric Blake   eblake redhat com    +1-919-301-3266
Libvirt virtualization library http://libvirt.org
#12 lore raw
In reply to [email protected]
Quoted 24 lines · On Thu, Feb 16, 2017 at 05:00:02PM +0800, [email protected] wrote:

From: tianqing <[email protected]>

Rbd can do readv and writev directly, so wo do not need to transform iov to buf or vice versa any more.

---
 block/rbd.c | 49 ++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 42 insertions(+), 7 deletions(-)
block/rbd.c +1 -1
index a57b3e3..75ae1d6 100644
@@ -47,7 +47,7 @@
  */
 
 /* rbd_aio_discard added in 0.1.2 */
-#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(0, 1, 2)
+#if LIBRBD_VERSION_CODE >= LIBRBD_VERSION(12, 0, 0)
 #define LIBRBD_SUPPORTS_DISCARD
 #else
 #undef LIBRBD_SUPPORTS_DISCARD

block/rbd.c typedef struct RBDAIOCB { · line 78

Quoted 6 lines
block/rbd.c +3 -0
@@ -73,7 +73,12 @@ typedef struct RBDAIOCB {
     BlockAIOCB common;
     int64_t ret;
     QEMUIOVector *qiov;
+/* Note:
+ * The LIBRBD_SUPPORTS_IOVEC is defined in librbd.h.
+ */

I understand that since this feature relies on new symbols in the library, the ifdef can't be entirely avoided. However, there are a few changes that can make the code much more readable, and greatly reduce the use of ifdefs through the code.

For instance, you could have helper constants:

#ifdef LIBRBD_SUPPORTS_IOVEC
#define LIBRBD_USE_IOVEC 1
#else
#define LIBRBD_USE_IOVEC 0
#endif

Now that can be used for a more natural code, with minimal ifdefs. The compiler optimizations should still remove the code, as well.

A helper function for memset will also keep things simpler:

static void qemu_rbd_memset(RADOSCB *rcb, int64_t offs)

{
    if (LIBRBD_USE_IOVEC) {
        RBDAIOCB *acb = rcb->acb;
        iov_memset(acb->qiov->iov, acb->qiov->niov, offs, 0,
                   acb->qiov->size - offs);
    } else {
        memset(rcb->buf + offs, 0, rcb->size - offs);
    }
}

block/rbd.c typedef struct RBDAIOCB { · line 81

Quoted 3 lines
block/rbd.c +2 -0
+#ifndef LIBRBD_SUPPORTS_IOVEC
     char *bounce;
+#endif

This conditional can go away (see [1]).

block/rbd.c typedef struct RADOSCB { · line 93

Quoted 10 lines
block/rbd.c +2 -0
     RBDAIOCmd cmd;
     int error;
     struct BDRVRBDState *s;
@@ -83,7 +88,9 @@ typedef struct RADOSCB {
     RBDAIOCB *acb;
     struct BDRVRBDState *s;
     int64_t size;
+#ifndef LIBRBD_SUPPORTS_IOVEC
     char *buf;
+#endif

As can this.

block/rbd.c static void qemu_rbd_complete_aio(RADOSCB *rcb) · line 440

Quoted 12 lines
block/rbd.c +4 -0
     int64_t ret;
 } RADOSCB;
 
@@ -426,11 +433,21 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
         }
     } else {
         if (r < 0) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
             memset(rcb->buf, 0, rcb->size);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov, 0, 0, acb->qiov->size);
+#endif

This just becomes:

qemu_rbd_memset(rcb, 0);

block/rbd.c static void qemu_rbd_complete_aio(RADOSCB *rcb) · line 450

Quoted 10 lines
block/rbd.c +6 -0
             acb->ret = r;
             acb->error = 1;
         } else if (r < rcb->size) {
+#ifndef LIBRBD_SUPPORTS_IOVEC
             memset(rcb->buf + r, 0, rcb->size - r);
+#else
+            iov_memset(acb->qiov->iov, acb->qiov->niov,
+                       r, 0, acb->qiov->size - r);
+#endif
+

Same here:

qemu_rbd_memset(rcb, r);

block/rbd.c static void qemu_rbd_complete_aio(RADOSCB *rcb) · line 466

Quoted 13 lines
block/rbd.c +2 -0
             if (!acb->error) {
                 acb->ret = rcb->size;
             }
@@ -441,10 +458,12 @@ static void qemu_rbd_complete_aio(RADOSCB *rcb)
 
     g_free(rcb);
 
+#ifndef LIBRBD_SUPPORTS_IOVEC
     if (acb->cmd == RBD_AIO_READ) {
         qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
     }
     qemu_vfree(acb->bounce);
+#endif

[1]

Instead of a conditional, you can now use the constant with an if statement:

if (!LIBRBD_USE_IOVEC) {
      if (acb->cmd == RBD_AIO_READ) {
          qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
      }
      qemu_vfree(acb->bounce);
}

block/rbd.c static BlockAIOCB *rbd_start_aio(BlockDriverState *bs, · line 680

Quoted 12 lines
block/rbd.c +3 -1
     acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret));
 
     qemu_aio_unref(acb);
@@ -655,8 +674,10 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
     RBDAIOCB *acb;
     RADOSCB *rcb = NULL;
     rbd_completion_t c;
-    char *buf;
     int r;
+#ifndef LIBRBD_SUPPORTS_IOVEC
+    char *buf = NULL;
+#endif

This variable can completely go away with some code re-arrangement (below)

block/rbd.c static BlockAIOCB *rbd_start_aio(BlockDriverState *bs, · line 687

Quoted 7 lines
block/rbd.c +0 -0
     BDRVRBDState *s = bs->opaque;
 
@@ -664,6 +685,8 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
     acb->cmd = cmd;
     acb->qiov = qiov;
     assert(!qiov || qiov->size == size);

[2]

Allocate rcb here

block/rbd.c static BlockAIOCB *rbd_start_aio(BlockDriverState *bs, · line 689

Quoted 2 lines
block/rbd.c +2 -0
+#ifndef LIBRBD_SUPPORTS_IOVEC
+

You can use if(!LIBRBD_USE_IOVEC) here then.

Quoted 6 lines
block/rbd.c +0 -0
     if (cmd == RBD_AIO_DISCARD || cmd == RBD_AIO_FLUSH) {
         acb->bounce = NULL;
     } else {
@@ -672,19 +695,21 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
             goto failed;
         }

block/rbd.c static BlockAIOCB *rbd_start_aio(BlockDriverState *bs, · line 701

Quoted 10 lines
block/rbd.c +0 -5
     }
-    acb->ret = 0;
-    acb->error = 0;
-    acb->s = s;
-
     if (cmd == RBD_AIO_WRITE) {
         qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
     }
-
     buf = acb->bounce;

This can just become rcb->buf = acb->bounce

block/rbd.c static BlockAIOCB *rbd_start_aio(BlockDriverState *bs, · line 707

Quoted 7 lines
block/rbd.c +4 -0
+#endif
 
+    acb->ret = 0;
+    acb->error = 0;
+    acb->s = s;
 
     rcb = g_new(RADOSCB, 1);

Move this to [2]

block/rbd.c static BlockAIOCB *rbd_start_aio(BlockDriverState *bs, · line 712

Quoted 5 lines
block/rbd.c +3 -0
+
     rcb->acb = acb;
+#ifndef LIBRBD_SUPPORTS_IOVEC
     rcb->buf = buf;
+#endif

These 3 lines can then go away

block/rbd.c static BlockAIOCB *rbd_start_aio(BlockDriverState *bs, · line 723

Quoted 9 lines
block/rbd.c +0 -0
     rcb->s = acb->s;
     rcb->size = size;
     r = rbd_aio_create_completion(rcb, (rbd_callback_t) rbd_finish_aiocb, &c);
block/rbd.c +1 -0
@@ -694,10 +719,18 @@ static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
 
     switch (cmd) {
     case RBD_AIO_WRITE:
+#ifndef LIBRBD_SUPPORTS_IOVEC
         r = rbd_aio_write(s->image, off, size, buf, c);

If you use rcb->buf here, that allows buf to go away completely.

block/rbd.c static BlockAIOCB *rbd_start_aio(BlockDriverState *bs, · line 730

Quoted 7 lines
block/rbd.c +4 -0
+#else
+        r = rbd_aio_writev(s->image, qiov->iov, qiov->niov, off, c);
+#endif
         break;
     case RBD_AIO_READ:
+#ifndef LIBRBD_SUPPORTS_IOVEC
         r = rbd_aio_read(s->image, off, size, buf, c);

Same

block/rbd.c failed_completion: · line 755

Quoted 11 lines
block/rbd.c +4 -0
+#else
+        r = rbd_aio_readv(s->image, qiov->iov, qiov->niov, off, c);
+#endif
         break;
     case RBD_AIO_DISCARD:
         r = rbd_aio_discard_wrapper(s->image, off, size, c);
@@ -719,7 +752,9 @@ failed_completion:
     rbd_aio_release(c);
 failed:
     g_free(rcb);
+#ifndef LIBRBD_SUPPORTS_IOVEC

And use the if(!LIBRBD_USE_IOVEC) here too.

Quoted 8 lines
block/rbd.c +1 -0
     qemu_vfree(acb->bounce);
+#endif
     qemu_aio_unref(acb);
     return NULL;
 }
Signature · 1 line
2.10.2

-Jeff