Skip to contentSkip to conversation

block/nfs optimizations

PATCH Applied Peter Lieven 4 participants 10 messages Feb 17, 2017 – Feb 24, 2017 Block layer

lore
Share & tools
lore
#0 lore raw

cover block/nfs optimizations

Peter Lieven (2):
block/nfs: convert to preadv / pwritev block/nfs: try to avoid the bounce buffer in pwritev

 block/nfs.c | 50 ++++++++++++++++++++++++++++----------------------
 1 file changed, 28 insertions(+), 22 deletions(-)
Signature · 1 line
1.9.1
#1 lore raw

PATCH 1/2 block/nfs: convert to preadv / pwritev

---
 block/nfs.c | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)
block/nfs.c +15 -18
index 689eaa7..ab5dcc2 100644
@@ -256,9 +256,9 @@ nfs_co_generic_cb(int ret, struct nfs_context *nfs, void *data,
                             nfs_co_generic_bh_cb, task);
 }
 
-static int coroutine_fn nfs_co_readv(BlockDriverState *bs,
-                                     int64_t sector_num, int nb_sectors,
-                                     QEMUIOVector *iov)
+static int coroutine_fn nfs_co_preadv(BlockDriverState *bs, uint64_t offset,
+                                      uint64_t bytes, QEMUIOVector *iov,
+                                      int flags)
 {
     NFSClient *client = bs->opaque;
     NFSRPC task;
@@ -267,9 +267,7 @@ static int coroutine_fn nfs_co_readv(BlockDriverState *bs,
     task.iov = iov;
 
     if (nfs_pread_async(client->context, client->fh,
-                        sector_num * BDRV_SECTOR_SIZE,
-                        nb_sectors * BDRV_SECTOR_SIZE,
-                        nfs_co_generic_cb, &task) != 0) {
+                        offset, bytes, nfs_co_generic_cb, &task) != 0) {
         return -ENOMEM;
     }
 
@@ -290,9 +288,9 @@ static int coroutine_fn nfs_co_readv(BlockDriverState *bs,
     return 0;
 }
 
-static int coroutine_fn nfs_co_writev(BlockDriverState *bs,
-                                        int64_t sector_num, int nb_sectors,
-                                        QEMUIOVector *iov)
+static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset,
+                                       uint64_t bytes, QEMUIOVector *iov,
+                                       int flags)
 {
     NFSClient *client = bs->opaque;
     NFSRPC task;
@@ -300,17 +298,16 @@ static int coroutine_fn nfs_co_writev(BlockDriverState *bs,
 
     nfs_co_init_task(bs, &task);
 
-    buf = g_try_malloc(nb_sectors * BDRV_SECTOR_SIZE);
-    if (nb_sectors && buf == NULL) {
+    buf = g_try_malloc(bytes);
+    if (bytes && buf == NULL) {
         return -ENOMEM;
     }
 
-    qemu_iovec_to_buf(iov, 0, buf, nb_sectors * BDRV_SECTOR_SIZE);
+    qemu_iovec_to_buf(iov, 0, buf, bytes);
 
     if (nfs_pwrite_async(client->context, client->fh,
-                         sector_num * BDRV_SECTOR_SIZE,
-                         nb_sectors * BDRV_SECTOR_SIZE,
-                         buf, nfs_co_generic_cb, &task) != 0) {
+                         offset, bytes, buf,
+                         nfs_co_generic_cb, &task) != 0) {
         g_free(buf);
         return -ENOMEM;
     }
@@ -322,7 +319,7 @@ static int coroutine_fn nfs_co_writev(BlockDriverState *bs,
 
     g_free(buf);
 
-    if (task.ret != nb_sectors * BDRV_SECTOR_SIZE) {
+    if (task.ret != bytes) {
         return task.ret < 0 ? task.ret : -EIO;
     }
 
@@ -856,8 +853,8 @@ static BlockDriver bdrv_nfs = {
     .bdrv_create                    = nfs_file_create,
     .bdrv_reopen_prepare            = nfs_reopen_prepare,
 
-    .bdrv_co_readv                  = nfs_co_readv,
-    .bdrv_co_writev                 = nfs_co_writev,
+    .bdrv_co_preadv                 = nfs_co_preadv,
+    .bdrv_co_pwritev                = nfs_co_pwritev,
     .bdrv_co_flush_to_disk          = nfs_co_flush,
 
     .bdrv_detach_aio_context        = nfs_detach_aio_context,
Signature · 1 line
1.9.1
Reviewed-by on v1 1/2
#2 lore raw

Re: [Qemu-devel] [PATCH 1/2] block/nfs: convert to preadv / pwritev

Quoted 92 lines · Peter Lieven
---
 block/nfs.c | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)
block/nfs.c +15 -18
index 689eaa7..ab5dcc2 100644
@@ -256,9 +256,9 @@ nfs_co_generic_cb(int ret, struct nfs_context *nfs, void *data,
                             nfs_co_generic_bh_cb, task);
 }
 
-static int coroutine_fn nfs_co_readv(BlockDriverState *bs,
-                                     int64_t sector_num, int nb_sectors,
-                                     QEMUIOVector *iov)
+static int coroutine_fn nfs_co_preadv(BlockDriverState *bs, uint64_t offset,
+                                      uint64_t bytes, QEMUIOVector *iov,
+                                      int flags)
 {
     NFSClient *client = bs->opaque;
     NFSRPC task;
@@ -267,9 +267,7 @@ static int coroutine_fn nfs_co_readv(BlockDriverState *bs,
     task.iov = iov;
 
     if (nfs_pread_async(client->context, client->fh,
-                        sector_num * BDRV_SECTOR_SIZE,
-                        nb_sectors * BDRV_SECTOR_SIZE,
-                        nfs_co_generic_cb, &task) != 0) {
+                        offset, bytes, nfs_co_generic_cb, &task) != 0) {
         return -ENOMEM;
     }
 
@@ -290,9 +288,9 @@ static int coroutine_fn nfs_co_readv(BlockDriverState *bs,
     return 0;
 }
 
-static int coroutine_fn nfs_co_writev(BlockDriverState *bs,
-                                        int64_t sector_num, int nb_sectors,
-                                        QEMUIOVector *iov)
+static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset,
+                                       uint64_t bytes, QEMUIOVector *iov,
+                                       int flags)
 {
     NFSClient *client = bs->opaque;
     NFSRPC task;
@@ -300,17 +298,16 @@ static int coroutine_fn nfs_co_writev(BlockDriverState *bs,
 
     nfs_co_init_task(bs, &task);
 
-    buf = g_try_malloc(nb_sectors * BDRV_SECTOR_SIZE);
-    if (nb_sectors && buf == NULL) {
+    buf = g_try_malloc(bytes);
+    if (bytes && buf == NULL) {
         return -ENOMEM;
     }
 
-    qemu_iovec_to_buf(iov, 0, buf, nb_sectors * BDRV_SECTOR_SIZE);
+    qemu_iovec_to_buf(iov, 0, buf, bytes);
 
     if (nfs_pwrite_async(client->context, client->fh,
-                         sector_num * BDRV_SECTOR_SIZE,
-                         nb_sectors * BDRV_SECTOR_SIZE,
-                         buf, nfs_co_generic_cb, &task) != 0) {
+                         offset, bytes, buf,
+                         nfs_co_generic_cb, &task) != 0) {
         g_free(buf);
         return -ENOMEM;
     }
@@ -322,7 +319,7 @@ static int coroutine_fn nfs_co_writev(BlockDriverState *bs,
 
     g_free(buf);
 
-    if (task.ret != nb_sectors * BDRV_SECTOR_SIZE) {
+    if (task.ret != bytes) {
         return task.ret < 0 ? task.ret : -EIO;
     }
 
@@ -856,8 +853,8 @@ static BlockDriver bdrv_nfs = {
     .bdrv_create                    = nfs_file_create,
     .bdrv_reopen_prepare            = nfs_reopen_prepare,
 
-    .bdrv_co_readv                  = nfs_co_readv,
-    .bdrv_co_writev                 = nfs_co_writev,
+    .bdrv_co_preadv                 = nfs_co_preadv,
+    .bdrv_co_pwritev                = nfs_co_pwritev,
     .bdrv_co_flush_to_disk          = nfs_co_flush,
 
     .bdrv_detach_aio_context        = nfs_detach_aio_context,
Signature · 1 line
1.9.1
#3 lore raw

PATCH 2/2 block/nfs: try to avoid the bounce buffer in pwritev

In reply to Peter Lieven

if the passed qiov contains exactly one iov we can pass the buffer directly.

---
 block/nfs.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)
block/nfs.c +16 -7
index ab5dcc2..bb4b75f 100644
@@ -295,20 +295,27 @@ static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset,
     NFSClient *client = bs->opaque;
     NFSRPC task;
     char *buf = NULL;
+    bool my_buffer = false;
 
     nfs_co_init_task(bs, &task);
 
-    buf = g_try_malloc(bytes);
-    if (bytes && buf == NULL) {
-        return -ENOMEM;
+    if (iov->niov != 1) {
+        buf = g_try_malloc(bytes);
+        if (bytes && buf == NULL) {
+            return -ENOMEM;
+        }
+        qemu_iovec_to_buf(iov, 0, buf, bytes);
+        my_buffer = true;
+    } else {
+        buf = iov->iov[0].iov_base;
     }
 
-    qemu_iovec_to_buf(iov, 0, buf, bytes);
-
     if (nfs_pwrite_async(client->context, client->fh,
                          offset, bytes, buf,
                          nfs_co_generic_cb, &task) != 0) {
-        g_free(buf);
+        if (my_buffer) {
+            g_free(buf);
+        }
         return -ENOMEM;
     }
 
@@ -317,7 +324,9 @@ static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset,
         qemu_coroutine_yield();
     }
 
-    g_free(buf);
+    if (my_buffer) {
+        g_free(buf);
+    }
 
     if (task.ret != bytes) {
         return task.ret < 0 ? task.ret : -EIO;
Signature · 1 line
1.9.1
#4 lore raw

Re: [Qemu-devel] [PATCH 2/2] block/nfs: try to avoid the bounce buffer in pwritev

Quoted 13 lines · Peter Lieven

if the passed qiov contains exactly one iov we can pass the buffer directly.

---
 block/nfs.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)
block/nfs.c +0 -0
index ab5dcc2..bb4b75f 100644

block/nfs.c static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset, · line 298

Quoted 4 lines
block/nfs.c +1 -0
@@ -295,20 +295,27 @@ static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset,
     NFSClient *client = bs->opaque;
     NFSRPC task;
     char *buf = NULL;
+    bool my_buffer = false;

g_free() is a nop if buf is NULL, so there is no need for the my_buffer variable & check.

Quoted 43 lines
block/nfs.c +15 -7
     nfs_co_init_task(bs, &task);
 
-    buf = g_try_malloc(bytes);
-    if (bytes && buf == NULL) {
-        return -ENOMEM;
+    if (iov->niov != 1) {
+        buf = g_try_malloc(bytes);
+        if (bytes && buf == NULL) {
+            return -ENOMEM;
+        }
+        qemu_iovec_to_buf(iov, 0, buf, bytes);
+        my_buffer = true;
+    } else {
+        buf = iov->iov[0].iov_base;
     }
 
-    qemu_iovec_to_buf(iov, 0, buf, bytes);
-
     if (nfs_pwrite_async(client->context, client->fh,
                          offset, bytes, buf,
                          nfs_co_generic_cb, &task) != 0) {
-        g_free(buf);
+        if (my_buffer) {
+            g_free(buf);
+        }
         return -ENOMEM;
     }
 
@@ -317,7 +324,9 @@ static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset,
         qemu_coroutine_yield();
     }
 
-    g_free(buf);
+    if (my_buffer) {
+        g_free(buf);
+    }
 
     if (task.ret != bytes) {
         return task.ret < 0 ? task.ret : -EIO;
Signature · 1 line
1.9.1
#5 lore raw

Re: [Qemu-devel] [PATCH 2/2] block/nfs: try to avoid the bounce buffer in pwritev

Quoted 9 lines · Jeff Cody
Quoted 8 lines · Peter Lieven

if the passed qiov contains exactly one iov we can pass the buffer directly.

---
 block/nfs.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)
Quoted 12 lines

block/nfs.c static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset, · line 298

Quoted 9 lines
block/nfs.c +1 -0
index ab5dcc2..bb4b75f 100644
@@ -295,20 +295,27 @@ static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset,
     NFSClient *client = bs->opaque;
     NFSRPC task;
     char *buf = NULL;
+    bool my_buffer = false;

g_free() is a nop if buf is NULL, so there is no need for the my_buffer variable & check.

Umm, yes there is:

Quoted 9 lines

block/nfs.c static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset, · line 310

Quoted 9 lines
block/nfs.c +9 -0
+    if (iov->niov != 1) {
+        buf = g_try_malloc(bytes);
+        if (bytes && buf == NULL) {
+            return -ENOMEM;
+        }
+        qemu_iovec_to_buf(iov, 0, buf, bytes);
+        my_buffer = true;
+    } else {
+        buf = iov->iov[0].iov_base;

If we took the else branch, then we definitely do not want to be calling g_free(buf).

Quoted 11 lines

block/nfs.c static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset, · line 318

Quoted 11 lines
block/nfs.c +3 -3
     }
 
-    qemu_iovec_to_buf(iov, 0, buf, bytes);
-
     if (nfs_pwrite_async(client->context, client->fh,
                          offset, bytes, buf,
                          nfs_co_generic_cb, &task) != 0) {
-        g_free(buf);
+        if (my_buffer) {
+            g_free(buf);
+        }

It looks correct as-is to me.

Signature · 2 lines
Eric Blake   eblake redhat com    +1-919-301-3266
Libvirt virtualization library http://libvirt.org
Reviewed-by
#6 lore raw

Re: [Qemu-devel] [PATCH 2/2] block/nfs: try to avoid the bounce buffer in pwritev

Quoted 35 lines · Eric Blake
Quoted 9 lines · Jeff Cody
Quoted 8 lines · Peter Lieven

if the passed qiov contains exactly one iov we can pass the buffer directly.

---
 block/nfs.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)
Quoted 12 lines

block/nfs.c static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset, · line 298

Quoted 9 lines
block/nfs.c +1 -0
index ab5dcc2..bb4b75f 100644
@@ -295,20 +295,27 @@ static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset,
     NFSClient *client = bs->opaque;
     NFSRPC task;
     char *buf = NULL;
+    bool my_buffer = false;

g_free() is a nop if buf is NULL, so there is no need for the my_buffer variable & check.

Umm, yes there is:

Quoted 9 lines

block/nfs.c static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset, · line 310

Quoted 9 lines
block/nfs.c +9 -0
+    if (iov->niov != 1) {
+        buf = g_try_malloc(bytes);
+        if (bytes && buf == NULL) {
+            return -ENOMEM;
+        }
+        qemu_iovec_to_buf(iov, 0, buf, bytes);
+        my_buffer = true;
+    } else {
+        buf = iov->iov[0].iov_base;
Quoted 2 lines

If we took the else branch, then we definitely do not want to be calling g_free(buf).

Doh!

Quoted 13 lines
Quoted 12 lines

block/nfs.c static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset, · line 318

Quoted 12 lines
block/nfs.c +3 -3
     }
 
-    qemu_iovec_to_buf(iov, 0, buf, bytes);
-
     if (nfs_pwrite_async(client->context, client->fh,
                          offset, bytes, buf,
                          nfs_co_generic_cb, &task) != 0) {
-        g_free(buf);
+        if (my_buffer) {
+            g_free(buf);
+        }
Quoted 1 line

It looks correct as-is to me.

Indeed.

#7 lore raw

Re: [Qemu-devel] [PATCH 2/2] block/nfs: try to avoid the bounce buffer in pwritev

Am 17.02.2017 um 22:51 hat Jeff Cody geschrieben:

Quoted 54 lines · Eric Blake
Quoted 35 lines · Jeff Cody
Quoted 9 lines · Peter Lieven
Quoted 8 lines

if the passed qiov contains exactly one iov we can pass the buffer directly.

---
 block/nfs.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)
Quoted 12 lines

block/nfs.c static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset, · line 298

Quoted 9 lines
block/nfs.c +1 -0
index ab5dcc2..bb4b75f 100644
@@ -295,20 +295,27 @@ static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset,
     NFSClient *client = bs->opaque;
     NFSRPC task;
     char *buf = NULL;
+    bool my_buffer = false;

g_free() is a nop if buf is NULL, so there is no need for the my_buffer variable & check.

Umm, yes there is:

Quoted 9 lines

block/nfs.c static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset, · line 310

Quoted 9 lines
block/nfs.c +9 -0
+    if (iov->niov != 1) {
+        buf = g_try_malloc(bytes);
+        if (bytes && buf == NULL) {
+            return -ENOMEM;
+        }
+        qemu_iovec_to_buf(iov, 0, buf, bytes);
+        my_buffer = true;
+    } else {
+        buf = iov->iov[0].iov_base;
Quoted 2 lines

If we took the else branch, then we definitely do not want to be calling g_free(buf).

Doh!

Quoted 13 lines
Quoted 12 lines
Quoted 12 lines
block/nfs.c +3 -3
     }
 
-    qemu_iovec_to_buf(iov, 0, buf, bytes);
-
     if (nfs_pwrite_async(client->context, client->fh,
                          offset, bytes, buf,
                          nfs_co_generic_cb, &task) != 0) {
-        g_free(buf);
+        if (my_buffer) {
+            g_free(buf);
+        }
Quoted 5 lines
Quoted 1 line

It looks correct as-is to me.

Indeed.

You gave R-b for both patches, but did not merge it - who is supposed to do that?

Kevin

#8 lore raw

Re: [Qemu-devel] [PATCH 2/2] block/nfs: try to avoid the bounce buffer in pwritev

Quoted 55 lines · Kevin Wolf

Am 17.02.2017 um 22:51 hat Jeff Cody geschrieben:

Quoted 54 lines · Eric Blake
Quoted 35 lines · Jeff Cody
Quoted 9 lines · Peter Lieven
Quoted 8 lines

if the passed qiov contains exactly one iov we can pass the buffer directly.

---
 block/nfs.c | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)
Quoted 12 lines

block/nfs.c static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset, · line 298

Quoted 9 lines
block/nfs.c +1 -0
index ab5dcc2..bb4b75f 100644
@@ -295,20 +295,27 @@ static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset,
     NFSClient *client = bs->opaque;
     NFSRPC task;
     char *buf = NULL;
+    bool my_buffer = false;

g_free() is a nop if buf is NULL, so there is no need for the my_buffer variable & check.

Umm, yes there is:

Quoted 9 lines

block/nfs.c static int coroutine_fn nfs_co_pwritev(BlockDriverState *bs, uint64_t offset, · line 310

Quoted 9 lines
block/nfs.c +9 -0
+    if (iov->niov != 1) {
+        buf = g_try_malloc(bytes);
+        if (bytes && buf == NULL) {
+            return -ENOMEM;
+        }
+        qemu_iovec_to_buf(iov, 0, buf, bytes);
+        my_buffer = true;
+    } else {
+        buf = iov->iov[0].iov_base;
Quoted 2 lines

If we took the else branch, then we definitely do not want to be calling g_free(buf).

Doh!

Quoted 13 lines
Quoted 12 lines
Quoted 12 lines
block/nfs.c +3 -3
     }
 
-    qemu_iovec_to_buf(iov, 0, buf, bytes);
-
     if (nfs_pwrite_async(client->context, client->fh,
                          offset, bytes, buf,
                          nfs_co_generic_cb, &task) != 0) {
-        g_free(buf);
+        if (my_buffer) {
+            g_free(buf);
+        }
Quoted 9 lines
Quoted 5 lines
Quoted 1 line

It looks correct as-is to me.

Indeed.

You gave R-b for both patches, but did not merge it - who is supposed to do that?

I am going to do it in my next round, I was just waiting to see if there were any other comments.

Applied
#9 lore raw
In reply to Peter Lieven
Quoted 10 lines · Peter Lieven

Peter Lieven (2):
block/nfs: convert to preadv / pwritev block/nfs: try to avoid the bounce buffer in pwritev

 block/nfs.c | 50 ++++++++++++++++++++++++++++----------------------
 1 file changed, 28 insertions(+), 22 deletions(-)
Signature · 1 line
1.9.1

Thanks,

Applied to my block branch:

git://github.com/codyprime/qemu-kvm-jtc.git block

-Jeff