Skip to contentSkip to conversation

linux-aio: add IO_CMD_FDSYNC command support

PATCH Superseded Prasad Pandit 2 participants 4 messages Mar 13, 2024 Block layer

v3 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

PATCH v3 1/1 linux-aio: add IO_CMD_FDSYNC command support

From: Prasad Pandit <[email protected]>

Libaio defines IO_CMD_FDSYNC command to sync all outstanding asynchronous I/O operations, by flushing out file data to the disk storage.

Enable linux-aio to submit such aio request. This helps to reduce latency induced via pthread_create calls by thread-pool (aio=threads).

---
 block/file-posix.c |  7 +++++++
 block/linux-aio.c  | 22 +++++++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

v3: check if host kernel supports aio_fsync call
-> https://lists.nongnu.org/archive/html/qemu-devel/2024-03/msg03210.html

block/file-posix.c +7 -0
index 35684f7e21..a30494d1a8 100644
@@ -2453,6 +2453,8 @@ static inline bool raw_check_linux_io_uring(BDRVRawState *s)
 #endif
 
 #ifdef CONFIG_LINUX_AIO
+extern bool laio_has_fdsync(int);
+
 static inline bool raw_check_linux_aio(BDRVRawState *s)
 {
     Error *local_err = NULL;
@@ -2599,6 +2601,11 @@ static int coroutine_fn raw_co_flush_to_disk(BlockDriverState *bs)
     if (raw_check_linux_io_uring(s)) {
         return luring_co_submit(bs, s->fd, 0, NULL, QEMU_AIO_FLUSH);
     }
+#endif
+#ifdef CONFIG_LINUX_AIO
+    if (raw_check_linux_aio(s) && laio_has_fdsync(s->fd)) {
+        return laio_co_submit(s->fd, 0, NULL, QEMU_AIO_FLUSH, 0);
+    }
 #endif
     return raw_thread_pool_submit(handle_aiocb_flush, &acb);
 }
block/linux-aio.c +21 -1
index ec05d946f3..ed20a503e9 100644
@@ -67,6 +67,7 @@ struct LinuxAioState {
     int event_max;
 };
 
+bool laio_has_fdsync(int);
 static void ioq_submit(LinuxAioState *s);
 
 static inline ssize_t io_event_ret(struct io_event *ev)
@@ -384,6 +385,9 @@ static int laio_do_submit(int fd, struct qemu_laiocb *laiocb, off_t offset,
     case QEMU_AIO_READ:
         io_prep_preadv(iocbs, fd, qiov->iov, qiov->niov, offset);
         break;
+    case QEMU_AIO_FLUSH:
+        io_prep_fdsync(iocbs, fd);
+        break;
     /* Currently Linux kernel does not support other operations */
     default:
         fprintf(stderr, "%s: invalid AIO request type 0x%x.\n",
@@ -412,7 +416,7 @@ int coroutine_fn laio_co_submit(int fd, uint64_t offset, QEMUIOVector *qiov,
     AioContext *ctx = qemu_get_current_aio_context();
     struct qemu_laiocb laiocb = {
         .co         = qemu_coroutine_self(),
-        .nbytes     = qiov->size,
+        .nbytes     = qiov ? qiov->size : 0,
         .ctx        = aio_get_linux_aio(ctx),
         .ret        = -EINPROGRESS,
         .is_read    = (type == QEMU_AIO_READ),
@@ -486,3 +490,19 @@ void laio_cleanup(LinuxAioState *s)
     }
     g_free(s);
 }
+
+bool laio_has_fdsync(int fd)
+{
+    struct iocb cb;
+    struct iocb *cbs[] = {&cb, NULL};
+
+    io_context_t ctx = 0;
+    io_setup(1, &ctx);
+
+    /* check if host kernel supports IO_CMD_FDSYNC */
+    io_prep_fdsync(&cb, fd);
+    int ret = io_submit(ctx, 1, cbs);
+
+    io_destroy(ctx);
+    return (ret == -EINVAL) ? false : true;
+}
Signature · 1 line
2.44.0
#1 lore raw
Quoted 24 lines · Prasad Pandit

From: Prasad Pandit <[email protected]>

Libaio defines IO_CMD_FDSYNC command to sync all outstanding asynchronous I/O operations, by flushing out file data to the disk storage.

Enable linux-aio to submit such aio request. This helps to reduce latency induced via pthread_create calls by thread-pool (aio=threads).

---
 block/file-posix.c |  7 +++++++
 block/linux-aio.c  | 22 +++++++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

v3: check if host kernel supports aio_fsync call
-> https://lists.nongnu.org/archive/html/qemu-devel/2024-03/msg03210.html

block/file-posix.c +0 -0
index 35684f7e21..a30494d1a8 100644

block/file-posix.c static inline bool raw_check_linux_io_uring(BDRVRawState *s) · line 2456

Quoted 4 lines
block/file-posix.c +1 -0
@@ -2453,6 +2453,8 @@ static inline bool raw_check_linux_io_uring(BDRVRawState *s)
 #endif
 
 #ifdef CONFIG_LINUX_AIO
+extern bool laio_has_fdsync(int);

Please declare this in include/block/raw-aio.h alongside the other laio APIs.

block/file-posix.c static int coroutine_fn raw_co_flush_to_disk(BlockDriverState *bs) · line 2606

Quoted 11 lines
block/file-posix.c +4 -0
+
 static inline bool raw_check_linux_aio(BDRVRawState *s)
 {
     Error *local_err = NULL;
@@ -2599,6 +2601,11 @@ static int coroutine_fn raw_co_flush_to_disk(BlockDriverState *bs)
     if (raw_check_linux_io_uring(s)) {
         return luring_co_submit(bs, s->fd, 0, NULL, QEMU_AIO_FLUSH);
     }
+#endif
+#ifdef CONFIG_LINUX_AIO
+    if (raw_check_linux_aio(s) && laio_has_fdsync(s->fd)) {

FDSYNC support should be probed at open() time and the result should be stored in a new bool field like s->laio_supports_fdsync. That way the cost of laio_has_fdsync() on every flush request is avoided.

Quoted 59 lines
block/file-posix.c +2 -0
+        return laio_co_submit(s->fd, 0, NULL, QEMU_AIO_FLUSH, 0);
+    }
 #endif
     return raw_thread_pool_submit(handle_aiocb_flush, &acb);
 }
block/linux-aio.c +21 -1
index ec05d946f3..ed20a503e9 100644
@@ -67,6 +67,7 @@ struct LinuxAioState {
     int event_max;
 };
 
+bool laio_has_fdsync(int);
 static void ioq_submit(LinuxAioState *s);
 
 static inline ssize_t io_event_ret(struct io_event *ev)
@@ -384,6 +385,9 @@ static int laio_do_submit(int fd, struct qemu_laiocb *laiocb, off_t offset,
     case QEMU_AIO_READ:
         io_prep_preadv(iocbs, fd, qiov->iov, qiov->niov, offset);
         break;
+    case QEMU_AIO_FLUSH:
+        io_prep_fdsync(iocbs, fd);
+        break;
     /* Currently Linux kernel does not support other operations */
     default:
         fprintf(stderr, "%s: invalid AIO request type 0x%x.\n",
@@ -412,7 +416,7 @@ int coroutine_fn laio_co_submit(int fd, uint64_t offset, QEMUIOVector *qiov,
     AioContext *ctx = qemu_get_current_aio_context();
     struct qemu_laiocb laiocb = {
         .co         = qemu_coroutine_self(),
-        .nbytes     = qiov->size,
+        .nbytes     = qiov ? qiov->size : 0,
         .ctx        = aio_get_linux_aio(ctx),
         .ret        = -EINPROGRESS,
         .is_read    = (type == QEMU_AIO_READ),
@@ -486,3 +490,19 @@ void laio_cleanup(LinuxAioState *s)
     }
     g_free(s);
 }
+
+bool laio_has_fdsync(int fd)
+{
+    struct iocb cb;
+    struct iocb *cbs[] = {&cb, NULL};
+
+    io_context_t ctx = 0;
+    io_setup(1, &ctx);
+
+    /* check if host kernel supports IO_CMD_FDSYNC */
+    io_prep_fdsync(&cb, fd);
+    int ret = io_submit(ctx, 1, cbs);
+
+    io_destroy(ctx);
+    return (ret == -EINVAL) ? false : true;
+}
Signature · 1 line
2.44.0
#2 lore raw
Quoted 6 lines · Stefan Hajnoczi
Quoted 1 line

+extern bool laio_has_fdsync(int);

Please declare this in include/block/raw-aio.h alongside the other laio APIs.

FDSYNC support should be probed at open() time and the result should be stored in a new bool field like s->laio_supports_fdsync. That way the cost of laio_has_fdsync() on every flush request is avoided.

* Okay. Here 's' is a BDRVRawState object and file open seems to happen in the raw_open_common() function? I'll move the laio_has_fdsync() call there and see how it works.

Thank you.
---
- Prasad

#3 lore raw
Quoted 11 lines · Prasad Pandit
Quoted 6 lines · Stefan Hajnoczi
Quoted 1 line

+extern bool laio_has_fdsync(int);

Please declare this in include/block/raw-aio.h alongside the other laio APIs.

FDSYNC support should be probed at open() time and the result should be stored in a new bool field like s->laio_supports_fdsync. That way the cost of laio_has_fdsync() on every flush request is avoided.

* Okay. Here 's' is a BDRVRawState object and file open seems to happen in the raw_open_common() function? I'll move the laio_has_fdsync() call there and see how it works.

Yes. Thanks!

Stefan