Make thread pool implementation modular
This patch series makes the thread pool implementation modular. This allows each drive to use a special implementation.
The patch series prepares qemu to be able to include thread pools different the one actually implemented. It will allow to implement approaches like paravirtualized block requests [1].
async.c | 4 ++-- block/raw-posix.c | 15 +++++++++++---- block/raw-win32.c | 9 +++++++-- blockdev.c | 13 +++++++++++++ include/block/aio.h | 2 +- include/block/thread-pool.h | 9 +++++++++ include/qemu-common.h | 2 ++ qemu-options.hx | 2 +- thread-pool.c | 32 ++++++++++++++++++++++++++++++++ 9 files changed, 78 insertions(+), 10 deletions(-)
[1] http://www.linux-kvm.org/wiki/images/5/53/2012-forum-Brugger-lightningtalk.pdf
PATCH 1/3 Make thread pool implementation modular
This patch introduces function pointers for the thread pool, so that it's implementation can be set at run-time.
--- include/block/thread-pool.h | 9 +++++++++ thread-pool.c | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+)
include/block/thread-pool.h +9 -0
@@ -38,4 +38,13 @@ int coroutine_fn thread_pool_submit_co(ThreadPool *pool, ThreadPoolFunc *func, void *arg);void thread_pool_submit (ThreadPool *pool,ThreadPoolFunc *func,void *arg); +ThreadPoolFuncArr *thread_pool_probe (void ); +void thread_pool_delete (ThreadPoolFuncArr *tpf); + +struct ThreadPoolFuncArr { +BlockDriverAIOCB *(*thread_pool_submit_aio)(ThreadPool *pool,ThreadPoolFunc *func,void *arg,BlockDriverCompletionFunc *cb,void *opaque); +ThreadPool *(*thread_pool_new)(AioContext *ctx); +}; + +#endif
thread-pool.c +32 -0
@@ -26,6 +26,7 @@#include "qemu/main-loop.h" static void do_spawn_thread (ThreadPool *pool); +void thread_pool_aio_free (ThreadPool *pool);typedef struct ThreadPoolElement ThreadPoolElement ;
@@ -77,6 +78,7 @@ struct ThreadPool {int pending_threads;/* threads created but not running yet */ int pending_cancellations;/* whether we need a cond_broadcast */ bool stopping; +void (*thread_pool_free)(ThreadPool *pool); };static void *worker_thread (void *opaque)
@@ -300,6 +302,7 @@ static void thread_pool_init_one(ThreadPool *pool, AioContext *ctx)qemu_sem_init (&pool->sem,0 ); pool->max_threads =64 ; pool->new_thread_bh =aio_bh_new (ctx, spawn_thread_bh_fn, pool); + pool->thread_pool_free = &thread_pool_aio_free;QLIST_INIT (&pool->head);QTAILQ_INIT (&pool->request_list);
@@ -316,6 +319,11 @@ ThreadPool *thread_pool_new(AioContext *ctx)void thread_pool_free (ThreadPool *pool) { + pool->thread_pool_free (pool); +} + +void thread_pool_aio_free (ThreadPool *pool) +{if (!pool) {return ; }
@@ -346,3 +354,27 @@ void thread_pool_free(ThreadPool *pool)event_notifier_cleanup (&pool->notifier);g_free (pool); } + +ThreadPoolFuncArr *thread_pool_probe (void ) +{ +ThreadPoolFuncArr *tpf_pool =NULL ; + +if (tpf_pool) +return tpf_pool; + + tpf_pool =g_new (ThreadPoolFuncArr,1 );//TODO right now, this leaks! +if (!tpf_pool) { +printf ("error allocating thread pool\n" ); +return NULL ; + } + + tpf_pool->thread_pool_submit_aio = thread_pool_submit_aio; + tpf_pool->thread_pool_new = thread_pool_new; + +return tpf_pool; +} + +void thread_pool_delete (ThreadPoolFuncArr *tpf) +{ +g_free (tpf); +}
Signature · 1 line
1.8.1.2
PATCH 2/3 Block layer uses modular thread pool
In reply to Matthias BruggerWith this patch, the calls to the thread pool functions pass through the new modular thread pool implementation.
--- async.c | 4 ++-- block/raw-posix.c | 15 +++++++++++---- block/raw-win32.c | 9 +++++++-- include/block/aio.h | 2 +- include/qemu-common.h | 2 ++ 5 files changed, 23 insertions(+), 9 deletions(-)
async.c +2 -2
@@ -232,10 +232,10 @@ GSource *aio_get_g_source(AioContext *ctx)return &ctx->source; } -ThreadPool *aio_get_thread_pool (AioContext *ctx) +ThreadPool *aio_get_thread_pool (AioContext *ctx,ThreadPoolFuncArr *tpf) {if (!ctx->thread_pool) { - ctx->thread_pool =thread_pool_new (ctx); + ctx->thread_pool = tpf->thread_pool_new (ctx); }return ctx->thread_pool; }
block/raw-posix.c +11 -4
@@ -142,6 +142,7 @@ typedef struct BDRVRawState { bool is_xfs : 1;#endif bool has_discard : 1; +ThreadPoolFuncArr *tpf; } BDRVRawState; typedef struct BDRVRawReopenState {
@@ -345,6 +346,9 @@ static int raw_open(BlockDriverState *bs, QDict *options, int flags,int ret; s->type =FTYPE_FILE ; + + s->tpf =thread_pool_probe (); + ret =raw_open_common (bs, options, flags,0 , &local_err);if (error_is_set (&local_err)) {error_propagate (errp, local_err);
@@ -792,6 +796,7 @@ static BlockDriverAIOCB *paio_submit(BlockDriverState *bs, int fd, int64_t sector_num, QEMUIOVector *qiov, int nb_sectors, BlockDriverCompletionFunc *cb, void *opaque, int type) { + BDRVRawState *s = bs->opaque;RawPosixAIOData *acb =g_slice_new (RawPosixAIOData);ThreadPool *pool;
@@ -807,8 +812,8 @@ static BlockDriverAIOCB *paio_submit(BlockDriverState *bs, int fd, acb->aio_offset = sector_num *512 ;trace_paio_submit (acb, opaque, sector_num, nb_sectors, type); - pool =aio_get_thread_pool (bdrv_get_aio_context (bs)); -return thread_pool_submit_aio (pool, aio_worker, acb, cb, opaque); + pool =aio_get_thread_pool (bdrv_get_aio_context (bs), s->tpf); +return s->tpf->thread_pool_submit_aio (pool, aio_worker, acb, cb, opaque); } static BlockDriverAIOCB *raw_aio_submit(BlockDriverState *bs,
@@ -874,6 +879,8 @@ static void raw_close(BlockDriverState *bs)qemu_close (s->fd); s->fd =-1 ; } + +thread_pool_delete (s->tpf); }static int raw_truncate (BlockDriverState *bs,int64_t offset)
@@ -1490,8 +1497,8 @@ static BlockDriverAIOCB *hdev_aio_ioctl(BlockDriverState *bs, acb->aio_offset =0 ; acb->aio_ioctl_buf = buf; acb->aio_ioctl_cmd = req; - pool =aio_get_thread_pool (bdrv_get_aio_context (bs)); - return thread_pool_submit_aio(pool, aio_worker, acb, cb, opaque); + pool =aio_get_thread_pool (bdrv_get_aio_context (bs), s->tpf); + return s->tpf->thread_pool_submit_aio(pool, aio_worker, acb, cb, opaque); }#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
block/raw-win32.c +7 -2
@@ -46,6 +46,7 @@ typedef struct RawWin32AIOData {size_t aio_nbytes;off64_t aio_offset;int aio_type; +ThreadPoolFuncArr *tpf; } RawWin32AIOData; typedef struct BDRVRawState {
@@ -159,8 +160,8 @@ static BlockDriverAIOCB *paio_submit(BlockDriverState *bs, HANDLE hfile, acb->aio_offset = sector_num *512 ;trace_paio_submit (acb, opaque, sector_num, nb_sectors, type); - pool =aio_get_thread_pool (bdrv_get_aio_context (bs)); -return thread_pool_submit_aio (pool, aio_worker, acb, cb, opaque); + pool =aio_get_thread_pool (bdrv_get_aio_context (bs), s->tpf); +return s->tpf->thread_pool_submit_aio (pool, aio_worker, acb, cb, opaque); }int qemu_ftruncate64 (int fd,int64_t length)
@@ -248,6 +249,8 @@ static int raw_open(BlockDriverState *bs, QDict *options, int flags, s->type =FTYPE_FILE ; + s->tpf =thread_pool_probe (); + opts =qemu_opts_create_nofail (&raw_runtime_opts);qemu_opts_absorb_qdict (opts, options, &local_err);if (error_is_set (&local_err)) {
@@ -339,6 +342,8 @@ static void raw_close(BlockDriverState *bs) {BDRVRawState *s = bs->opaque;CloseHandle (s->hfile); + +thread_pool_delete (s->tpf); }static int raw_truncate (BlockDriverState *bs,int64_t offset)
include/block/aio.h +1 -1
@@ -231,7 +231,7 @@ void aio_set_event_notifier(AioContext *ctx,GSource *aio_get_g_source (AioContext *ctx);/* Return the ThreadPool bound to this AioContext */ -struct ThreadPool *aio_get_thread_pool (AioContext *ctx); +struct ThreadPool *aio_get_thread_pool (AioContext *ctx,ThreadPoolFuncArr *tpf);/* Functions to operate on the main QEMU AioContext. */
include/qemu-common.h +2 -0
@@ -478,4 +478,6 @@ size_t buffer_find_nonzero_offset(const void *buf, size_t len);*/ int parse_debug_env (const char *name,int max,int initial); +typedef struct ThreadPoolFuncArr ThreadPoolFuncArr ; +#endif
Signature · 1 line
1.8.1.2
PATCH 3/3 Add workerthreads configuration option
In reply to Matthias BruggerThis patch allows the definition which thread pool will be used by every block device. The defintion of the workerthreads option allows this at the command line level.
At the moment only the thread pool implementation "pool" can be chosen.
--- blockdev.c | 13 +++++++++++++ qemu-options.hx | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-)
blockdev.c +13 -0
@@ -388,6 +388,15 @@ static DriveInfo *blockdev_init(QDict *bs_opts, }#endif +if ((buf =qemu_opt_get (opts,"workerthreads" )) !=NULL ) { +if (!strcmp (buf,"pool" )) { +/* this is the default */ + }else { +error_report ("invalid workerthreads option" ); +return NULL ; + } + } +if ((buf =qemu_opt_get (opts,"format" )) !=NULL ) {if (is_help_option (buf)) {error_printf ("Supported formats:" );
@@ -2269,6 +2278,10 @@ QemuOptsList qemu_common_drive_opts = { .type = QEMU_OPT_STRING, .help = "disk serial number", },{ + .name = "workerthreads", + .type = QEMU_OPT_STRING, + .help = "type of worker threads (pool)", + },{ .name = "rerror", .type = QEMU_OPT_STRING, .help = "read error action",
qemu-options.hx +1 -1
@@ -408,7 +408,7 @@ DEF("drive", HAS_ARG, QEMU_OPTION_drive, " [,cyls=c,heads=h,secs=s[,trans=t]][,snapshot=on|off]\n" " [,cache=writethrough|writeback|none|directsync|unsafe][,format=f]\n" " [,serial=s][,addr=A][,id=name][,aio=threads|native]\n" - " [,readonly=on|off][,copy-on-read=on|off]\n" + " [,workerthreads=pool][,readonly=on|off][,copy-on-read=on|off]\n" " [[,bps=b]|[[,bps_rd=r][,bps_wr=w]]]\n" " [[,iops=i]|[[,iops_rd=r][,iops_wr=w]]]\n" " [[,bps_max=bm]|[[,bps_rd_max=rm][,bps_wr_max=wm]]]\n"
Signature · 1 line
1.8.1.2
Am 02.11.2013 00:34, schrieb Matthias Brugger:
Quoted 6 lines
This patch series makes the thread pool implementation modular. This allows each drive to use a special implementation.
The patch series prepares qemu to be able to include thread pools different the one actually implemented. It will allow to implement approaches like paravirtualized block requests [1].
Please use scripts/checkpatch.pl to check any patch before sending it to the list.
There are currently errors and warnings for patch 1 and patch 3 which should be
fixed.
In patch 3 you can also fix "defintion" in the patch description.
Regards,
Stefan