dataplane snapshot fixes
v2 cover dataplane snapshot fixes
The following test
while /bin/true ; do
virsh snapshot-create rhel7
sleep 10
virsh snapshot-delete rhel7 --current
done
with enabled iothreads on a running VM leads to a lot of troubles: hangs, asserts, errors.
Though (in general) HMP snapshot code is terrible. I think it should be dropped at once and replaced with blkdev transactions code. Though is could not fit to QEMU 2.5/stable at all.
Anyway, I think that the construction like
assert(aio_context_is_locked(aio_context));
should be widely used to ensure proper locking.
Changes from v1:
- aio-context locking added
- comment is rewritten
Denis V. Lunev (4):
fifolock: create rfifolock_is_locked helper
aio_context: create aio_context_is_locked helper
io: add locking constraints check into bdrv_drain to ensure locking
migration: add missed aio_context_acquire into hmp_savevm/hmp_delvm
Pavel Butsykin (1):
virtio: sync the dataplane vring state to the virtqueue before
virtio_save
async.c | 5 +++++ block/io.c | 5 ++++- block/snapshot.c | 5 +++++ hw/block/virtio-blk.c | 5 +++++ hw/scsi/virtio-scsi.c | 5 +++++ include/block/aio.h | 3 +++ include/qemu/rfifolock.h | 1 + migration/savevm.c | 7 +++++++ util/rfifolock.c | 9 +++++++-- 9 files changed, 42 insertions(+), 3 deletions(-)
Signature · 1 line
2.1.4
PATCH v2 1/5 fifolock: create rfifolock_is_locked helper
This helper is necessary to ensure locking constraints.
--- include/qemu/rfifolock.h | 1 + util/rfifolock.c | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-)
include/qemu/rfifolock.h +1 -0
@@ -50,5 +50,6 @@ void rfifolock_init(RFifoLock *r, void (*cb)(void *), void *opaque);void rfifolock_destroy (RFifoLock *r);void rfifolock_lock (RFifoLock *r);void rfifolock_unlock (RFifoLock *r); +int rfifolock_is_locked (RFifoLock *r);#endif /* QEMU_RFIFOLOCK_H */
util/rfifolock.c +7 -2
@@ -48,7 +48,7 @@ void rfifolock_lock(RFifoLock *r)/* Take a ticket */ unsigned int ticket = r->tail++; -if (r->nesting >0 &&qemu_thread_is_self (&r->owner_thread)) { +if (r->nesting >0 &&rfifolock_is_locked (r)) { r->tail--;/* put ticket back, we're nesting */ }else {while (ticket != r->head) {
@@ -69,10 +69,15 @@ void rfifolock_unlock(RFifoLock *r) {qemu_mutex_lock (&r->lock);assert (r->nesting >0 ); -assert (qemu_thread_is_self (&r->owner_thread)); +assert (rfifolock_is_locked (r));if (--r->nesting ==0 ) { r->head++;qemu_cond_broadcast (&r->cond); }qemu_mutex_unlock (&r->lock); } + +int rfifolock_is_locked (RFifoLock *r) +{ +return qemu_thread_is_self (&r->owner_thread); +}
Signature · 1 line
2.1.4
PATCH v2 2/5 aio_context: create aio_context_is_locked helper
In reply to Denis V. LunevThis helper is necessary to ensure locking constraints.
--- async.c | 5 +++++ include/block/aio.h | 3 +++ 2 files changed, 8 insertions(+)
async.c +5 -0
@@ -361,3 +361,8 @@ void aio_context_release(AioContext *ctx) {rfifolock_unlock (&ctx->lock); } + +int aio_context_is_locked (AioContext *ctx) +{ +return rfifolock_is_locked (&ctx->lock); +}
include/block/aio.h +3 -0
@@ -166,6 +166,9 @@ void aio_context_acquire(AioContext *ctx);/* Relinquish ownership of the AioContext. */ void aio_context_release (AioContext *ctx); +/* Check that AioContext is locked by the caller. */ +int aio_context_is_locked (AioContext *ctx); +/** * aio_bh_new: Allocate a new bottom half structure. *
Signature · 1 line
2.1.4
PATCH v2 3/5 io: add locking constraints check into bdrv_drain to ensure locking
In reply to Denis V. Lunevas described in the comment of the function
--- block/io.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
block/io.c +4 -1
@@ -247,12 +247,15 @@ bool bdrv_requests_pending(BlockDriverState *bs)void bdrv_drain (BlockDriverState *bs) {bool busy = true; +AioContext *aio_context =bdrv_get_aio_context (bs); + +assert (aio_context_is_locked (aio_context));while (busy) {/* Keep iterating */ bdrv_flush_io_queue (bs); busy =bdrv_requests_pending (bs); - busy |=aio_poll (bdrv_get_aio_context (bs), busy); + busy |=aio_poll (aio_context, busy); } }
Signature · 1 line
2.1.4
PATCH v2 4/5 migration: add missed aio_context_acquire into hmp_savevm/hmp_delvm
In reply to Denis V. Lunevaio_context should be locked in the similar way as was done in QMP snapshot creation in the other case there are a lot of possible troubles if native AIO mode is enabled for disk.
- the command can hang (HMP thread) with missed wakeup (the operation is actually complete)
io_submit
ioq_submit
laio_submit
raw_aio_submit
raw_aio_readv
bdrv_co_io_em
bdrv_co_readv_em
bdrv_aligned_preadv
bdrv_co_do_preadv
bdrv_co_do_readv
bdrv_co_readv
qcow2_co_readv
bdrv_aligned_preadv
bdrv_co_do_pwritev
bdrv_rw_co_entry
- QEMU can assert in coroutine re-enter
__GI_abort
qemu_coroutine_enter
bdrv_co_io_em_complete
qemu_laio_process_completion
qemu_laio_completion_bh
aio_bh_poll
aio_dispatch
aio_poll
iothread_runAioContext lock is reqursive. Thus nested locking should not be a problem.
--- block/snapshot.c | 5 +++++ migration/savevm.c | 7 +++++++ 2 files changed, 12 insertions(+)
block/snapshot.c +5 -0
@@ -259,6 +259,9 @@ void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs, {int ret;Error *local_err =NULL ; +AioContext *aio_context =bdrv_get_aio_context (bs); + +aio_context_acquire (aio_context); ret =bdrv_snapshot_delete (bs, id_or_name,NULL , &local_err);if (ret == -ENOENT || ret == -EINVAL ) {
@@ -267,6 +270,8 @@ void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs, ret =bdrv_snapshot_delete (bs,NULL , id_or_name, &local_err); } +aio_context_release (aio_context); +if (ret <0 ) {error_propagate (errp, local_err); }
migration/savevm.c +6 -0
@@ -1289,6 +1289,7 @@ void hmp_savevm(Monitor *mon, const QDict *qdict)struct tm tm;const char *name =qdict_get_try_str (qdict,"name" );Error *local_err =NULL ; +AioContext *aio_context;/* Verify if there is a device that doesn't support snapshots and is writable */ bs =NULL ;
@@ -1320,6 +1321,9 @@ void hmp_savevm(Monitor *mon, const QDict *qdict) }vm_stop (RUN_STATE_SAVE_VM ); + aio_context =bdrv_get_aio_context (bs); +aio_context_acquire (aio_context); +memset (sn,0 ,sizeof (*sn));/* fill auxiliary fields */
@@ -1378,6 +1382,8 @@ void hmp_savevm(Monitor *mon, const QDict *qdict) } the_end: +aio_context_release (aio_context); +if (saved_vm_running) {vm_start (); }
Re: [Qemu-devel] [PATCH 4/5] migration: add missed aio_context_acquire into hmp_savevm/hmp_delvm
Quoted 60 lines · Denis V. Lunev
aio_contextshould be locked in the similar way as was done in QMP snapshot creation in the other case there are a lot of possible troubles if native AIO mode is enabled for disk.- the command can hang (HMP thread) with missed wakeup (the operation is actually complete)
io_submit
ioq_submit
laio_submit
raw_aio_submit
raw_aio_readv
bdrv_co_io_em
bdrv_co_readv_em
bdrv_aligned_preadv
bdrv_co_do_preadv
bdrv_co_do_readv
bdrv_co_readv
qcow2_co_readv
bdrv_aligned_preadv
bdrv_co_do_pwritev
bdrv_rw_co_entry- QEMU can assert in coroutine re-enter
__GI_abort qemu_coroutine_enter bdrv_co_io_em_complete qemu_laio_process_completion qemu_laio_completion_bh aio_bh_poll aio_dispatch aio_poll iothread_runAioContext lock is reqursive. Thus nested locking should not be a problem.
Signed-off-by Denis V. Lunev <[email protected]>CC Stefan Hajnoczi <[email protected]>CC Paolo Bonzini <[email protected]>CC Juan Quintela <[email protected]>CC Amit Shah <[email protected]>--- block/snapshot.c | 5 +++++ migration/savevm.c | 7 +++++++ 2 files changed, 12 insertions(+)block/snapshot.c +3
-0@@ -259,6 +259,9 @@ void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs, {int ret;Error *local_err =NULL ; +AioContext *aio_context =bdrv_get_aio_context (bs); + +aio_context_acquire (aio_context); ret =bdrv_snapshot_delete (bs, id_or_name,NULL , &local_err);if (ret == -ENOENT || ret == -EINVAL ) {
block/snapshot.c void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs, · line 273
Quoted 4 lines
block/snapshot.c +1
-0@@ -267,6 +270,8 @@ void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs, ret =bdrv_snapshot_delete (bs,NULL , id_or_name, &local_err); } +aio_context_release (aio_context);
Why here and not in hmp_delvm, for consistency?
The call from hmp_savevm is already protected.
Thanks for fixing the bug!
Paolo
Quoted 36 lines
block/snapshot.c +0
-0if (ret <0 ) {error_propagate (errp, local_err); }migration/savevm.c +6
-0@@ -1289,6 +1289,7 @@ void hmp_savevm(Monitor *mon, const QDict *qdict)struct tm tm;const char *name =qdict_get_try_str (qdict,"name" );Error *local_err =NULL ; +AioContext *aio_context;/* Verify if there is a device that doesn't support snapshots and is writable */ bs =NULL ;@@ -1320,6 +1321,9 @@ void hmp_savevm(Monitor *mon, const QDict *qdict) }vm_stop (RUN_STATE_SAVE_VM ); + aio_context =bdrv_get_aio_context (bs); +aio_context_acquire (aio_context); +memset (sn,0 ,sizeof (*sn));/* fill auxiliary fields */ @@ -1378,6 +1382,8 @@ void hmp_savevm(Monitor *mon, const QDict *qdict) } the_end: +aio_context_release (aio_context); +if (saved_vm_running) {vm_start (); }
Re: [Qemu-devel] [PATCH 4/5] migration: add missed aio_context_acquire into hmp_savevm/hmp_delvm
Quoted 65 lines · Paolo Bonzini
Quoted 63 lines · Denis V. Lunev
aio_contextshould be locked in the similar way as was done in QMP snapshot creation in the other case there are a lot of possible troubles if native AIO mode is enabled for disk.- the command can hang (HMP thread) with missed wakeup (the operation is actually complete)
io_submit
ioq_submit
laio_submit
raw_aio_submit
raw_aio_readv
bdrv_co_io_em
bdrv_co_readv_em
bdrv_aligned_preadv
bdrv_co_do_preadv
bdrv_co_do_readv
bdrv_co_readv
qcow2_co_readv
bdrv_aligned_preadv
bdrv_co_do_pwritev
bdrv_rw_co_entry- QEMU can assert in coroutine re-enter
__GI_abort qemu_coroutine_enter bdrv_co_io_em_complete qemu_laio_process_completion qemu_laio_completion_bh aio_bh_poll aio_dispatch aio_poll iothread_runAioContext lock is reqursive. Thus nested locking should not be a problem.
Signed-off-by Denis V. Lunev <[email protected]>CC Stefan Hajnoczi <[email protected]>CC Paolo Bonzini <[email protected]>CC Juan Quintela <[email protected]>CC Amit Shah <[email protected]>--- block/snapshot.c | 5 +++++ migration/savevm.c | 7 +++++++ 2 files changed, 12 insertions(+)block/snapshot.c +3
-0@@ -259,6 +259,9 @@ void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs, {int ret;Error *local_err =NULL ; +AioContext *aio_context =bdrv_get_aio_context (bs); + +aio_context_acquire (aio_context); ret =bdrv_snapshot_delete (bs, id_or_name,NULL , &local_err);if (ret == -ENOENT || ret == -EINVAL ) {@@ -267,6 +270,8 @@ void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs, ret =bdrv_snapshot_delete (bs,NULL , id_or_name, &local_err); }
Quoted 8 lines
block/snapshot.c void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs, · line 273
Quoted 1 line
block/snapshot.c +1
-0+aio_context_release (aio_context);Why here and not in
hmp_delvm, for consistency?The call from
hmp_savevmis already protected.Thanks for fixing the bug!
Paolo
the situation is more difficult. There are several disks in VM. One disk is used for state saving (protected in savevm) and there are several disks touched via
static int del_existing_snapshots(Monitor *mon, const char *name)
while ((bs = bdrv_next(bs))) {
if (bdrv_can_snapshot(bs) &&
bdrv_snapshot_find(bs, snapshot, name) >= 0) {
bdrv_snapshot_delete_by_id_or_name(bs, name, &err);
}
}in savevm and similar looking code in delvm with similar cycle implemented differently.
This patchset looks minimal for me to kludge situation enough.
True fix would be a drop of this code in favour of blockdev transactions. At least this is my opinion. Though I can not do this at this stage, this will take a lot of time.
Den
Re: [Qemu-devel] [PATCH 4/5] migration: add missed aio_context_acquire into hmp_savevm/hmp_delvm
In reply to Denis V. LunevQuoted 40 lines · Denis V. Lunev
aio_contextshould be locked in the similar way as was done in QMP snapshot creation in the other case there are a lot of possible troubles if native AIO mode is enabled for disk.- the command can hang (HMP thread) with missed wakeup (the operation is actually complete)
io_submit
ioq_submit
laio_submit
raw_aio_submit
raw_aio_readv
bdrv_co_io_em
bdrv_co_readv_em
bdrv_aligned_preadv
bdrv_co_do_preadv
bdrv_co_do_readv
bdrv_co_readv
qcow2_co_readv
bdrv_aligned_preadv
bdrv_co_do_pwritev
bdrv_rw_co_entry- QEMU can assert in coroutine re-enter
__GI_abort qemu_coroutine_enter bdrv_co_io_em_complete qemu_laio_process_completion qemu_laio_completion_bh aio_bh_poll aio_dispatch aio_poll iothread_runAioContext lock is reqursive. Thus nested locking should not be a problem.
Signed-off-by Denis V. Lunev <[email protected]>CC Stefan Hajnoczi <[email protected]>CC Paolo Bonzini <[email protected]>CC Juan Quintela <[email protected]>CC Amit Shah <[email protected]>
Quoted 4 lines
--- block/snapshot.c | 5 +++++ migration/savevm.c | 7 +++++++ 2 files changed, 12 insertions(+)
But once there, I can't understand why migration have to know about aio_contexts at all.
I *think* that it would be a good idea to "hide" the adi_context_acquire(aio_context) inside qemu_fopen_bdrv() (yes, it is still in migration/*, but you get the idea). But don't propose it, because we don't have qemu_fclose_bdrv(). Yes we could add an aio_context inside QemuFile, and release it on qemu_fclose(), but I guess this needs more thought yet.
BTW, once that I got your attention, why is this needed on hmp_savevm() but it is not needed on load_vmstate()? We are also using qemu_fopen_bdrv()? Because we are only reading from there? Just curios the reason or if we are missing something there.
Thanks, Juan.
Quoted 55 lines
block/snapshot.c +5
-0@@ -259,6 +259,9 @@ void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs, {int ret;Error *local_err =NULL ; +AioContext *aio_context =bdrv_get_aio_context (bs); + +aio_context_acquire (aio_context); ret =bdrv_snapshot_delete (bs, id_or_name,NULL , &local_err);if (ret == -ENOENT || ret == -EINVAL ) {@@ -267,6 +270,8 @@ void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs, ret =bdrv_snapshot_delete (bs,NULL , id_or_name, &local_err); } +aio_context_release (aio_context); +if (ret <0 ) {error_propagate (errp, local_err); }migration/savevm.c +6
-0@@ -1289,6 +1289,7 @@ void hmp_savevm(Monitor *mon, const QDict *qdict)struct tm tm;const char *name =qdict_get_try_str (qdict,"name" );Error *local_err =NULL ; +AioContext *aio_context;/* Verify if there is a device that doesn't support snapshots and is writable */ bs =NULL ;@@ -1320,6 +1321,9 @@ void hmp_savevm(Monitor *mon, const QDict *qdict) }vm_stop (RUN_STATE_SAVE_VM ); + aio_context =bdrv_get_aio_context (bs); +aio_context_acquire (aio_context); +memset (sn,0 ,sizeof (*sn));/* fill auxiliary fields */ @@ -1378,6 +1382,8 @@ void hmp_savevm(Monitor *mon, const QDict *qdict) } the_end: +aio_context_release (aio_context); +if (saved_vm_running) {vm_start (); }
Re: [Qemu-devel] [PATCH 4/5] migration: add missed aio_context_acquire into hmp_savevm/hmp_delvm
Quoted 59 lines · Juan Quintela
Quoted 40 lines · Denis V. Lunev
aio_contextshould be locked in the similar way as was done in QMP snapshot creation in the other case there are a lot of possible troubles if native AIO mode is enabled for disk.- the command can hang (HMP thread) with missed wakeup (the operation is actually complete)
io_submit
ioq_submit
laio_submit
raw_aio_submit
raw_aio_readv
bdrv_co_io_em
bdrv_co_readv_em
bdrv_aligned_preadv
bdrv_co_do_preadv
bdrv_co_do_readv
bdrv_co_readv
qcow2_co_readv
bdrv_aligned_preadv
bdrv_co_do_pwritev
bdrv_rw_co_entry- QEMU can assert in coroutine re-enter
__GI_abort qemu_coroutine_enter bdrv_co_io_em_complete qemu_laio_process_completion qemu_laio_completion_bh aio_bh_poll aio_dispatch aio_poll iothread_runAioContext lock is reqursive. Thus nested locking should not be a problem.
Signed-off-by Denis V. Lunev <[email protected]>CC Stefan Hajnoczi <[email protected]>CC Paolo Bonzini <[email protected]>CC Juan Quintela <[email protected]>CC Amit Shah <[email protected]>Quoted 4 lines
--- block/snapshot.c | 5 +++++ migration/savevm.c | 7 +++++++ 2 files changed, 12 insertions(+)Reviewed-by Juan Quintela <[email protected]>But once there, I can't understand why migration have to know about
aio_contextsat all.I *think* that it would be a good idea to "hide" the
adi_context_acquire(aio_context)insideqemu_fopen_bdrv()(yes, it is still in migration/*, but you get the idea). But don't propose it, because we don't haveqemu_fclose_bdrv(). Yes we could add anaio_contextinside QemuFile, and release it onqemu_fclose(), but I guess this needs more thought yet.
Quoted 6 lines
BTW, once that I got your attention, why is this needed on
hmp_savevm()but it is not needed onload_vmstate()? We are also usingqemu_fopen_bdrv()? Because we are only reading from there? Just curios the reason or if we are missing something there.Thanks, Juan.
I think that the race is still there (I have checked this several times but less amount of times then create/delete snapshot) but the windows is seriously reduced due to bdrv_drain_all at the beginning.
In general your are right. But in this case we are almost doomed. Any single read/write operation could executed in iothread only. May be I have missed something in this puzzle.
OK. What about bdrv_fclose callback and similar (new) callback for open which should be called through qemu_fopen_bdrv for block driver only?
Den
PATCH v2 5/5 virtio: sync the dataplane vring state to the virtqueue before virtio_save
In reply to Denis V. LunevFrom: Pavel Butsykin <[email protected]>
When creating snapshot with the dataplane enabled, the snapshot file gets not the actual state of virtqueue, because the current state is stored in VirtIOBlockDataPlane. Therefore, before saving snapshot need to sync the dataplane vring state to the virtqueue. The dataplane will resume its work at the next notify virtqueue.
When snapshot loads with loadvm we get a message:
VQ 0 size 0x80 Guest index 0x15f5 inconsistent with Host index 0x0:
delta 0x15f5
error while loading state for instance 0x0 of device
'0000:00:08.0/virtio-blk'Error -1 while loading VM state
--- hw/block/virtio-blk.c | 5 +++++ hw/scsi/virtio-scsi.c | 5 +++++ 2 files changed, 10 insertions(+)
hw/block/virtio-blk.c +5 -0
@@ -798,6 +798,11 @@ static void virtio_blk_set_status(VirtIODevice *vdev, uint8_t status)static void virtio_blk_save (QEMUFile *f,void *opaque) {VirtIODevice *vdev =VIRTIO_DEVICE (opaque); +VirtIOBlock *s =VIRTIO_BLK (vdev); + +if (s->dataplane) { +virtio_blk_data_plane_stop (s->dataplane); + }virtio_save (vdev, f); }
hw/scsi/virtio-scsi.c +5 -0
@@ -665,6 +665,11 @@ static void virtio_scsi_reset(VirtIODevice *vdev)static void virtio_scsi_save (QEMUFile *f,void *opaque) {VirtIODevice *vdev =VIRTIO_DEVICE (opaque); +VirtIOSCSI *s =VIRTIO_SCSI (vdev); + +if (s->dataplane_started) { +virtio_scsi_dataplane_stop (s); + }virtio_save (vdev, f); }
Signature · 1 line
2.1.4
Quoted 17 lines · Denis V. Lunev
The following test
while/bin/true; do
virsh snapshot-create rhel7
sleep 10
virsh snapshot-delete rhel7 --current
done
with enabled iothreads on a running VM leads to a lot of troubles: hangs, asserts, errors.Though (in general) HMP snapshot code is terrible. I think it should be dropped at once and replaced with blkdev transactions code. Though is could not fit to QEMU 2.5/stable at all.
Anyway, I think that the construction like
assert(aio_context_is_locked(aio_context));should be widely used to ensure proper locking.
Quoted 7 lines
Changes from v1:
- aio-context locking added
- comment is rewrittenSigned-off-by Denis V. Lunev <[email protected]>CC Stefan Hajnoczi <[email protected]>CC Paolo Bonzini <[email protected]>
For patches 4-5:
For patches 1-3 I'm not sure, because we will remove RFifoLock relatively soon and regular pthread recursive mutexes do not have an equivalent of rfifolock_is_locked.
Paolo
Quoted 27 lines · Paolo Bonzini
Quoted 17 lines · Denis V. Lunev
The following test
while/bin/true; do
virsh snapshot-create rhel7
sleep 10
virsh snapshot-delete rhel7 --current
done
with enabled iothreads on a running VM leads to a lot of troubles: hangs, asserts, errors.Though (in general) HMP snapshot code is terrible. I think it should be dropped at once and replaced with blkdev transactions code. Though is could not fit to QEMU 2.5/stable at all.
Anyway, I think that the construction like
assert(aio_context_is_locked(aio_context));should be widely used to ensure proper locking.
Quoted 7 lines
Changes from v1:
- aio-context locking added
- comment is rewrittenSigned-off-by Denis V. Lunev <[email protected]>CC Stefan Hajnoczi <[email protected]>CC Paolo Bonzini <[email protected]>For patches 4-5:
Quoted 7 lines
Reviewed-by Paolo Bonzini <[email protected]>For patches 1-3 I'm not sure, because we will remove RFifoLock relatively soon and regular pthread recursive mutexes do not have an equivalent of
rfifolock_is_locked.Paolo
This does not break any future.
Yes, FifoLock will go away, but aio_context_is_locked will survive like it stays in the kernel code. We can either have plain pthread_mutex_try_lock/unlock at first or we can have additional stubs for linux with checks like this
(gdb) p *(pthread_mutex_t*)0x6015a0
$3 = {
__data = {
__lock = 2,
__count = 0,
__owner = 12276, <== LWP12276 is Thread 3
__nusers = 1,
__kind = 0, <== non-recursive
__spins = 0,
__list = {
__prev = 0x0,
__next = 0x0
} },
__size = "\002\000\000\000\000\000\000\000\364/\000\000\001",'\000' <repeats26 times>,
__align = 2
}
in debug mode. Yes, they relays on internal representation, but they are useful.
This assert was VERY useful for me. I presume that there are a LOT of similar places in the code with different functions where aio_context lock was not acquired and there was no way to ensure consistency.
Den
Quoted 65 lines · Denis V. Lunev
Quoted 27 lines · Paolo Bonzini
Quoted 17 lines · Denis V. Lunev
The following test
while/bin/true; do
virsh snapshot-create rhel7
sleep 10
virsh snapshot-delete rhel7 --current
done
with enabled iothreads on a running VM leads to a lot of troubles: hangs, asserts, errors.Though (in general) HMP snapshot code is terrible. I think it should be dropped at once and replaced with blkdev transactions code. Though is could not fit to QEMU 2.5/stable at all.
Anyway, I think that the construction like
assert(aio_context_is_locked(aio_context));should be widely used to ensure proper locking.
Quoted 7 lines
Changes from v1:
- aio-context locking added
- comment is rewrittenSigned-off-by Denis V. Lunev <[email protected]>CC Stefan Hajnoczi <[email protected]>CC Paolo Bonzini <[email protected]>For patches 4-5:
Quoted 7 lines
Reviewed-by Paolo Bonzini <[email protected]>For patches 1-3 I'm not sure, because we will remove RFifoLock relatively soon and regular pthread recursive mutexes do not have an equivalent of
rfifolock_is_locked.Paolo
This does not break any future.
Yes, FifoLock will go away, but
aio_context_is_lockedwill survive like it stays in the kernel code. We can either have plain pthread_mutex_try_lock/unlock at first or we can have additional stubs for linux with checks like this(gdb) p *(pthread_mutex_t*)0x6015a0
$3 = {__data = { __lock = 2, __count = 0, __owner = 12276, <== LWP12276 is Thread 3 __nusers = 1, __kind = 0, <== non-recursive __spins = 0, __list = { __prev = 0x0, __next = 0x0 }},
__size= "\002\000\000\000\000\000\000\000\364/\000\000\001",'\000'
<repeats26 times>,
__align= 2
}in debug mode. Yes, they relays on internal representation, but they are useful.
Quoted 6 lines
This assert was VERY useful for me. I presume that there are a LOT of similar places in the code with different functions where
aio_contextlock was not acquired and there was no way to ensure consistency.Den
there is a sane fast crossplatform approach :)
struct AioContextLock {
pthread_t owner;
pthread_mutex_t mutex;
int count;
};void aio_context_lock(struct AioContextLock *lock)
{
pthread_mutex_lock(&lock->mutex);
lock->owner = pthread_self();
lock->count++;
}void aio_context_unlock(struct AioContextLock *lock)
{
if (--lock->count == 0) {
lock->owner = NULL;
}pthread_mutex_lock(&lock->mutex);
}
int aio_context_is_locked(struct AioContextLock *lock)
{
if (pthread_mutex_trylock(&lock->mutex)) {
int own_by_us = lock->owner == pthread_self();
pthread_mutex_unlock(&lock->mutex);
return own_by_us ? 1 : 0;
}
/* the lock was taken by other thread */
return 0;
}Den