blkdebug: fix racing condition when iterating on
v3 cover blkdebug: fix racing condition when iterating on
When qemu_coroutine_enter is executed in a loop
(even QEMU_FOREACH_SAFE), the new routine can modify the list, for example removing an element, causing problem when control is given back to the caller that continues iterating on the same list.
Patch 1 solves the issue in blkdebug_debug_resume by restarting the list walk after every coroutine_enter if list has to be fully iterated. Patches 2,3,4 aim to fix blkdebug_debug_event by gathering
all actions that the rules make in a counter and invoking
the respective coroutine_yeld only after processing all requests.
Patch 5 is somewhat independent of the others, it adds a lock to protect rules and suspended_reqs; right now everything works because it's protected by the AioContext lock.
This is a preparation for the current proposal of removing the AioContext lock and instead using smaller granularity locks to allow multiple iothread execution in the same block device.
---
v2 -> v3
* Fix "yeld"->"yield" in patches 3-4 [Eric]
* Use lock guard instead of lock/unlock in patch 5 [Eric]
Emanuele Giuseppe Esposito (5):
blkdebug: refactor removal of a suspended request
blkdebug: move post-resume handling to resume_req_by_tag
blkdebug: track all actions
blkdebug: do not suspend in the middle of QLIST_FOREACH_SAFE
blkdebug: protect rules and suspended_reqs with a lock
block/blkdebug.c | 124 +++++++++++++++++++++++++++++++---------------- 1 file changed, 83 insertions(+), 41 deletions(-)
Signature · 1 line
2.30.2
PATCH v3 1/5 blkdebug: refactor removal of a suspended request
Extract to a separate function. Do not rely on FOREACH_SAFE, which is only "safe" if the *current* node is removed---not if another node is removed. Instead, just walk the entire list from the beginning when asked to resume all suspended requests with a given tag.
--- block/blkdebug.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-)
block/blkdebug.c +17 -11
@@ -793,7 +793,6 @@ static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule)printf ("blkdebug: Resuming request '%s'\n" , r.tag); } -QLIST_REMOVE (&r, next);g_free (r.tag); }
@@ -869,25 +868,35 @@ static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event,return 0 ; } -static int blkdebug_debug_resume (BlockDriverState *bs,const char *tag) +static int resume_req_by_tag (BDRVBlkdebugState *s,const char *tag,bool all) { -BDRVBlkdebugState *s = bs->opaque; -BlkdebugSuspendedReq *r, *next; +BlkdebugSuspendedReq *r; - QLIST_FOREACH_SAFE(r, &s->suspended_reqs, next, next) { +retry: + QLIST_FOREACH(r, &s->suspended_reqs, next) {if (!strcmp (r->tag, tag)) { +QLIST_REMOVE (r, next);qemu_coroutine_enter (r->co); +if (all) { + goto retry; + }return 0 ; } }return -ENOENT ; } +static int blkdebug_debug_resume (BlockDriverState *bs,const char *tag) +{ +BDRVBlkdebugState *s = bs->opaque; + +return resume_req_by_tag (s, tag, false); +} +static int blkdebug_debug_remove_breakpoint (BlockDriverState *bs,const char *tag) {BDRVBlkdebugState *s = bs->opaque; -BlkdebugSuspendedReq *r, *r_next;BlkdebugRule *rule, *next;int i, ret = -ENOENT ;
@@ -900,11 +909,8 @@ static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs, } } } - QLIST_FOREACH_SAFE(r, &s->suspended_reqs, next, r_next) { -if (!strcmp (r->tag, tag)) { -qemu_coroutine_enter (r->co); - ret =0 ; - } +if (resume_req_by_tag (s, tag, true) ==0 ) { + ret =0 ; }return ret; }
Signature · 1 line
2.30.2
Re: [PATCH v3 1/5] blkdebug: refactor removal of a suspended request
Quoted 69 lines · Emanuele Giuseppe Esposito
Extract to a separate function. Do not rely on
FOREACH_SAFE, which is only "safe" if the *current* node is removed---not if another node is removed. Instead, just walk the entire list from the beginning when asked to resume all suspended requests with a given tag.Co-developed-by Paolo Bonzini <[email protected]>Signed-off-by Emanuele Giuseppe Esposito <[email protected]>--- block/blkdebug.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-)block/blkdebug.c +15
-6@@ -793,7 +793,6 @@ static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule)printf ("blkdebug: Resuming request '%s'\n" , r.tag); } -QLIST_REMOVE (&r, next);g_free (r.tag); }@@ -869,25 +868,35 @@ static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event,return 0 ; } -static int blkdebug_debug_resume (BlockDriverState *bs,const char *tag) +static int resume_req_by_tag (BDRVBlkdebugState *s,const char *tag,bool all) { -BDRVBlkdebugState *s = bs->opaque; -BlkdebugSuspendedReq *r, *next; +BlkdebugSuspendedReq *r; - QLIST_FOREACH_SAFE(r, &s->suspended_reqs, next, next) { +retry: + QLIST_FOREACH(r, &s->suspended_reqs, next) {if (!strcmp (r->tag, tag)) { +QLIST_REMOVE (r, next);qemu_coroutine_enter (r->co); +if (all) { + goto retry; + }return 0 ; } }return -ENOENT ; } +static int blkdebug_debug_resume (BlockDriverState *bs,const char *tag) +{ +BDRVBlkdebugState *s = bs->opaque; + +return resume_req_by_tag (s, tag, false); +} +static int blkdebug_debug_remove_breakpoint (BlockDriverState *bs,const char *tag) {BDRVBlkdebugState *s = bs->opaque; -BlkdebugSuspendedReq *r, *r_next;BlkdebugRule *rule, *next;int i, ret = -ENOENT ;@@ -900,11 +909,8 @@ static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs, } } }
block/blkdebug.c static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs, · line 916
Quoted 10 lines
block/blkdebug.c +2
-5- QLIST_FOREACH_SAFE(r, &s->suspended_reqs, next, r_next) { -if (!strcmp (r->tag, tag)) { -qemu_coroutine_enter (r->co); - ret =0 ; - } +if (resume_req_by_tag (s, tag, true) ==0 ) { + ret =0 ; }return ret; }
Interesting, could we really have several suspended_reqs with same tag, keeping in mind suspend_requests() removes rule before creating suspended_req with same tag.. Probably user could create new rule with same tag, when existing requests is suspended? Not sure is such behavior expected or should be abandoned. Still, it's all not about that patch:
Signature · 2 lines
Best regards, Vladimir
PATCH v3 2/5 blkdebug: move post-resume handling to resume_req_by_tag
In reply to Emanuele Giuseppe EspositoWe want to move qemu_coroutine_yield() after the loop on rules, because QLIST_FOREACH_SAFE is wrong if the rule list is modified while the coroutine has yielded. Therefore move the suspended request to the heap and clean it up from the remove side. All that is left is for blkdebug_debug_event to handle the yielding.
--- block/blkdebug.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-)
block/blkdebug.c +18 -13
@@ -775,25 +775,20 @@ static void blkdebug_close(BlockDriverState *bs)static void suspend_request (BlockDriverState *bs,BlkdebugRule *rule) {BDRVBlkdebugState *s = bs->opaque; -BlkdebugSuspendedReq r; +BlkdebugSuspendedReq *r; - r = (BlkdebugSuspendedReq ) { - .co =qemu_coroutine_self (), - .tag =g_strdup (rule->options.suspend.tag), - }; + r =g_new (BlkdebugSuspendedReq,1 ); + + r->co =qemu_coroutine_self (); + r->tag =g_strdup (rule->options.suspend.tag);remove_rule (rule); -QLIST_INSERT_HEAD (&s->suspended_reqs, &r, next); +QLIST_INSERT_HEAD (&s->suspended_reqs, r, next);if (!qtest_enabled ()) { -printf ("blkdebug: Suspended request '%s'\n" , r.tag); +printf ("blkdebug: Suspended request '%s'\n" , r->tag); }qemu_coroutine_yield (); -if (!qtest_enabled ()) { -printf ("blkdebug: Resuming request '%s'\n" , r.tag); - } - -g_free (r.tag); } static bool process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,
@@ -875,8 +870,18 @@ static int resume_req_by_tag(BDRVBlkdebugState *s, const char *tag, bool all) retry: QLIST_FOREACH(r, &s->suspended_reqs, next) {if (!strcmp (r->tag, tag)) { +Coroutine *co = r->co; + +if (!qtest_enabled ()) { +printf ("blkdebug: Resuming request '%s'\n" , r->tag); + } +QLIST_REMOVE (r, next); -qemu_coroutine_enter (r->co); +g_free (r->tag); +g_free (r); + +qemu_coroutine_enter (co); +if (all) { goto retry; }
Signature · 1 line
2.30.2
Re: [PATCH v3 2/5] blkdebug: move post-resume handling to resume_req_by_tag
Quoted 9 lines · Emanuele Giuseppe Esposito
We want to move
qemu_coroutine_yield()after the loop on rules, becauseQLIST_FOREACH_SAFEis wrong if the rule list is modified while the coroutine has yielded. Therefore move the suspended request to the heap and clean it up from the remove side. All that is left is forblkdebug_debug_eventto handle the yielding.Co-developed-by Paolo Bonzini<[email protected]>Signed-off-by Emanuele Giuseppe Esposito<[email protected]>
Signature · 2 lines
Best regards, Vladimir
PATCH v3 3/5 blkdebug: track all actions
In reply to Emanuele Giuseppe EspositoAdd a counter for each action that a rule can trigger. This is mainly used to keep track of how many coroutine_yield() we need to perform after processing all rules in the list.
--- block/blkdebug.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-)
block/blkdebug.c +8 -9
@@ -74,6 +74,7 @@ enum {ACTION_INJECT_ERROR ,ACTION_SET_STATE ,ACTION_SUSPEND , + ACTION__MAX, }; typedef struct BlkdebugRule {
@@ -791,22 +792,22 @@ static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule)qemu_coroutine_yield (); } -static bool process_rule (BlockDriverState *bs,struct BlkdebugRule *rule, -bool injected) +static void process_rule (BlockDriverState *bs,struct BlkdebugRule *rule, +int *action_count) {BDRVBlkdebugState *s = bs->opaque;/* Only process rules for the current state */ if (rule->state && rule->state != s->state) { -return injected; +return ; }/* Take the action */ + action_count[rule->action]++;switch (rule->action) {case ACTION_INJECT_ERROR : -if (!injected) { +if (action_count[ACTION_INJECT_ERROR ] ==1 ) {QSIMPLEQ_INIT (&s->active_rules); - injected = true; }QSIMPLEQ_INSERT_HEAD (&s->active_rules, rule, active_next);break ;
@@ -819,21 +820,19 @@ static bool process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,suspend_request (bs, rule);break ; } -return injected; }static void blkdebug_debug_event (BlockDriverState *bs,BlkdebugEvent event) {BDRVBlkdebugState *s = bs->opaque;struct BlkdebugRule *rule, *next; -bool injected; +int actions_count[ACTION__MAX ] = {0 };assert ((int )event >=0 && event <BLKDBG__MAX ); - injected = false; s->new_state = s->state; QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) { - injected =process_rule (bs, rule, injected); +process_rule (bs, rule, actions_count); } s->state = s->new_state; }
Signature · 1 line
2.30.2
Re: [PATCH v3 3/5] blkdebug: track all actions
Quoted 6 lines · Emanuele Giuseppe Esposito
Add a counter for each action that a rule can trigger. This is mainly used to keep track of how many
coroutine_yield()we need to perform after processing all rules in the list.Co-developed-by Paolo Bonzini<[email protected]>Signed-off-by Emanuele Giuseppe Esposito<[email protected]>
Signature · 2 lines
Best regards, Vladimir
PATCH v3 4/5 blkdebug: do not suspend in the middle of QLIST_FOREACH_SAFE
In reply to Emanuele Giuseppe EspositoThat would be unsafe in case a rule other than the current one is removed while the coroutine has yielded.
Keep FOREACH_SAFE because suspend_request deletes the current rule.
After this patch, *all* matching rules are deleted before suspending the coroutine, rather than just one.
This doesn't affect the existing testcases.
Use actions_count to see how many yield to issue.
--- block/blkdebug.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)
block/blkdebug.c +6 -1
@@ -789,7 +789,6 @@ static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule)if (!qtest_enabled ()) {printf ("blkdebug: Suspended request '%s'\n" , r->tag); } -qemu_coroutine_yield (); } static void process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,
@@ -834,6 +833,12 @@ static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event) QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) {process_rule (bs, rule, actions_count); } + +while (actions_count[ACTION_SUSPEND ] >0 ) { +qemu_coroutine_yield (); + actions_count[ACTION_SUSPEND ]--; + } + s->state = s->new_state; }
Signature · 1 line
2.30.2
Re: [PATCH v3 4/5] blkdebug: do not suspend in the middle of QLIST_FOREACH_SAFE
Quoted 32 lines · Emanuele Giuseppe Esposito
That would be unsafe in case a rule other than the current one is removed while the coroutine has yielded.
KeepFOREACH_SAFEbecausesuspend_requestdeletes the current rule.After this patch, *all* matching rules are deleted before suspending the coroutine, rather than just one.
This doesn't affect the existing testcases.Use
actions_countto see how many yield to issue.Co-developed-by Paolo Bonzini <[email protected]>Signed-off-by Emanuele Giuseppe Esposito <[email protected]>--- block/blkdebug.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)block/blkdebug.c +0
-1@@ -789,7 +789,6 @@ static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule)if (!qtest_enabled ()) {printf ("blkdebug: Suspended request '%s'\n" , r->tag); } -qemu_coroutine_yield (); } static void process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,@@ -834,6 +833,12 @@ static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event) QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) {process_rule (bs, rule, actions_count);
block/blkdebug.c static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event) · line 844
Quoted 10 lines
block/blkdebug.c +6
-0} + +while (actions_count[ACTION_SUSPEND ] >0 ) { +qemu_coroutine_yield (); + actions_count[ACTION_SUSPEND ]--; + } + s->state = s->new_state; }
two thoughts:
- suspend_request() should probably be renamed
- actions_count logic is a bit overcomplicated, actually we need only suspend_count.
Signature · 2 lines
Best regards, Vladimir
PATCH v3 5/5 blkdebug: protect rules and suspended_reqs with a lock
In reply to Emanuele Giuseppe Esposito--- block/blkdebug.c | 53 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 13 deletions(-)
block/blkdebug.c +40 -13
@@ -54,6 +54,7 @@ typedef struct BDRVBlkdebugState {/* For blkdebug_refresh_filename() */ char *config_file; +QemuMutex lock; QLIST_HEAD(, BlkdebugRule) rules[BLKDBG__MAX]; QSIMPLEQ_HEAD(, BlkdebugRule) active_rules; QLIST_HEAD(, BlkdebugSuspendedReq) suspended_reqs;
@@ -245,7 +246,9 @@ static int add_rule(void *opaque, QemuOpts *opts, Error **errp) };/* Add the rule */ +qemu_mutex_lock (&s->lock);QLIST_INSERT_HEAD (&s->rules[event], rule, next); +qemu_mutex_unlock (&s->lock);return 0 ; }
@@ -468,6 +471,7 @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags,int ret;uint64_t align; +qemu_mutex_init (&s->lock); opts =qemu_opts_create (&runtime_opts,NULL ,0 , &error_abort);if (!qemu_opts_absorb_qdict (opts, options, errp)) { ret = -EINVAL ;
@@ -568,6 +572,7 @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags, ret =0 ; out:if (ret <0 ) { +qemu_mutex_destroy (&s->lock);g_free (s->config_file); }qemu_opts_del (opts);
@@ -582,6 +587,7 @@ static int rule_check(BlockDriverState *bs, uint64_t offset, uint64_t bytes,int error;bool immediately; +qemu_mutex_lock (&s->lock); QSIMPLEQ_FOREACH(rule, &s->active_rules, active_next) {uint64_t inject_offset = rule->options.inject.offset;
@@ -595,6 +601,7 @@ static int rule_check(BlockDriverState *bs, uint64_t offset, uint64_t bytes, }if (!rule || !rule->options.inject.error) { +qemu_mutex_unlock (&s->lock);return 0 ; }
@@ -606,6 +613,7 @@ static int rule_check(BlockDriverState *bs, uint64_t offset, uint64_t bytes,remove_rule (rule); } +qemu_mutex_unlock (&s->lock);if (!immediately) {aio_co_schedule (qemu_get_current_aio_context (),qemu_coroutine_self ());qemu_coroutine_yield ();
@@ -771,8 +779,10 @@ static void blkdebug_close(BlockDriverState *bs) }g_free (s->config_file); +qemu_mutex_destroy (&s->lock); } +/* Called with lock held. */ static void suspend_request (BlockDriverState *bs,BlkdebugRule *rule) {BDRVBlkdebugState *s = bs->opaque;
@@ -791,6 +801,7 @@ static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule) } } +/* Called with lock held. */ static void process_rule (BlockDriverState *bs,struct BlkdebugRule *rule,int *action_count) {
@@ -829,9 +840,11 @@ static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event)assert ((int )event >=0 && event <BLKDBG__MAX ); - s->new_state = s->state; - QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) { -process_rule (bs, rule, actions_count); + WITH_QEMU_LOCK_GUARD(&s->lock) { + s->new_state = s->state; + QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) { +process_rule (bs, rule, actions_count); + } }while (actions_count[ACTION_SUSPEND ] >0 ) {
@@ -839,7 +852,9 @@ static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event) actions_count[ACTION_SUSPEND ]--; } +qemu_mutex_lock (&s->lock); s->state = s->new_state; +qemu_mutex_unlock (&s->lock); } static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event,
@@ -862,11 +877,14 @@ static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event, .options.suspend.tag = g_strdup(tag), }; +qemu_mutex_lock (&s->lock);QLIST_INSERT_HEAD (&s->rules[blkdebug_event], rule, next); +qemu_mutex_unlock (&s->lock);return 0 ; } +/* Called with lock held. */ static int resume_req_by_tag (BDRVBlkdebugState *s,const char *tag,bool all) {BlkdebugSuspendedReq *r;
@@ -884,7 +902,9 @@ retry:g_free (r->tag);g_free (r); +qemu_mutex_unlock (&s->lock);qemu_coroutine_enter (co); +qemu_mutex_lock (&s->lock);if (all) { goto retry;
@@ -898,8 +918,12 @@ retry:static int blkdebug_debug_resume (BlockDriverState *bs,const char *tag) {BDRVBlkdebugState *s = bs->opaque; +int rc; -return resume_req_by_tag (s, tag, false); +qemu_mutex_lock (&s->lock); + rc =resume_req_by_tag (s, tag, false); +qemu_mutex_unlock (&s->lock); +return rc; } static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs,
@@ -909,17 +933,19 @@ static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs,BlkdebugRule *rule, *next;int i, ret = -ENOENT ; -for (i =0 ; i <BLKDBG__MAX ; i++) { - QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) { -if (rule->action ==ACTION_SUSPEND && - !strcmp (rule->options.suspend.tag, tag)) { -remove_rule (rule); - ret =0 ; + WITH_QEMU_LOCK_GUARD(&s->lock) { +for (i =0 ; i <BLKDBG__MAX ; i++) { + QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) { +if (rule->action ==ACTION_SUSPEND && + !strcmp (rule->options.suspend.tag, tag)) { +remove_rule (rule); + ret =0 ; + } } } - } -if (resume_req_by_tag (s, tag, true) ==0 ) { - ret =0 ; +if (resume_req_by_tag (s, tag, true) ==0 ) { + ret =0 ; + } }return ret; }
@@ -929,6 +955,7 @@ static bool blkdebug_debug_is_suspended(BlockDriverState *bs, const char *tag)BDRVBlkdebugState *s = bs->opaque;BlkdebugSuspendedReq *r; +QEMU_LOCK_GUARD (&s->lock); QLIST_FOREACH(r, &s->suspended_reqs, next) {if (!strcmp (r->tag, tag)) {return true;
Signature · 1 line
2.30.2
Re: [PATCH v3 5/5] blkdebug: protect rules and suspended_reqs with a lock
Quoted 11 lines · Emanuele Giuseppe Esposito
Co-developed-by Paolo Bonzini <[email protected]>Signed-off-by Emanuele Giuseppe Esposito <[email protected]>--- block/blkdebug.c | 53 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 13 deletions(-)block/blkdebug.c +0
-0
block/blkdebug.c typedef struct BDRVBlkdebugState { · line 57
Quoted 4 lines
block/blkdebug.c +1
-0@@ -54,6 +54,7 @@ typedef struct BDRVBlkdebugState {/* For blkdebug_refresh_filename() */ char *config_file; +QemuMutex lock;
we'll need a comments, which fields are protected by the lock, like in other your series.
and also a comment which functions are thread-safe after the patch.
remove_rule() lacks comment, like "Called with lock held or from .bdrv_close"
Quoted 88 lines
block/blkdebug.c +15
-3QLIST_HEAD(, BlkdebugRule) rules[BLKDBG__MAX]; QSIMPLEQ_HEAD(, BlkdebugRule) active_rules; QLIST_HEAD(, BlkdebugSuspendedReq) suspended_reqs;@@ -245,7 +246,9 @@ static int add_rule(void *opaque, QemuOpts *opts, Error **errp) };/* Add the rule */ +qemu_mutex_lock (&s->lock);QLIST_INSERT_HEAD (&s->rules[event], rule, next); +qemu_mutex_unlock (&s->lock);return 0 ; }@@ -468,6 +471,7 @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags,int ret;uint64_t align; +qemu_mutex_init (&s->lock); opts =qemu_opts_create (&runtime_opts,NULL ,0 , &error_abort);if (!qemu_opts_absorb_qdict (opts, options, errp)) { ret = -EINVAL ;@@ -568,6 +572,7 @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags, ret =0 ; out:if (ret <0 ) { +qemu_mutex_destroy (&s->lock);g_free (s->config_file); }qemu_opts_del (opts);@@ -582,6 +587,7 @@ static int rule_check(BlockDriverState *bs, uint64_t offset, uint64_t bytes,int error;bool immediately; +qemu_mutex_lock (&s->lock); QSIMPLEQ_FOREACH(rule, &s->active_rules, active_next) {uint64_t inject_offset = rule->options.inject.offset;@@ -595,6 +601,7 @@ static int rule_check(BlockDriverState *bs, uint64_t offset, uint64_t bytes, }if (!rule || !rule->options.inject.error) { +qemu_mutex_unlock (&s->lock);return 0 ; }@@ -606,6 +613,7 @@ static int rule_check(BlockDriverState *bs, uint64_t offset, uint64_t bytes,remove_rule (rule); } +qemu_mutex_unlock (&s->lock);if (!immediately) {aio_co_schedule (qemu_get_current_aio_context (),qemu_coroutine_self ());qemu_coroutine_yield ();@@ -771,8 +779,10 @@ static void blkdebug_close(BlockDriverState *bs) }g_free (s->config_file); +qemu_mutex_destroy (&s->lock); } +/* Called with lock held. */ static void suspend_request (BlockDriverState *bs,BlkdebugRule *rule) {BDRVBlkdebugState *s = bs->opaque;@@ -791,6 +801,7 @@ static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule) } } +/* Called with lock held. */ static void process_rule (BlockDriverState *bs,struct BlkdebugRule *rule,int *action_count) {@@ -829,9 +840,11 @@ static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event)assert ((int )event >=0 && event <BLKDBG__MAX ); - s->new_state = s->state; - QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) { -process_rule (bs, rule, actions_count); + WITH_QEMU_LOCK_GUARD(&s->lock) { + s->new_state = s->state; + QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) { +process_rule (bs, rule, actions_count); + } }while (actions_count[ACTION_SUSPEND ] >0 ) {
block/blkdebug.c static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event) · line 858
Quoted 7 lines
block/blkdebug.c +2
-0@@ -839,7 +852,9 @@ static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event) actions_count[ACTION_SUSPEND ]--; } +qemu_mutex_lock (&s->lock); s->state = s->new_state; +qemu_mutex_unlock (&s->lock); }
Preexising: honestly, I don't understand the logic around state and new_state.. (and therefore, I'm not sure, is it in good relation with patch 04)
It come in the commit
commit 8f96b5be92fbd74798b97b1dc1ff5fbbe249ed11
Author: Paolo Bonzini <[email protected]> Date: Fri Sep 28 17:23:00 2012 +0200
blkdebug: process all set_state rules in the old state
Currently it is impossible to write a blkdebug script that ping-pongs
between two states, because the second set-state rule will use the
state that is set in the first. If you have
[set-state]
event = "..."
state = "1"
new_state = "2"
[set-state]
event = "..."
state = "2"
new_state = "1"
for example the state will remain locked at 1. This can be fixed
by first processing all rules, and then setting the state.But I don't understand, whey it should remain locked..
And this logic is not safe: another event may happen during the yield, and will operate on the same s->state and s->new_state, leading the the mess.
Quoted 4 lines
block/blkdebug.c +0
-0static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event,@@ -862,11 +877,14 @@ static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event, .options.suspend.tag = g_strdup(tag),
block/blkdebug.c static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event, · line 887
Quoted 10 lines
block/blkdebug.c +3
-0}; +qemu_mutex_lock (&s->lock);QLIST_INSERT_HEAD (&s->rules[blkdebug_event], rule, next); +qemu_mutex_unlock (&s->lock);return 0 ; } +/* Called with lock held. */
I think, it would be a very good practice to include into convention:
May temporarily release lock.
Quoted 14 lines
block/blkdebug.c +2
-0static int resume_req_by_tag (BDRVBlkdebugState *s,const char *tag,bool all) {BlkdebugSuspendedReq *r;@@ -884,7 +902,9 @@ retry:g_free (r->tag);g_free (r); +qemu_mutex_unlock (&s->lock);qemu_coroutine_enter (co); +qemu_mutex_lock (&s->lock);if (all) { goto retry;
block/blkdebug.c retry: · line 921
Quoted 4 lines
block/blkdebug.c +1
-0@@ -898,8 +918,12 @@ retry:static int blkdebug_debug_resume (BlockDriverState *bs,const char *tag) {BDRVBlkdebugState *s = bs->opaque; +int rc;
Hmm, you can simply use QEMU_LOCK_GUARD
Quoted 10 lines
block/blkdebug.c +4
-1-return resume_req_by_tag (s, tag, false); +qemu_mutex_lock (&s->lock); + rc =resume_req_by_tag (s, tag, false); +qemu_mutex_unlock (&s->lock); +return rc; } static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs,
block/blkdebug.c static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs, · line 936
Quoted 10 lines
block/blkdebug.c +1
-6@@ -909,17 +933,19 @@ static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs,BlkdebugRule *rule, *next;int i, ret = -ENOENT ; -for (i =0 ; i <BLKDBG__MAX ; i++) { - QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) { -if (rule->action ==ACTION_SUSPEND && - !strcmp (rule->options.suspend.tag, tag)) { -remove_rule (rule); - ret =0 ; + WITH_QEMU_LOCK_GUARD(&s->lock) {
And here, instead of increasing nesting, let's use QEMU_LOCK_GUARD()
Quoted 27 lines
block/blkdebug.c +11
-3+for (i =0 ; i <BLKDBG__MAX ; i++) { + QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) { +if (rule->action ==ACTION_SUSPEND && + !strcmp (rule->options.suspend.tag, tag)) { +remove_rule (rule); + ret =0 ; + } } } - } -if (resume_req_by_tag (s, tag, true) ==0 ) { - ret =0 ; +if (resume_req_by_tag (s, tag, true) ==0 ) { + ret =0 ; + } }return ret; }@@ -929,6 +955,7 @@ static bool blkdebug_debug_is_suspended(BlockDriverState *bs, const char *tag)BDRVBlkdebugState *s = bs->opaque;BlkdebugSuspendedReq *r; +QEMU_LOCK_GUARD (&s->lock); QLIST_FOREACH(r, &s->suspended_reqs, next) {if (!strcmp (r->tag, tag)) {return true;
Signature · 2 lines
Best regards, Vladimir
Re: [PATCH v3 5/5] blkdebug: protect rules and suspended_reqs with a lock
Quoted 18 lines · Vladimir Sementsov-Ogievskiy
Quoted 11 lines · Emanuele Giuseppe Esposito
Co-developed-by Paolo Bonzini <[email protected]>Signed-off-by Emanuele Giuseppe Esposito <[email protected]>---
block/blkdebug.c| 53 ++++++++++++++++++++++++++++++++++++------------1 file changed, 40 insertions(+), 13 deletions(-)block/blkdebug.c +0
-0block/blkdebug.c typedef struct BDRVBlkdebugState { · line 56
Quoted 3 lines
block/blkdebug.c +1
-0@@ -54,6 +54,7 @@ typedef struct BDRVBlkdebugState { /* For blkdebug_refresh_filename() */ char *config_file; + QemuMutex lock;we'll need a comments, which fields are protected by the lock, like in other your series.
Quoted 3 lines
and also a comment which functions are thread-safe after the patch.
remove_rule()lacks comment, like "Called with lock held or from .bdrv_close"
Will apply these feedback in next version, thanks.
[...]
Quoted 49 lines
Quoted 18 lines
block/blkdebug.c +6
-3+/* Called with lock held. */ static void process_rule(BlockDriverState *bs, struct BlkdebugRule *rule, int *action_count) {@@ -829,9 +840,11 @@ static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event) assert((int)event >= 0 && event < BLKDBG__MAX); - s->new_state = s->state; - QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) { - process_rule(bs, rule, actions_count); + WITH_QEMU_LOCK_GUARD(&s->lock) { + s->new_state = s->state; + QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) { + process_rule(bs, rule, actions_count); + } } while (actions_count[ACTION_SUSPEND] > 0) {block/blkdebug.c static void blkdebug_debug_event(BlockDriverState · line 858
Quoted 7 lines
block/blkdebug.c +2
-0@@ -839,7 +852,9 @@ static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event) actions_count[ACTION_SUSPEND]--; } + qemu_mutex_lock(&s->lock); s->state = s->new_state; + qemu_mutex_unlock(&s->lock); }Preexising: honestly, I don't understand the logic around state and
new_state.. (and therefore, I'm not sure, is it in good relation with patch 04)It come in the commit
commit
8f96b5be92fbd74798b97b1dc1ff5fbbe249ed11
Author: Paolo Bonzini <[email protected]>
Date: Fri Sep 28 17:23:00 2012 +0200blkdebug: process all
set_staterules in the old state Currently it is impossible to write a blkdebug script that ping-pongs between two states, because the second set-state rule will use the state that is set in the first. If you have
[set-state]
event = "..."
state = "1"
new_state= "2"
[set-state]
event = "..."
state = "2"
new_state= "1"
for example the state will remain locked at 1. This can be fixed by first processing all rules, and then setting the state.
Quoted 1 line
But I don't understand, whey it should remain locked..
From what I understand, when bdrv_debug_event is invoked the code before this patch will simply change the state in 1 - 2 - 1 in the same rule iteration. So following Paolo's example, having these two rules in the same rules["..."] list, will make only one bdrv_debug_event change the state from 1 to 2, and 2 to 1 (if they are ordered in this way), removing both rules from the list.
This is not the expected behavior: we want two bdrv_debug_event to trigger the two state changes, so in the first bdrv_debug_event call we have 1-2 and next 2-1.
Quoted 2 lines
And this logic is not safe: another event may happen during the yield, and will operate on the same
s->stateands->new_state, leading the the mess.
Yes, I think you are right. The state update should go in the same lock guard above, like this:
WITH_QEMU_LOCK_GUARD(&s->lock) {
s->new_state = s->state;
QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) {
process_rule(bs, rule, actions_count);
}
+ s->state = s->new_state;
}
while (actions_count[ACTION_SUSPEND] > 0) {
qemu_coroutine_yield();
actions_count[ACTION_SUSPEND]--;
}- qemu_mutex_lock(&s->lock); - s->state = s->new_state; - qemu_mutex_unlock(&s->lock);
The other comments below make sense to me, will also apply them.
Thank you,
Emanuele
Quoted 86 lines
block/blkdebug.c static int · line 885
Quoted 12 lines
block/blkdebug.c +3
-0static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event,@@ -862,11 +877,14 @@ static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event, .options.suspend.tag = g_strdup(tag), }; + qemu_mutex_lock(&s->lock); QLIST_INSERT_HEAD(&s->rules[blkdebug_event], rule, next); + qemu_mutex_unlock(&s->lock); return 0; } +/* Called with lock held. */ I think, it would be a very good practice to include into convention:
May temporarily release lock. >
Quoted 12 lines
block/blkdebug.c +2
-0static int resume_req_by_tag(BDRVBlkdebugState *s, const char *tag, bool all) { BlkdebugSuspendedReq *r;@@ -884,7 +902,9 @@ retry: g_free(r->tag); g_free(r); + qemu_mutex_unlock(&s->lock); qemu_coroutine_enter(co); + qemu_mutex_lock(&s->lock); if (all) { goto retry;block/blkdebug.c retry: · line 921
Quoted 4 lines
block/blkdebug.c +1
-0@@ -898,8 +918,12 @@ retry: static int blkdebug_debug_resume(BlockDriverState *bs, const char *tag) { BDRVBlkdebugState *s = bs->opaque; + int rc;Hmm, you can simply use
QEMU_LOCK_GUARDQuoted 8 lines
block/blkdebug.c +4
-1- return resume_req_by_tag(s, tag, false); + qemu_mutex_lock(&s->lock); + rc = resume_req_by_tag(s, tag, false); + qemu_mutex_unlock(&s->lock); + return rc; } static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs,block/blkdebug.c static int · line 936
Quoted 10 lines
block/blkdebug.c +1
-6@@ -909,17 +933,19 @@ static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs, BlkdebugRule *rule, *next; int i, ret = -ENOENT; - for (i = 0; i < BLKDBG__MAX; i++) { - QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) { - if (rule->action == ACTION_SUSPEND && - !strcmp(rule->options.suspend.tag, tag)) { - remove_rule(rule); - ret = 0; + WITH_QEMU_LOCK_GUARD(&s->lock) {And here, instead of increasing nesting, let's use
QEMU_LOCK_GUARD()Quoted 27 lines
block/blkdebug.c +11
-3+ for (i = 0; i < BLKDBG__MAX; i++) { + QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) { + if (rule->action == ACTION_SUSPEND && + !strcmp(rule->options.suspend.tag, tag)) { + remove_rule(rule); + ret = 0; + } } } - } - if (resume_req_by_tag(s, tag, true) == 0) { - ret = 0; + if (resume_req_by_tag(s, tag, true) == 0) { + ret = 0; + } } return ret; }@@ -929,6 +955,7 @@ static bool blkdebug_debug_is_suspended(BlockDriverState *bs, const char *tag) BDRVBlkdebugState *s = bs->opaque; BlkdebugSuspendedReq *r; + QEMU_LOCK_GUARD(&s->lock); QLIST_FOREACH(r, &s->suspended_reqs, next) { if (!strcmp(r->tag, tag)) { return true;
Re: [PATCH v3 5/5] blkdebug: protect rules and suspended_reqs with a lock
Quoted 76 lines · Emanuele Giuseppe Esposito
Quoted 18 lines · Vladimir Sementsov-Ogievskiy
Quoted 11 lines · Emanuele Giuseppe Esposito
Co-developed-by Paolo Bonzini <[email protected]>Signed-off-by Emanuele Giuseppe Esposito <[email protected]>---
block/blkdebug.c| 53 ++++++++++++++++++++++++++++++++++++------------1 file changed, 40 insertions(+), 13 deletions(-)block/blkdebug.c +0
-0block/blkdebug.c typedef struct BDRVBlkdebugState { · line 56
Quoted 3 lines
block/blkdebug.c +1
-0@@ -54,6 +54,7 @@ typedef struct BDRVBlkdebugState { /* For blkdebug_refresh_filename() */ char *config_file; + QemuMutex lock;we'll need a comments, which fields are protected by the lock, like in other your series.
Quoted 3 lines
and also a comment which functions are thread-safe after the patch.
remove_rule()lacks comment, like "Called with lock held or from .bdrv_close"Will apply these feedback in next version, thanks.
[...]
Quoted 47 lines
Quoted 17 lines
block/blkdebug.c +6
-3+/* Called with lock held. */ static void process_rule(BlockDriverState *bs, struct BlkdebugRule *rule, int *action_count) {@@ -829,9 +840,11 @@ static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event) assert((int)event >= 0 && event < BLKDBG__MAX); - s->new_state = s->state; - QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) { - process_rule(bs, rule, actions_count); + WITH_QEMU_LOCK_GUARD(&s->lock) { + s->new_state = s->state; + QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) { + process_rule(bs, rule, actions_count); + } } while (actions_count[ACTION_SUSPEND] > 0) {block/blkdebug.c static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event) · line 857
Quoted 6 lines
block/blkdebug.c +2
-0@@ -839,7 +852,9 @@ static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event) actions_count[ACTION_SUSPEND]--; } + qemu_mutex_lock(&s->lock); s->state = s->new_state; + qemu_mutex_unlock(&s->lock); }Preexising: honestly, I don't understand the logic around state and
new_state.. (and therefore, I'm not sure, is it in good relation with patch 04)It come in the commit
commit
8f96b5be92fbd74798b97b1dc1ff5fbbe249ed11
Author: Paolo Bonzini <[email protected]>
Date: Fri Sep 28 17:23:00 2012 +0200blkdebug: process all
set_staterules in the old state Currently it is impossible to write a blkdebug script that ping-pongs between two states, because the second set-state rule will use the state that is set in the first. If you have
[set-state]
event = "..."
state = "1"
new_state= "2"
[set-state]
event = "..."
state = "2"
new_state= "1"
for example the state will remain locked at 1. This can be fixed by first processing all rules, and then setting the state.
Quoted 5 lines
Quoted 1 line
But I don't understand, whey it should remain locked..
From what I understand, when
bdrv_debug_eventis invoked the code before this patch will simply change the state in 1 - 2 - 1 in the same rule iteration. So following Paolo's example, having these two rules in the same rules["..."] list, will make only onebdrv_debug_eventchange the state from 1 to 2, and 2 to 1 (if they are ordered in this way), removing both rules from the list.This is not the expected behavior: we want two
bdrv_debug_eventto trigger the two state changes, so in the firstbdrv_debug_eventcall we have 1-2 and next 2-1.
Oh, understand now, thanks.
Quoted 5 lines
Quoted 3 lines
And this logic is not safe: another event may happen during the yield, and will operate on the same
s->stateands->new_state, leading the the mess.Yes, I think you are right. The state update should go in the same lock guard above, like this:
Agreed.
Probably good refactoring would be return new_state from process_rule, this way we can drop extra state variable s->new_state and use local variable instead (like we do for actions_count)
Quoted 107 lines
WITH_QEMU_LOCK_GUARD(&s->lock){
s->new_state=s->state;QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next){process_rule(bs, rule, actions_count);
}
+s->state=s->new_state;
}while (actions_count[ACTION_SUSPEND] > 0) {
qemu_coroutine_yield();
actions_count[ACTION_SUSPEND]--;
}-
qemu_mutex_lock(&s->lock);
-s->state=s->new_state;
-qemu_mutex_unlock(&s->lock);The other comments below make sense to me, will also apply them.
Thank you,
EmanueleQuoted 83 lines
block/blkdebug.c static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event, · line 884
Quoted 11 lines
block/blkdebug.c +3
-0static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event,@@ -862,11 +877,14 @@ static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event, .options.suspend.tag = g_strdup(tag), }; + qemu_mutex_lock(&s->lock); QLIST_INSERT_HEAD(&s->rules[blkdebug_event], rule, next); + qemu_mutex_unlock(&s->lock); return 0; } +/* Called with lock held. */ I think, it would be a very good practice to include into convention:
May temporarily release lock. >
Quoted 12 lines
block/blkdebug.c +2
-0static int resume_req_by_tag(BDRVBlkdebugState *s, const char *tag, bool all) { BlkdebugSuspendedReq *r;@@ -884,7 +902,9 @@ retry: g_free(r->tag); g_free(r); + qemu_mutex_unlock(&s->lock); qemu_coroutine_enter(co); + qemu_mutex_lock(&s->lock); if (all) { goto retry;block/blkdebug.c retry: · line 921
Quoted 4 lines
block/blkdebug.c +1
-0@@ -898,8 +918,12 @@ retry: static int blkdebug_debug_resume(BlockDriverState *bs, const char *tag) { BDRVBlkdebugState *s = bs->opaque; + int rc;Hmm, you can simply use
QEMU_LOCK_GUARDQuoted 8 lines
block/blkdebug.c +4
-1- return resume_req_by_tag(s, tag, false); + qemu_mutex_lock(&s->lock); + rc = resume_req_by_tag(s, tag, false); + qemu_mutex_unlock(&s->lock); + return rc; } static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs,block/blkdebug.c static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs, · line 935
Quoted 9 lines
block/blkdebug.c +1
-6@@ -909,17 +933,19 @@ static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs, BlkdebugRule *rule, *next; int i, ret = -ENOENT; - for (i = 0; i < BLKDBG__MAX; i++) { - QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) { - if (rule->action == ACTION_SUSPEND && - !strcmp(rule->options.suspend.tag, tag)) { - remove_rule(rule); - ret = 0; + WITH_QEMU_LOCK_GUARD(&s->lock) {And here, instead of increasing nesting, let's use
QEMU_LOCK_GUARD()Quoted 26 lines
block/blkdebug.c +11
-3+ for (i = 0; i < BLKDBG__MAX; i++) { + QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) { + if (rule->action == ACTION_SUSPEND && + !strcmp(rule->options.suspend.tag, tag)) { + remove_rule(rule); + ret = 0; + } } } - } - if (resume_req_by_tag(s, tag, true) == 0) { - ret = 0; + if (resume_req_by_tag(s, tag, true) == 0) { + ret = 0; + } } return ret; }@@ -929,6 +955,7 @@ static bool blkdebug_debug_is_suspended(BlockDriverState *bs, const char *tag) BDRVBlkdebugState *s = bs->opaque; BlkdebugSuspendedReq *r; + QEMU_LOCK_GUARD(&s->lock); QLIST_FOREACH(r, &s->suspended_reqs, next) { if (!strcmp(r->tag, tag)) { return true;
Signature · 2 lines
Best regards, Vladimir