blkdebug: fix racing condition when iterating on
cover blkdebug: fix racing condition when iterating on
When qemu_coroutine_enter is executed in a loop
(even QEMU_FOREACH_SAFE), the new routine can modify the list, for example removing an element, causing problem when control is given back to the caller that continues iterating on the same list.
Patch 1 solves the issue in blkdebug_debug_resume by restarting the list walk after every coroutine_enter if list has to be fully iterated. Patches 2,3,4 aim to fix blkdebug_debug_event by gathering
all actions that the rules make in a counter and invoking
the respective coroutine_yeld only after processing all requests. Patch 5 adds a lock to protect rules and suspended_reqs.
Emanuele Giuseppe Esposito (5):
blkdebug: refactor removal of a suspended request
blkdebug: move post-resume handling to resume_req_by_tag
blkdebug: track all actions
blkdebug: do not suspend in the middle of QLIST_FOREACH_SAFE
blkdebug: protect rules and suspended_reqs with a lock
block/blkdebug.c | 113 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 79 insertions(+), 34 deletions(-)
Signature · 1 line
2.30.2
PATCH 1/5 blkdebug: refactor removal of a suspended request
Extract to a separate function. Do not rely on FOREACH_SAFE, which is only "safe" if the *current* node is removed---not if another node is removed. Instead, just walk the entire list from the beginning when asked to resume all suspended requests with a given tag.
--- block/blkdebug.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-)
block/blkdebug.c +17 -11
@@ -793,7 +793,6 @@ static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule)printf ("blkdebug: Resuming request '%s'\n" , r.tag); } -QLIST_REMOVE (&r, next);g_free (r.tag); }
@@ -869,25 +868,35 @@ static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event,return 0 ; } -static int blkdebug_debug_resume (BlockDriverState *bs,const char *tag) +static int resume_req_by_tag (BDRVBlkdebugState *s,const char *tag,bool all) { -BDRVBlkdebugState *s = bs->opaque; -BlkdebugSuspendedReq *r, *next; +BlkdebugSuspendedReq *r; - QLIST_FOREACH_SAFE(r, &s->suspended_reqs, next, next) { +retry: + QLIST_FOREACH(r, &s->suspended_reqs, next) {if (!strcmp (r->tag, tag)) { +QLIST_REMOVE (r, next);qemu_coroutine_enter (r->co); +if (all) { + goto retry; + }return 0 ; } }return -ENOENT ; } +static int blkdebug_debug_resume (BlockDriverState *bs,const char *tag) +{ +BDRVBlkdebugState *s = bs->opaque; + +return resume_req_by_tag (s, tag, false); +} +static int blkdebug_debug_remove_breakpoint (BlockDriverState *bs,const char *tag) {BDRVBlkdebugState *s = bs->opaque; -BlkdebugSuspendedReq *r, *r_next;BlkdebugRule *rule, *next;int i, ret = -ENOENT ;
@@ -900,11 +909,8 @@ static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs, } } } - QLIST_FOREACH_SAFE(r, &s->suspended_reqs, next, r_next) { -if (!strcmp (r->tag, tag)) { -qemu_coroutine_enter (r->co); - ret =0 ; - } +if (resume_req_by_tag (s, tag, true) ==0 ) { + ret =0 ; }return ret; }
Signature · 1 line
2.30.2
PATCH 2/5 blkdebug: move post-resume handling to resume_req_by_tag
In reply to Emanuele Giuseppe EspositoWe want to move qemu_coroutine_yield() after the loop on rules, because QLIST_FOREACH_SAFE is wrong if the rule list is modified while the coroutine has yielded. Therefore move the suspended request to the heap and clean it up from the remove side. All that is left is for blkdebug_debug_event to handle the yielding.
--- block/blkdebug.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-)
block/blkdebug.c +18 -13
@@ -775,25 +775,20 @@ static void blkdebug_close(BlockDriverState *bs)static void suspend_request (BlockDriverState *bs,BlkdebugRule *rule) {BDRVBlkdebugState *s = bs->opaque; -BlkdebugSuspendedReq r; +BlkdebugSuspendedReq *r; - r = (BlkdebugSuspendedReq ) { - .co =qemu_coroutine_self (), - .tag =g_strdup (rule->options.suspend.tag), - }; + r =g_new (BlkdebugSuspendedReq,1 ); + + r->co =qemu_coroutine_self (); + r->tag =g_strdup (rule->options.suspend.tag);remove_rule (rule); -QLIST_INSERT_HEAD (&s->suspended_reqs, &r, next); +QLIST_INSERT_HEAD (&s->suspended_reqs, r, next);if (!qtest_enabled ()) { -printf ("blkdebug: Suspended request '%s'\n" , r.tag); +printf ("blkdebug: Suspended request '%s'\n" , r->tag); }qemu_coroutine_yield (); -if (!qtest_enabled ()) { -printf ("blkdebug: Resuming request '%s'\n" , r.tag); - } - -g_free (r.tag); } static bool process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,
@@ -875,8 +870,18 @@ static int resume_req_by_tag(BDRVBlkdebugState *s, const char *tag, bool all) retry: QLIST_FOREACH(r, &s->suspended_reqs, next) {if (!strcmp (r->tag, tag)) { +Coroutine *co = r->co; + +if (!qtest_enabled ()) { +printf ("blkdebug: Resuming request '%s'\n" , r->tag); + } +QLIST_REMOVE (r, next); -qemu_coroutine_enter (r->co); +g_free (r->tag); +g_free (r); + +qemu_coroutine_enter (co); +if (all) { goto retry; }
Signature · 1 line
2.30.2
PATCH 3/5 blkdebug: track all actions
In reply to Emanuele Giuseppe EspositoAdd a counter for each action that a rule can trigger. This is mainly used to keep track of how many coroutine_yeld() we need to perform after processing all rules in the list.
--- block/blkdebug.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-)
block/blkdebug.c +8 -9
@@ -74,6 +74,7 @@ enum {ACTION_INJECT_ERROR ,ACTION_SET_STATE ,ACTION_SUSPEND , + ACTION__MAX, }; typedef struct BlkdebugRule {
@@ -791,22 +792,22 @@ static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule)qemu_coroutine_yield (); } -static bool process_rule (BlockDriverState *bs,struct BlkdebugRule *rule, -bool injected) +static void process_rule (BlockDriverState *bs,struct BlkdebugRule *rule, +int *action_count) {BDRVBlkdebugState *s = bs->opaque;/* Only process rules for the current state */ if (rule->state && rule->state != s->state) { -return injected; +return ; }/* Take the action */ + action_count[rule->action]++;switch (rule->action) {case ACTION_INJECT_ERROR : -if (!injected) { +if (action_count[ACTION_INJECT_ERROR ] ==1 ) {QSIMPLEQ_INIT (&s->active_rules); - injected = true; }QSIMPLEQ_INSERT_HEAD (&s->active_rules, rule, active_next);break ;
@@ -819,21 +820,19 @@ static bool process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,suspend_request (bs, rule);break ; } -return injected; }static void blkdebug_debug_event (BlockDriverState *bs,BlkdebugEvent event) {BDRVBlkdebugState *s = bs->opaque;struct BlkdebugRule *rule, *next; -bool injected; +int actions_count[ACTION__MAX ] = {0 };assert ((int )event >=0 && event <BLKDBG__MAX ); - injected = false; s->new_state = s->state; QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) { - injected =process_rule (bs, rule, injected); +process_rule (bs, rule, actions_count); } s->state = s->new_state; }
Signature · 1 line
2.30.2
PATCH 4/5 blkdebug: do not suspend in the middle of QLIST_FOREACH_SAFE
In reply to Emanuele Giuseppe EspositoUse actions_count to see how many yeld to issue.
--- block/blkdebug.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)
block/blkdebug.c +6 -1
@@ -789,7 +789,6 @@ static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule)if (!qtest_enabled ()) {printf ("blkdebug: Suspended request '%s'\n" , r->tag); } -qemu_coroutine_yield (); } static void process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,
@@ -834,6 +833,12 @@ static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event) QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) {process_rule (bs, rule, actions_count); } + +while (actions_count[ACTION_SUSPEND ] >0 ) { +qemu_coroutine_yield (); + actions_count[ACTION_SUSPEND ]--; + } + s->state = s->new_state; }
Signature · 1 line
2.30.2
Re: [PATCH 4/5] blkdebug: do not suspend in the middle of QLIST_FOREACH_SAFE
On 08/04/21 17:59, Emanuele Giuseppe Esposito wrote:
Perhaps insert here:
That would be unsafe in case a rule other than the current one is removed while the coroutine has yielded. Keep FOREACH_SAFE because suspend_request deletes the current rule.
After this patch, *all* matching rules are deleted before suspending the coroutine, rather than just one. This doesn't affect the existing testcases.
Quoted 34 lines
Use
actions_countto see how many yeld to issue.Co-developed-by Paolo Bonzini <[email protected]>Signed-off-by Emanuele Giuseppe Esposito <[email protected]>--- block/blkdebug.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)block/blkdebug.c +6
-1@@ -789,7 +789,6 @@ static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule)if (!qtest_enabled ()) {printf ("blkdebug: Suspended request '%s'\n" , r->tag); } -qemu_coroutine_yield (); } static void process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,@@ -834,6 +833,12 @@ static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event) QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) {process_rule (bs, rule, actions_count); } + +while (actions_count[ACTION_SUSPEND ] >0 ) { +qemu_coroutine_yield (); + actions_count[ACTION_SUSPEND ]--; + } + s->state = s->new_state; }
PATCH 5/5 blkdebug: protect rules and suspended_reqs with a lock
In reply to Emanuele Giuseppe Esposito--- block/blkdebug.c | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-)
block/blkdebug.c +31 -1
@@ -54,6 +54,7 @@ typedef struct BDRVBlkdebugState {/* For blkdebug_refresh_filename() */ char *config_file; +QemuMutex lock; QLIST_HEAD(, BlkdebugRule) rules[BLKDBG__MAX]; QSIMPLEQ_HEAD(, BlkdebugRule) active_rules; QLIST_HEAD(, BlkdebugSuspendedReq) suspended_reqs;
@@ -245,7 +246,9 @@ static int add_rule(void *opaque, QemuOpts *opts, Error **errp) };/* Add the rule */ +qemu_mutex_lock (&s->lock);QLIST_INSERT_HEAD (&s->rules[event], rule, next); +qemu_mutex_unlock (&s->lock);return 0 ; }
@@ -468,6 +471,7 @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags,int ret;uint64_t align; +qemu_mutex_init (&s->lock); opts =qemu_opts_create (&runtime_opts,NULL ,0 , &error_abort);if (!qemu_opts_absorb_qdict (opts, options, errp)) { ret = -EINVAL ;
@@ -568,6 +572,7 @@ static int blkdebug_open(BlockDriverState *bs, QDict *options, int flags, ret =0 ; out:if (ret <0 ) { +qemu_mutex_destroy (&s->lock);g_free (s->config_file); }qemu_opts_del (opts);
@@ -582,6 +587,7 @@ static int rule_check(BlockDriverState *bs, uint64_t offset, uint64_t bytes,int error;bool immediately; +qemu_mutex_lock (&s->lock); QSIMPLEQ_FOREACH(rule, &s->active_rules, active_next) {uint64_t inject_offset = rule->options.inject.offset;
@@ -595,6 +601,7 @@ static int rule_check(BlockDriverState *bs, uint64_t offset, uint64_t bytes, }if (!rule || !rule->options.inject.error) { +qemu_mutex_unlock (&s->lock);return 0 ; }
@@ -606,6 +613,7 @@ static int rule_check(BlockDriverState *bs, uint64_t offset, uint64_t bytes,remove_rule (rule); } +qemu_mutex_unlock (&s->lock);if (!immediately) {aio_co_schedule (qemu_get_current_aio_context (),qemu_coroutine_self ());qemu_coroutine_yield ();
@@ -771,8 +779,10 @@ static void blkdebug_close(BlockDriverState *bs) }g_free (s->config_file); +qemu_mutex_destroy (&s->lock); } +/* Called with lock held. */ static void suspend_request (BlockDriverState *bs,BlkdebugRule *rule) {BDRVBlkdebugState *s = bs->opaque;
@@ -791,6 +801,7 @@ static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule) } } +/* Called with lock held. */ static void process_rule (BlockDriverState *bs,struct BlkdebugRule *rule,int *action_count) {
@@ -829,17 +840,22 @@ static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event)assert ((int )event >=0 && event <BLKDBG__MAX ); +qemu_mutex_lock (&s->lock); s->new_state = s->state; QLIST_FOREACH_SAFE(rule, &s->rules[event], next, next) {process_rule (bs, rule, actions_count); } +qemu_mutex_unlock (&s->lock); +while (actions_count[ACTION_SUSPEND ] >0 ) {qemu_coroutine_yield (); actions_count[ACTION_SUSPEND ]--; } +qemu_mutex_lock (&s->lock); s->state = s->new_state; +qemu_mutex_unlock (&s->lock); } static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event,
@@ -862,11 +878,14 @@ static int blkdebug_debug_breakpoint(BlockDriverState *bs, const char *event, .options.suspend.tag = g_strdup(tag), }; +qemu_mutex_lock (&s->lock);QLIST_INSERT_HEAD (&s->rules[blkdebug_event], rule, next); +qemu_mutex_unlock (&s->lock);return 0 ; } +/* Called with lock held. */ static int resume_req_by_tag (BDRVBlkdebugState *s,const char *tag,bool all) {BlkdebugSuspendedReq *r;
@@ -884,7 +903,9 @@ retry:g_free (r->tag);g_free (r); +qemu_mutex_unlock (&s->lock);qemu_coroutine_enter (co); +qemu_mutex_lock (&s->lock);if (all) { goto retry;
@@ -898,8 +919,12 @@ retry:static int blkdebug_debug_resume (BlockDriverState *bs,const char *tag) {BDRVBlkdebugState *s = bs->opaque; +int rc; -return resume_req_by_tag (s, tag, false); +qemu_mutex_lock (&s->lock); + rc =resume_req_by_tag (s, tag, false); +qemu_mutex_unlock (&s->lock); +return rc; } static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs,
@@ -909,6 +934,7 @@ static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs,BlkdebugRule *rule, *next;int i, ret = -ENOENT ; +qemu_mutex_lock (&s->lock);for (i =0 ; i <BLKDBG__MAX ; i++) { QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) { if (rule->action == ACTION_SUSPEND &&
@@ -921,6 +947,7 @@ static int blkdebug_debug_remove_breakpoint(BlockDriverState *bs,if (resume_req_by_tag (s, tag, true) ==0 ) { ret =0 ; } +qemu_mutex_unlock (&s->lock);return ret; }
@@ -929,11 +956,14 @@ static bool blkdebug_debug_is_suspended(BlockDriverState *bs, const char *tag)BDRVBlkdebugState *s = bs->opaque;BlkdebugSuspendedReq *r; +qemu_mutex_lock (&s->lock); QLIST_FOREACH(r, &s->suspended_reqs, next) {if (!strcmp (r->tag, tag)) { +qemu_mutex_unlock (&s->lock);return true; } } +qemu_mutex_unlock (&s->lock);return false; }
Signature · 1 line
2.30.2
Quoted 11 lines · Emanuele Giuseppe Esposito
When
qemu_coroutine_enteris executed in a loop
(evenQEMU_FOREACH_SAFE), the new routine can modify the list, for example removing an element, causing problem when control is given back to the caller that continues iterating on the same list.Patch 1 solves the issue in
blkdebug_debug_resumeby restarting the list walk after everycoroutine_enterif list has to be fully iterated. Patches 2,3,4 aim to fixblkdebug_debug_eventby gathering
all actions that the rules make in a counter and invoking
the respectivecoroutine_yeldonly after processing all requests. Patch 5 adds a lock to protect rules andsuspended_reqs.
Patch 5 is somewhat independent of the others; right now everything works because it's protected by the AioContext lock.
On the other hand the scenarios in patches 1-4 are bugs even without patch 5. They become more obvious if you see an explicit unlock/lock pair within QTAILQ_FOREACH_SAFE, but they can happen already with just a qemu_coroutine_yield or qemu_coroutine_enter within the iteration.
Paolo