bcache: fix deadlock in bcache_allocator
PATCH v2 1/1 bcache: fix deadlock in bcache_allocator
bcache_allocator() can call the following:
bch_allocator_thread()
-> bch_prio_write()
-> bch_bucket_alloc()
-> wait on &ca->set->bucket_waitBut the wake up event on bucket_wait is supposed to come from bch_allocator_thread() itself => deadlock:
[ 1158.490744] INFO: task bcache_allocato:15861 blocked for more than 10 seconds. [ 1158.495929] Not tainted 5.3.0-050300rc3-generic #201908042232 [ 1158.500653] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1158.504413] bcache_allocato D 0 15861 2 0x80004000 [ 1158.504419] Call Trace: [ 1158.504429] __schedule+0x2a8/0x670 [ 1158.504432] schedule+0x2d/0x90 [ 1158.504448] bch_bucket_alloc+0xe5/0x370 [bcache] [ 1158.504453] ? wait_woken+0x80/0x80 [ 1158.504466] bch_prio_write+0x1dc/0x390 [bcache] [ 1158.504476] bch_allocator_thread+0x233/0x490 [bcache] [ 1158.504491] kthread+0x121/0x140 [ 1158.504503] ? invalidate_buckets+0x890/0x890 [bcache] [ 1158.504506] ? kthread_park+0xb0/0xb0 [ 1158.504510] ret_from_fork+0x35/0x40
Fix by making the call to bch_prio_write() non-blocking, so that bch_allocator_thread() never waits on itself.
Moreover, make sure to wake up the garbage collector thread when bch_prio_write() is failing to allocate buckets.
---
Changes in v2:
- prevent retry_invalidate busy loop in bch_allocator_thread()
drivers/md/bcache/alloc.c | 5 ++++- drivers/md/bcache/bcache.h | 2 +- drivers/md/bcache/super.c | 13 +++++++++---- 3 files changed, 14 insertions(+), 6 deletions(-)
drivers/md/bcache/alloc.c +4 -1
@@ -377,7 +377,10 @@ static int bch_allocator_thread(void *arg)if (!fifo_full (&ca->free_inc)) goto retry_invalidate; -bch_prio_write (ca); +if (bch_prio_write (ca, false) <0 ) { + ca->invalidate_needs_gc =1 ; +wake_up_gc (ca->set); + } } } out:
drivers/md/bcache/bcache.h +1 -1
@@ -977,7 +977,7 @@ bool bch_cached_dev_error(struct cached_dev *dc); __printf(2, 3)bool bch_cache_set_error (struct cache_set *c,const char *fmt, ...); -void bch_prio_write (struct cache *ca); +int bch_prio_write (struct cache *ca,bool wait);void bch_write_bdev_super (struct cached_dev *dc,struct closure *parent);extern struct workqueue_struct *bcache_wq;
drivers/md/bcache/super.c +9 -4
@@ -529,7 +529,7 @@ static void prio_io(struct cache *ca, uint64_t bucket, int op,closure_sync (cl); } -void bch_prio_write (struct cache *ca) +int bch_prio_write (struct cache *ca,bool wait) {int i;struct bucket *b;
@@ -564,8 +564,12 @@ void bch_prio_write(struct cache *ca) p->magic =pset_magic (&ca->sb); p->csum =bch_crc64 (&p->magic,bucket_bytes (ca) -8 ); - bucket =bch_bucket_alloc (ca,RESERVE_PRIO , true); -BUG_ON (bucket ==-1 ); + bucket =bch_bucket_alloc (ca,RESERVE_PRIO , wait); +if (bucket ==-1 ) { +if (!wait) +return -ENOMEM ; +BUG_ON (1 ); + }mutex_unlock (&ca->set->bucket_lock);prio_io (ca, bucket,REQ_OP_WRITE ,0 );
@@ -593,6 +597,7 @@ void bch_prio_write(struct cache *ca) ca->prio_last_buckets[i] = ca->prio_buckets[i]; } +return 0 ; }static void prio_read (struct cache *ca,uint64_t bucket)
@@ -1954,7 +1959,7 @@ static int run_cache_set(struct cache_set *c)mutex_lock (&c->bucket_lock);for_each_cache (ca, c, i) -bch_prio_write (ca); +bch_prio_write (ca, true);mutex_unlock (&c->bucket_lock); err ="cannot allocate new UUID bucket" ;
Signature · 1 line
2.20.1
Quoted 29 lines · Andrea Righi
bcache_allocator()can call the following:
bch_allocator_thread()
->bch_prio_write()-> bch_bucket_alloc() -> wait on &ca->set->bucket_waitBut the wake up event on
bucket_waitis supposed to come frombch_allocator_thread()itself => deadlock:[ 1158.490744] INFO: task bcache_allocato:15861 blocked for more than 10 seconds. [ 1158.495929] Not tainted 5.3.0-050300rc3-generic #201908042232 [ 1158.500653] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1158.504413] bcache_allocato D 0 15861 2 0x80004000 [ 1158.504419] Call Trace: [ 1158.504429] __schedule+0x2a8/0x670 [ 1158.504432] schedule+0x2d/0x90 [ 1158.504448] bch_bucket_alloc+0xe5/0x370 [bcache] [ 1158.504453] ? wait_woken+0x80/0x80 [ 1158.504466] bch_prio_write+0x1dc/0x390 [bcache] [ 1158.504476] bch_allocator_thread+0x233/0x490 [bcache] [ 1158.504491] kthread+0x121/0x140 [ 1158.504503] ? invalidate_buckets+0x890/0x890 [bcache] [ 1158.504506] ? kthread_park+0xb0/0xb0 [ 1158.504510] ret_from_fork+0x35/0x40Fix by making the call to
bch_prio_write()non-blocking, so thatbch_allocator_thread()never waits on itself.
Quoted 6 lines
Moreover, make sure to wake up the garbage collector thread when
bch_prio_write()is failing to allocate buckets.Signed-off-by Andrea Righi <[email protected]>
At this moment I am not able to find a better solution, so I take this patch in my for-test.
Thank you. And I hope you may continue to maintain this change if people report problem (I mean if) in future.
Coly Li
Quoted 84 lines
---
Changes in v2:
- preventretry_invalidatebusy loop inbch_allocator_thread()drivers/md/bcache/alloc.c | 5 ++++- drivers/md/bcache/bcache.h | 2 +- drivers/md/bcache/super.c | 13 +++++++++---- 3 files changed, 14 insertions(+), 6 deletions(-)drivers/md/bcache/alloc.c +4
-1@@ -377,7 +377,10 @@ static int bch_allocator_thread(void *arg)if (!fifo_full (&ca->free_inc)) goto retry_invalidate; -bch_prio_write (ca); +if (bch_prio_write (ca, false) <0 ) { + ca->invalidate_needs_gc =1 ; +wake_up_gc (ca->set); + } } } out:drivers/md/bcache/bcache.h +1
-1@@ -977,7 +977,7 @@ bool bch_cached_dev_error(struct cached_dev *dc); __printf(2, 3)bool bch_cache_set_error (struct cache_set *c,const char *fmt, ...); -void bch_prio_write (struct cache *ca); +int bch_prio_write (struct cache *ca,bool wait);void bch_write_bdev_super (struct cached_dev *dc,struct closure *parent);extern struct workqueue_struct *bcache_wq;drivers/md/bcache/super.c +9
-4@@ -529,7 +529,7 @@ static void prio_io(struct cache *ca, uint64_t bucket, int op,closure_sync (cl); } -void bch_prio_write (struct cache *ca) +int bch_prio_write (struct cache *ca,bool wait) {int i;struct bucket *b;@@ -564,8 +564,12 @@ void bch_prio_write(struct cache *ca) p->magic =pset_magic (&ca->sb); p->csum =bch_crc64 (&p->magic,bucket_bytes (ca) -8 ); - bucket =bch_bucket_alloc (ca,RESERVE_PRIO , true); -BUG_ON (bucket ==-1 ); + bucket =bch_bucket_alloc (ca,RESERVE_PRIO , wait); +if (bucket ==-1 ) { +if (!wait) +return -ENOMEM ; +BUG_ON (1 ); + }mutex_unlock (&ca->set->bucket_lock);prio_io (ca, bucket,REQ_OP_WRITE ,0 );@@ -593,6 +597,7 @@ void bch_prio_write(struct cache *ca) ca->prio_last_buckets[i] = ca->prio_buckets[i]; } +return 0 ; }static void prio_read (struct cache *ca,uint64_t bucket)@@ -1954,7 +1959,7 @@ static int run_cache_set(struct cache_set *c)mutex_lock (&c->bucket_lock);for_each_cache (ca, c, i) -bch_prio_write (ca); +bch_prio_write (ca, true);mutex_unlock (&c->bucket_lock); err ="cannot allocate new UUID bucket" ;
Signature · 1 line
Coly Li
Quoted 88 lines · Andrea Righi
bcache_allocator()can call the following:
bch_allocator_thread()
->bch_prio_write()-> bch_bucket_alloc() -> wait on &ca->set->bucket_waitBut the wake up event on
bucket_waitis supposed to come frombch_allocator_thread()itself => deadlock:[ 1158.490744] INFO: task bcache_allocato:15861 blocked for more than 10 seconds. [ 1158.495929] Not tainted 5.3.0-050300rc3-generic #201908042232 [ 1158.500653] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1158.504413] bcache_allocato D 0 15861 2 0x80004000 [ 1158.504419] Call Trace: [ 1158.504429] __schedule+0x2a8/0x670 [ 1158.504432] schedule+0x2d/0x90 [ 1158.504448] bch_bucket_alloc+0xe5/0x370 [bcache] [ 1158.504453] ? wait_woken+0x80/0x80 [ 1158.504466] bch_prio_write+0x1dc/0x390 [bcache] [ 1158.504476] bch_allocator_thread+0x233/0x490 [bcache] [ 1158.504491] kthread+0x121/0x140 [ 1158.504503] ? invalidate_buckets+0x890/0x890 [bcache] [ 1158.504506] ? kthread_park+0xb0/0xb0 [ 1158.504510] ret_from_fork+0x35/0x40Fix by making the call to
bch_prio_write()non-blocking, so thatbch_allocator_thread()never waits on itself.Moreover, make sure to wake up the garbage collector thread when
bch_prio_write()is failing to allocate buckets.Signed-off-by Andrea Righi <[email protected]>---
Changes in v2:
- preventretry_invalidatebusy loop inbch_allocator_thread()drivers/md/bcache/alloc.c | 5 ++++- drivers/md/bcache/bcache.h | 2 +- drivers/md/bcache/super.c | 13 +++++++++---- 3 files changed, 14 insertions(+), 6 deletions(-)drivers/md/bcache/alloc.c +4
-1@@ -377,7 +377,10 @@ static int bch_allocator_thread(void *arg)if (!fifo_full (&ca->free_inc)) goto retry_invalidate; -bch_prio_write (ca); +if (bch_prio_write (ca, false) <0 ) { + ca->invalidate_needs_gc =1 ; +wake_up_gc (ca->set); + } } } out:drivers/md/bcache/bcache.h +1
-1@@ -977,7 +977,7 @@ bool bch_cached_dev_error(struct cached_dev *dc); __printf(2, 3)bool bch_cache_set_error (struct cache_set *c,const char *fmt, ...); -void bch_prio_write (struct cache *ca); +int bch_prio_write (struct cache *ca,bool wait);void bch_write_bdev_super (struct cached_dev *dc,struct closure *parent);extern struct workqueue_struct *bcache_wq;drivers/md/bcache/super.c +1
-1@@ -529,7 +529,7 @@ static void prio_io(struct cache *ca, uint64_t bucket, int op,closure_sync (cl); } -void bch_prio_write (struct cache *ca) +int bch_prio_write (struct cache *ca,bool wait) {int i;struct bucket *b;@@ -564,8 +564,12 @@ void bch_prio_write(struct cache *ca) p->magic =pset_magic (&ca->sb);
drivers/md/bcache/super.c void bch_prio_write(struct cache *ca) · line 572
Quoted 10 lines
drivers/md/bcache/super.c +6
-2p->csum =bch_crc64 (&p->magic,bucket_bytes (ca) -8 ); - bucket =bch_bucket_alloc (ca,RESERVE_PRIO , true); -BUG_ON (bucket ==-1 ); + bucket =bch_bucket_alloc (ca,RESERVE_PRIO , wait); +if (bucket ==-1 ) { +if (!wait) +return -ENOMEM ; +BUG_ON (1 ); + }
Coly,
looking more at this change, I think we should handle the failure path properly or we may leak buckets, am I right? (sorry for not realizing this before). Maybe we need something like the following on top of my previous patch.
I'm going to run more stress tests with this patch applied and will try to figure out if we're actually leaking buckets without it.
---
Subject: bcache: prevent leaking buckets in bch_prio_write()
Handle the allocation failure path properly in bch_prio_write() to avoid leaking buckets from the previous successful iterations.
--- drivers/md/bcache/super.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-)
drivers/md/bcache/super.c +11 -9
@@ -531,7 +531,7 @@ static void prio_io(struct cache *ca, uint64_t bucket, int op,int bch_prio_write (struct cache *ca,bool wait) { -int i; +int i, j, ret =0 ;struct bucket *b;struct closure cl;
@@ -566,9 +566,11 @@ int bch_prio_write(struct cache *ca, bool wait) bucket =bch_bucket_alloc (ca,RESERVE_PRIO , wait);if (bucket ==-1 ) { -if (!wait) -return -ENOMEM ; -BUG_ON (1 ); +if (!wait) { + ret = -ENOMEM ; +break ; + } +BUG (); }mutex_unlock (&ca->set->bucket_lock);
@@ -590,14 +592,14 @@ int bch_prio_write(struct cache *ca, bool wait)* Don't want the old priorities to get garbage collected until after we * finish writing the new ones, and they're journalled */ -for (i =0 ; i <prio_buckets (ca); i++) { -if (ca->prio_last_buckets[i]) +for (j =prio_buckets (ca) -1 ; j > i; --j) { +if (ca->prio_last_buckets[j])__bch_bucket_free (ca, - &ca->buckets[ca->prio_last_buckets[i]]); + &ca->buckets[ca->prio_last_buckets[j]]); - ca->prio_last_buckets[i] = ca->prio_buckets[i]; + ca->prio_last_buckets[j] = ca->prio_buckets[j]; } -return 0 ; +return ret; }static void prio_read (struct cache *ca,uint64_t bucket)
Signature · 1 line
2.7.4
Quoted 110 lines · Andrea Righi
Quoted 88 lines · Andrea Righi
bcache_allocator()can call the following:
bch_allocator_thread()
->bch_prio_write()-> bch_bucket_alloc() -> wait on &ca->set->bucket_waitBut the wake up event on
bucket_waitis supposed to come frombch_allocator_thread()itself => deadlock:[ 1158.490744] INFO: task bcache_allocato:15861 blocked for more than 10 seconds. [ 1158.495929] Not tainted 5.3.0-050300rc3-generic #201908042232 [ 1158.500653] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1158.504413] bcache_allocato D 0 15861 2 0x80004000 [ 1158.504419] Call Trace: [ 1158.504429] __schedule+0x2a8/0x670 [ 1158.504432] schedule+0x2d/0x90 [ 1158.504448] bch_bucket_alloc+0xe5/0x370 [bcache] [ 1158.504453] ? wait_woken+0x80/0x80 [ 1158.504466] bch_prio_write+0x1dc/0x390 [bcache] [ 1158.504476] bch_allocator_thread+0x233/0x490 [bcache] [ 1158.504491] kthread+0x121/0x140 [ 1158.504503] ? invalidate_buckets+0x890/0x890 [bcache] [ 1158.504506] ? kthread_park+0xb0/0xb0 [ 1158.504510] ret_from_fork+0x35/0x40Fix by making the call to
bch_prio_write()non-blocking, so thatbch_allocator_thread()never waits on itself.Moreover, make sure to wake up the garbage collector thread when
bch_prio_write()is failing to allocate buckets.Signed-off-by Andrea Righi <[email protected]>---
Changes in v2:
- preventretry_invalidatebusy loop inbch_allocator_thread()drivers/md/bcache/alloc.c | 5 ++++- drivers/md/bcache/bcache.h | 2 +- drivers/md/bcache/super.c | 13 +++++++++---- 3 files changed, 14 insertions(+), 6 deletions(-)drivers/md/bcache/alloc.c +4
-1@@ -377,7 +377,10 @@ static int bch_allocator_thread(void *arg)if (!fifo_full (&ca->free_inc)) goto retry_invalidate; -bch_prio_write (ca); +if (bch_prio_write (ca, false) <0 ) { + ca->invalidate_needs_gc =1 ; +wake_up_gc (ca->set); + } } } out:drivers/md/bcache/bcache.h +1
-1@@ -977,7 +977,7 @@ bool bch_cached_dev_error(struct cached_dev *dc); __printf(2, 3)bool bch_cache_set_error (struct cache_set *c,const char *fmt, ...); -void bch_prio_write (struct cache *ca); +int bch_prio_write (struct cache *ca,bool wait);void bch_write_bdev_super (struct cached_dev *dc,struct closure *parent);extern struct workqueue_struct *bcache_wq;drivers/md/bcache/super.c +1
-1@@ -529,7 +529,7 @@ static void prio_io(struct cache *ca, uint64_t bucket, int op,closure_sync (cl); } -void bch_prio_write (struct cache *ca) +int bch_prio_write (struct cache *ca,bool wait) {int i;struct bucket *b;@@ -564,8 +564,12 @@ void bch_prio_write(struct cache *ca) p->magic =pset_magic (&ca->sb);drivers/md/bcache/super.c void bch_prio_write(struct cache *ca) · line 572
Quoted 10 lines
drivers/md/bcache/super.c +6
-2p->csum =bch_crc64 (&p->magic,bucket_bytes (ca) -8 ); - bucket =bch_bucket_alloc (ca,RESERVE_PRIO , true); -BUG_ON (bucket ==-1 ); + bucket =bch_bucket_alloc (ca,RESERVE_PRIO , wait); +if (bucket ==-1 ) { +if (!wait) +return -ENOMEM ; +BUG_ON (1 ); + }Coly,
looking more at this change, I think we should handle the failure path properly or we may leak buckets, am I right? (sorry for not realizing this before). Maybe we need something like the following on top of my previous patch.
I'm going to run more stress tests with this patch applied and will try to figure out if we're actually leaking buckets without it.
Quoted 7 lines
---
Subject: bcache: prevent leaking buckets inbch_prio_write()Handle the allocation failure path properly in
bch_prio_write()to avoid leaking buckets from the previous successful iterations.Signed-off-by Andrea Righi <[email protected]>
Coly, ignore this one please. A v3 of the previous patch with a better fix for this potential buckets leak is on the way.
Thanks,
-Andrea
Quoted 114 lines · Andrea Righi
Quoted 113 lines · Andrea Righi
Quoted 88 lines · Andrea Righi
bcache_allocator()can call the following:
bch_allocator_thread()
->bch_prio_write()-> bch_bucket_alloc() -> wait on &ca->set->bucket_waitBut the wake up event on
bucket_waitis supposed to come frombch_allocator_thread()itself => deadlock:[ 1158.490744] INFO: task bcache_allocato:15861 blocked for more than 10 seconds. [ 1158.495929] Not tainted 5.3.0-050300rc3-generic #201908042232 [ 1158.500653] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [ 1158.504413] bcache_allocato D 0 15861 2 0x80004000 [ 1158.504419] Call Trace: [ 1158.504429] __schedule+0x2a8/0x670 [ 1158.504432] schedule+0x2d/0x90 [ 1158.504448] bch_bucket_alloc+0xe5/0x370 [bcache] [ 1158.504453] ? wait_woken+0x80/0x80 [ 1158.504466] bch_prio_write+0x1dc/0x390 [bcache] [ 1158.504476] bch_allocator_thread+0x233/0x490 [bcache] [ 1158.504491] kthread+0x121/0x140 [ 1158.504503] ? invalidate_buckets+0x890/0x890 [bcache] [ 1158.504506] ? kthread_park+0xb0/0xb0 [ 1158.504510] ret_from_fork+0x35/0x40Fix by making the call to
bch_prio_write()non-blocking, so thatbch_allocator_thread()never waits on itself.Moreover, make sure to wake up the garbage collector thread when
bch_prio_write()is failing to allocate buckets.Signed-off-by Andrea Righi <[email protected]>---
Changes in v2:
- preventretry_invalidatebusy loop inbch_allocator_thread()drivers/md/bcache/alloc.c | 5 ++++- drivers/md/bcache/bcache.h | 2 +- drivers/md/bcache/super.c | 13 +++++++++---- 3 files changed, 14 insertions(+), 6 deletions(-)drivers/md/bcache/alloc.c +4
-1@@ -377,7 +377,10 @@ static int bch_allocator_thread(void *arg)if (!fifo_full (&ca->free_inc)) goto retry_invalidate; -bch_prio_write (ca); +if (bch_prio_write (ca, false) <0 ) { + ca->invalidate_needs_gc =1 ; +wake_up_gc (ca->set); + } } } out:drivers/md/bcache/bcache.h +1
-1@@ -977,7 +977,7 @@ bool bch_cached_dev_error(struct cached_dev *dc); __printf(2, 3)bool bch_cache_set_error (struct cache_set *c,const char *fmt, ...); -void bch_prio_write (struct cache *ca); +int bch_prio_write (struct cache *ca,bool wait);void bch_write_bdev_super (struct cached_dev *dc,struct closure *parent);extern struct workqueue_struct *bcache_wq;drivers/md/bcache/super.c +1
-1@@ -529,7 +529,7 @@ static void prio_io(struct cache *ca, uint64_t bucket, int op,closure_sync (cl); } -void bch_prio_write (struct cache *ca) +int bch_prio_write (struct cache *ca,bool wait) {int i;struct bucket *b;@@ -564,8 +564,12 @@ void bch_prio_write(struct cache *ca) p->magic =pset_magic (&ca->sb);drivers/md/bcache/super.c void bch_prio_write(struct cache *ca) · line 572
Quoted 10 lines
drivers/md/bcache/super.c +6
-2p->csum =bch_crc64 (&p->magic,bucket_bytes (ca) -8 ); - bucket =bch_bucket_alloc (ca,RESERVE_PRIO , true); -BUG_ON (bucket ==-1 ); + bucket =bch_bucket_alloc (ca,RESERVE_PRIO , wait); +if (bucket ==-1 ) { +if (!wait) +return -ENOMEM ; +BUG_ON (1 ); + }Coly,
looking more at this change, I think we should handle the failure path properly or we may leak buckets, am I right? (sorry for not realizing this before). Maybe we need something like the following on top of my previous patch.
I'm going to run more stress tests with this patch applied and will try to figure out if we're actually leaking buckets without it.
---
Subject: bcache: prevent leaking buckets inbch_prio_write()
Quoted 7 lines
Quoted 4 lines
Handle the allocation failure path properly in
bch_prio_write()to avoid leaking buckets from the previous successful iterations.Signed-off-by Andrea Righi <[email protected]>Coly, ignore this one please. A v3 of the previous patch with a better fix for this potential buckets leak is on the way.
Sure, waiting for next version :-)
Signature · 1 line
Coly Li