Skip to contentSkip to conversation

bcache: fix race between setting bdev state to none and new write request direct to backing

PATCH In review Dongsheng Yang 2 participants 3 messages Oct 14, 2020 – Nov 12, 2020 bcache (block layer cache)

lore
Share & tools
lore
#0 lore raw

PATCH 1/2 bcache: fix race between setting bdev state to none and new write request direct to backing

There is a race condition in detaching as below:
A. detaching B. Write request
(1) writing back
(2) write back done, set bdev state to clean.
(3) cached_dev_put() and schedule_work(&dc->detach);
(4) write data [0 - 4K] directly into backing and ack to user.
(5) power-failure...

When we restart this bcache device, this bdev is clean but not detached, and read [0 - 4K], we will get unexpected old data from cache device.

To fix this problem, set the bdev state to none when we writeback done in detaching, and then if power-failure happend as above, the data in cache will not be used in next bcache device starting, it's detached, we will read the correct data from backing derectly.

---
 drivers/md/bcache/super.c     | 9 ---------
 drivers/md/bcache/writeback.c | 9 +++++++++
 2 files changed, 9 insertions(+), 9 deletions(-)
drivers/md/bcache/super.c +0 -9
index 1bbdc41..9298fc7 100644
@@ -1128,9 +1128,6 @@ static void cancel_writeback_rate_update_dwork(struct cached_dev *dc)
 static void cached_dev_detach_finish(struct work_struct *w)
 {
 	struct cached_dev *dc = container_of(w, struct cached_dev, detach);
-	struct closure cl;
-
-	closure_init_stack(&cl);
 
 	BUG_ON(!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags));
 	BUG_ON(refcount_read(&dc->count));
@@ -1144,12 +1141,6 @@ static void cached_dev_detach_finish(struct work_struct *w)
 		dc->writeback_thread = NULL;
 	}
 
-	memset(&dc->sb.set_uuid, 0, 16);
-	SET_BDEV_STATE(&dc->sb, BDEV_STATE_NONE);
-
-	bch_write_bdev_super(dc, &cl);
-	closure_sync(&cl);
-
 	mutex_lock(&bch_register_lock);
 
 	calc_cached_dev_sectors(dc->disk.c);
drivers/md/bcache/writeback.c +9 -0
index 4f4ad6b..2cd0340 100644
@@ -705,6 +705,15 @@ static int bch_writeback_thread(void *arg)
 			 * bch_cached_dev_detach().
 			 */
 			if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags)) {
+				struct closure cl;
+
+				closure_init_stack(&cl);
+				memset(&dc->sb.set_uuid, 0, 16);
+				SET_BDEV_STATE(&dc->sb, BDEV_STATE_NONE);
+
+				bch_write_bdev_super(dc, &cl);
+				closure_sync(&cl);
+
 				up_write(&dc->writeback_lock);
 				break;
 			}
Signature · 1 line
1.8.3.1
#1 lore raw

PATCH 2/2 bcache: send request to backing when detaching and there is no dirty

If bcache device is detaching and there is no dirty data in cache, we should send request to backing rather than get dc->count to do cached_dev_read or cached_dev_write.

Otherwise, if there are lots of requests here to call cached_dev_get(dc), we could never put dc->count to be 0 where calling schedule_work(&dc->detach) to finish detaching, that means this detaching will never finish.

This commit introduce a new bit in bcache dev flags BCACHE_DEV_DETACHING_WB_DONE, this will be set when we finish writeback in detaching and cleared in cached_dev_detach_finish(). when the request find BCACHE_DEV_DETACHING_WB_DONE is set, that means, this request should go into backing device directly.

---
 drivers/md/bcache/bcache.h    |  1 +
 drivers/md/bcache/request.c   | 11 ++++++++++-
 drivers/md/bcache/super.c     |  1 +
 drivers/md/bcache/writeback.c |  7 +++++++
 4 files changed, 19 insertions(+), 1 deletion(-)
drivers/md/bcache/bcache.h +1 -0
index 870f146..aedf211 100644
@@ -265,6 +265,7 @@ struct bcache_device {
 #define BCACHE_DEV_UNLINK_DONE		2
 #define BCACHE_DEV_WB_RUNNING		3
 #define BCACHE_DEV_RATE_DW_RUNNING	4
+#define BCACHE_DEV_DETACHING_WB_DONE	5
 	int			nr_stripes;
 	unsigned int		stripe_size;
 	atomic_t		*stripe_sectors_dirty;
drivers/md/bcache/request.c +10 -1
index c7cadaa..94c44f2 100644
@@ -1194,7 +1194,16 @@ blk_qc_t cached_dev_submit_bio(struct bio *bio)
 	bio_set_dev(bio, dc->bdev);
 	bio->bi_iter.bi_sector += dc->sb.data_offset;
 
-	if (cached_dev_get(dc)) {
+	/*
+	 * If BCACHE_DEV_DETACHING_WB_DONE is set,
+	 * we should send request to backing rather than get dc->count to do
+	 * cached_dev_read or cached_dev_write. Otherwise, if there are lots
+	 * of requests here to call cached_dev_get(dc), we could never put
+	 * dc->count to be 0 where calling schedule_work(&dc->detach) to finish
+	 * detaching, that means this detaching will never finish.
+	 */
+	if (!test_bit(BCACHE_DEV_DETACHING_WB_DONE, &dc->disk.flags) &&
+	    cached_dev_get(dc)) {
 		s = search_alloc(bio, d);
 		trace_bcache_request_start(s->d, bio);
 
drivers/md/bcache/super.c +1 -0
index 9298fc7..3c32ddb 100644
@@ -1147,6 +1147,7 @@ static void cached_dev_detach_finish(struct work_struct *w)
 	bcache_device_detach(&dc->disk);
 	list_move(&dc->list, &uncached_devices);
 
+	clear_bit(BCACHE_DEV_DETACHING_WB_DONE, &dc->disk.flags);
 	clear_bit(BCACHE_DEV_DETACHING, &dc->disk.flags);
 	clear_bit(BCACHE_DEV_UNLINK_DONE, &dc->disk.flags);
 
drivers/md/bcache/writeback.c +7 -0
index 2cd0340..6799f99 100644
@@ -714,6 +714,13 @@ static int bch_writeback_thread(void *arg)
 				bch_write_bdev_super(dc, &cl);
 				closure_sync(&cl);
 
+				/*
+				 * BCACHE_DEV_DETACHING_WB_DONE will tell
+				 * request should be sent directly to backing
+				 * device directly, without getting dc->count.
+				 */
+				set_bit(BCACHE_DEV_DETACHING_WB_DONE,
+					&dc->disk.flags);
 				up_write(&dc->writeback_lock);
 				break;
 			}
Signature · 1 line
1.8.3.1
#2 lore raw
In reply to Dongsheng Yang
Quoted 8 lines · Dongsheng Yang

There is a race condition in detaching as below:
A. detaching B. Write request
(1) writing back
(2) write back done, set bdev state to clean.
(3) cached_dev_put() and schedule_work(&dc->detach);
(4) write data [0 - 4K] directly into backing and ack to user.
(5) power-failure...

Quoted 8 lines

When we restart this bcache device, this bdev is clean but not detached, and read [0 - 4K], we will get unexpected old data from cache device.

To fix this problem, set the bdev state to none when we writeback done in detaching, and then if power-failure happend as above, the data in cache will not be used in next bcache device starting, it's detached, we will read the correct data from backing derectly.

Hi Dongsheng,

It takes me for a while to understand and make sure the whole code flow works correctly. Thank you for catching such a very rare problem. IMHO this patch is cool, I will add it into the for-test queue.

Thanks.

Coly Li

Quoted 53 lines
---
 drivers/md/bcache/super.c     | 9 ---------
 drivers/md/bcache/writeback.c | 9 +++++++++
 2 files changed, 9 insertions(+), 9 deletions(-)
drivers/md/bcache/super.c +0 -9
index 1bbdc41..9298fc7 100644
@@ -1128,9 +1128,6 @@ static void cancel_writeback_rate_update_dwork(struct cached_dev *dc)
 static void cached_dev_detach_finish(struct work_struct *w)
 {
 	struct cached_dev *dc = container_of(w, struct cached_dev, detach);
-	struct closure cl;
-
-	closure_init_stack(&cl);
 
 	BUG_ON(!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags));
 	BUG_ON(refcount_read(&dc->count));
@@ -1144,12 +1141,6 @@ static void cached_dev_detach_finish(struct work_struct *w)
 		dc->writeback_thread = NULL;
 	}
 
-	memset(&dc->sb.set_uuid, 0, 16);
-	SET_BDEV_STATE(&dc->sb, BDEV_STATE_NONE);
-
-	bch_write_bdev_super(dc, &cl);
-	closure_sync(&cl);
-
 	mutex_lock(&bch_register_lock);
 
 	calc_cached_dev_sectors(dc->disk.c);
drivers/md/bcache/writeback.c +9 -0
index 4f4ad6b..2cd0340 100644
@@ -705,6 +705,15 @@ static int bch_writeback_thread(void *arg)
 			 * bch_cached_dev_detach().
 			 */
 			if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags)) {
+				struct closure cl;
+
+				closure_init_stack(&cl);
+				memset(&dc->sb.set_uuid, 0, 16);
+				SET_BDEV_STATE(&dc->sb, BDEV_STATE_NONE);
+
+				bch_write_bdev_super(dc, &cl);
+				closure_sync(&cl);
+
 				up_write(&dc->writeback_lock);
 				break;
 			}