Skip to contentSkip to conversation

xen-blkfront: introduce blkfront_gather_backend_features()

PATCH Superseded Bob Liu 2 participants 17 messages Jul 21, 2015 – Jul 22, 2015

Subsystems (3)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Bob Liu

author
#0 lore raw

PATCH 1/3 xen-blkfront: introduce blkfront_gather_backend_features()

There is a bug when migrate from !feature-persistent host to feature-persistent host, because domU still think new host/backend don't support persistent. Dmesg like:
backed has not unmapped grant: 839
backed has not unmapped grant: 773
backed has not unmapped grant: 773
backed has not unmapped grant: 773
backed has not unmapped grant: 839

The fix is to recheck feature-persistent of new backend in blkif_recover().
See: https://lkml.org/lkml/2015/5/25/469

As Roger suggested, we can split the part of blkfront_connect that checks for optional features, like persistent grants, indirect descriptors and flush/barrier features to a separate function and call it from both blkfront_connect and blkif_recover

---
 drivers/block/xen-blkfront.c |  118 +++++++++++++++++++++++-------------------
 1 file changed, 66 insertions(+), 52 deletions(-)
drivers/block/xen-blkfront.c +66 -52
index 5b45ee5..e266d17 100644
@@ -181,6 +181,7 @@ static DEFINE_SPINLOCK(minor_lock);
 	((_segs + SEGS_PER_INDIRECT_FRAME - 1)/SEGS_PER_INDIRECT_FRAME)
 
 static int blkfront_setup_indirect(struct blkfront_info *info);
+static void blkfront_gather_backend_features(struct blkfront_info *info);
 
 static int get_id_from_freelist(struct blkfront_info *info)
 {
@@ -1514,6 +1515,7 @@ static int blkif_recover(struct blkfront_info *info)
 	info->shadow_free = info->ring.req_prod_pvt;
 	info->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff;
 
+	blkfront_gather_backend_features(info);
 	rc = blkfront_setup_indirect(info);
 	if (rc) {
 		kfree(copy);
@@ -1694,20 +1696,13 @@ static void blkfront_setup_discard(struct blkfront_info *info)
 
 static int blkfront_setup_indirect(struct blkfront_info *info)
 {
-	unsigned int indirect_segments, segs;
+	unsigned int segs;
 	int err, i;
 
-	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
-			    "feature-max-indirect-segments", "%u", &indirect_segments,
-			    NULL);
-	if (err) {
-		info->max_indirect_segments = 0;
+	if (info->max_indirect_segments == 0)
 		segs = BLKIF_MAX_SEGMENTS_PER_REQUEST;
-	} else {
-		info->max_indirect_segments = min(indirect_segments,
-						  xen_blkif_max_segments);
+	else
 		segs = info->max_indirect_segments;
-	}
 
 	err = fill_grant_buffer(info, (segs + INDIRECT_GREFS(segs)) * BLK_RING_SIZE(info));
 	if (err)
@@ -1771,6 +1766,66 @@ out_of_memory:
 }
 
 /*
+ * Gather all backend feature-*
+ */
+static void blkfront_gather_backend_features(struct blkfront_info *info)
+{
+	int err;
+	int barrier, flush, discard, persistent;
+	unsigned int indirect_segments;
+
+	info->feature_flush = 0;
+
+	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
+			"feature-barrier", "%d", &barrier,
+			NULL);
+
+	/*
+	 * If there's no "feature-barrier" defined, then it means
+	 * we're dealing with a very old backend which writes
+	 * synchronously; nothing to do.
+	 *
+	 * If there are barriers, then we use flush.
+	 */
+	if (!err && barrier)
+		info->feature_flush = REQ_FLUSH | REQ_FUA;
+	/*
+	 * And if there is "feature-flush-cache" use that above
+	 * barriers.
+	 */
+	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
+			"feature-flush-cache", "%d", &flush,
+			NULL);
+
+	if (!err && flush)
+		info->feature_flush = REQ_FLUSH;
+
+	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
+			"feature-discard", "%d", &discard,
+			NULL);
+
+	if (!err && discard)
+		blkfront_setup_discard(info);
+
+	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
+			"feature-persistent", "%u", &persistent,
+			NULL);
+	if (err)
+		info->feature_persistent = 0;
+	else
+		info->feature_persistent = persistent;
+
+	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
+			    "feature-max-indirect-segments", "%u", &indirect_segments,
+			    NULL);
+	if (err)
+		info->max_indirect_segments = 0;
+	else
+		info->max_indirect_segments = min(indirect_segments,
+						  xen_blkif_max_segments);
+}
+
+/*
  * Invoked when the backend is finally 'ready' (and has told produced
  * the details about the physical device - #sectors, size, etc).
  */
@@ -1781,7 +1836,6 @@ static void blkfront_connect(struct blkfront_info *info)
 	unsigned int physical_sector_size;
 	unsigned int binfo;
 	int err;
-	int barrier, flush, discard, persistent;
 
 	switch (info->connected) {
 	case BLKIF_STATE_CONNECTED:
@@ -1838,47 +1892,7 @@ static void blkfront_connect(struct blkfront_info *info)
 	if (err != 1)
 		physical_sector_size = sector_size;
 
-	info->feature_flush = 0;
-
-	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
-			    "feature-barrier", "%d", &barrier,
-			    NULL);
-
-	/*
-	 * If there's no "feature-barrier" defined, then it means
-	 * we're dealing with a very old backend which writes
-	 * synchronously; nothing to do.
-	 *
-	 * If there are barriers, then we use flush.
-	 */
-	if (!err && barrier)
-		info->feature_flush = REQ_FLUSH | REQ_FUA;
-	/*
-	 * And if there is "feature-flush-cache" use that above
-	 * barriers.
-	 */
-	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
-			    "feature-flush-cache", "%d", &flush,
-			    NULL);
-
-	if (!err && flush)
-		info->feature_flush = REQ_FLUSH;
-
-	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
-			    "feature-discard", "%d", &discard,
-			    NULL);
-
-	if (!err && discard)
-		blkfront_setup_discard(info);
-
-	err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
-			    "feature-persistent", "%u", &persistent,
-			    NULL);
-	if (err)
-		info->feature_persistent = 0;
-	else
-		info->feature_persistent = persistent;
-
+	blkfront_gather_backend_features(info);
 	err = blkfront_setup_indirect(info);
 	if (err) {
 		xenbus_dev_fatal(info->xbdev, err, "setup_indirect at %s",
Signature · 1 line
1.7.10.4
author
#1 lore raw

PATCH 2/3 xen-blkfront: rm BUG_ON(info->feature_persistent) in blkif_free

This BUG_ON() in blkif_free() is incorrect, because indirect page can be added to list info->indirect_pages in blkif_completion() no matter feature_persistent is true or false.

---
 drivers/block/xen-blkfront.c |    1 -
 1 file changed, 1 deletion(-)
drivers/block/xen-blkfront.c +0 -1
index e266d17..c98fcd0 100644
@@ -986,7 +986,6 @@ static void blkif_free(struct blkfront_info *info, int suspend)
 	if (!list_empty(&info->indirect_pages)) {
 		struct page *indirect_page, *n;
 
-		BUG_ON(info->feature_persistent);
 		list_for_each_entry_safe(indirect_page, n, &info->indirect_pages, lru) {
 			list_del(&indirect_page->lru);
 			__free_page(indirect_page);
Signature · 1 line
1.7.10.4
Acked-by on v1 2/3
#2 lore raw

Re: [PATCH 2/3] xen-blkfront: rm BUG_ON(info->feature_persistent) in blkif_free

El 21/07/15 a les 5.30, Bob Liu ha escrit:

Quoted 5 lines

This BUG_ON() in blkif_free() is incorrect, because indirect page can be added to list info->indirect_pages in blkif_completion() no matter feature_persistent is true or false.

This was probably an oversight from when blkif_completion was changed to check for gnttab_query_foreign_access. It should be backported to stable trees.

Roger.

author
#3 lore raw

Re: [PATCH 2/3] xen-blkfront: rm BUG_ON(info->feature_persistent) in blkif_free

Quoted 5 lines · Roger Pau Monné

El 21/07/15 a les 5.30, Bob Liu ha escrit:

Quoted 4 lines

This BUG_ON() in blkif_free() is incorrect, because indirect page can be added to list info->indirect_pages in blkif_completion() no matter feature_persistent is true or false.

Quoted 8 lines
Quoted 1 line

This was probably an oversight from when blkif_completion was changed to check for gnttab_query_foreign_access. It should be backported to stable trees.

Sorry, this patch is buggy and I haven't figure out why.

general protection fault: 0000 [#1] SMP 
Modules linked in:
CPU: 0 PID: 39 Comm: xenwatch Tainted: G        W       4.1.0-rc3-00003-g718cf80-dirty #67
Hardware name: Xen HVM domU, BIOS 4.5.0-rc 11/23/2014
task: ffff880283f4eca0 ti: ffff880283fb4000 task.ti: ffff880283fb4000
RIP: 0010:[<ffffffff813d577b>]  [<ffffffff813d577b>] blkif_free+0x162/0x5a9
RSP: 0018:ffff880283fb7c48  EFLAGS: 00010087
RAX: dead000000200200 RBX: ffff880141400000 RCX: 0000000000000000
RDX: dead000000100100 RSI: dead000000100100 RDI: ffff88028f418bb8
RBP: ffff880283fb7ca8 R08: dead000000200200 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000000 R12: ffff8801414481c8
R13: dead0000001000e0 R14: ffff8801414481b8 R15: ffffea0000000000
FS:  0000000000000000(0000) GS:ffff88028f400000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001582e08 CR3: 000000013345b000 CR4: 00000000001406f0
Stack:
 ffff880023aa8420 0000000000000286 ffff880283fb7cb7 ffff880023aa8420
 ffff8800363fe240 ffffffff81862c50 ffff880283fb7ce8 ffff880023aa8440
 ffffffff81870000 ffff880023aa8400 ffff880141400000 ffff880141400008
Call Trace:
 [<ffffffff813d8e76>] blkfront_remove+0x4c/0xff
 [<ffffffff813772fa>] xenbus_dev_remove+0x76/0xb0
 [<ffffffff813bd611>] __device_release_driver+0x84/0xf8
 [<ffffffff813bd6a3>] device_release_driver+0x1e/0x2b
 [<ffffffff813bd1ef>] bus_remove_device+0x12c/0x141
 [<ffffffff813ba51d>] device_del+0x161/0x1e5
 [<ffffffff81375ef3>] ? xenbus_thread+0x239/0x239
 [<ffffffff813ba5e4>] device_unregister+0x43/0x4f
 [<ffffffff81377853>] xenbus_dev_changed+0x82/0x17f
 [<ffffffff81377566>] ? xenbus_otherend_changed+0xf0/0xff
 [<ffffffff81378d8d>] frontend_changed+0x43/0x48
 [<ffffffff81375fec>] xenwatch_thread+0xf9/0x127
 [<ffffffff81078fc4>] ? add_wait_queue+0x44/0x44
 [<ffffffff8106195b>] kthread+0xcd/0xd5
 [<ffffffff81060000>] ? alloc_pid+0xe8/0x492
 [<ffffffff8106188e>] ? kthread_freezable_should_stop+0x48/0x48
 [<ffffffff81533ee2>] ret_from_fork+0x42/0x70
 [<ffffffff8106188e>] ? kthread_freezable_should_stop+0x48/0x48
Code: 04 00 4c 8b 28 48 8d 78 e0 49 83 ed 20 eb 3d 48 8b 47 28 48 8b 57 20 48 be 00 01 10 00 00 00 ad de 49 b8 00 02 20 00 00 00 ad de <48> 89 42 08 48 89 10 48 89 77 20 4c 89 47 28 31 f6 e8 26 7d cf 
RIP  [<ffffffff813d577b>] blkif_free+0x162/0x5a9
 RSP <ffff880283fb7c48>
---[ end trace 5321d7f1ef8414d0 ]---
author
#4 lore raw

Re: [PATCH 2/3] xen-blkfront: rm BUG_ON(info->feature_persistent) in blkif_free

In reply to Roger Pau Monné
Quoted 5 lines · Roger Pau Monné

El 21/07/15 a les 5.30, Bob Liu ha escrit:

Quoted 4 lines

This BUG_ON() in blkif_free() is incorrect, because indirect page can be added to list info->indirect_pages in blkif_completion() no matter feature_persistent is true or false.

Quoted 8 lines
Quoted 1 line

This was probably an oversight from when blkif_completion was changed to check for gnttab_query_foreign_access. It should be backported to stable trees.

Sorry, this patch is buggy and I haven't figure out why.

general protection fault: 0000 [#1] SMP 
Modules linked in:
CPU: 0 PID: 39 Comm: xenwatch Tainted: G        W       4.1.0-rc3-00003-g718cf80-dirty #67
Hardware name: Xen HVM domU, BIOS 4.5.0-rc 11/23/2014
task: ffff880283f4eca0 ti: ffff880283fb4000 task.ti: ffff880283fb4000
RIP: 0010:[<ffffffff813d577b>]  [<ffffffff813d577b>] blkif_free+0x162/0x5a9
RSP: 0018:ffff880283fb7c48  EFLAGS: 00010087
RAX: dead000000200200 RBX: ffff880141400000 RCX: 0000000000000000
RDX: dead000000100100 RSI: dead000000100100 RDI: ffff88028f418bb8
RBP: ffff880283fb7ca8 R08: dead000000200200 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000000 R12: ffff8801414481c8
R13: dead0000001000e0 R14: ffff8801414481b8 R15: ffffea0000000000
FS:  0000000000000000(0000) GS:ffff88028f400000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001582e08 CR3: 000000013345b000 CR4: 00000000001406f0
Stack:
 ffff880023aa8420 0000000000000286 ffff880283fb7cb7 ffff880023aa8420
 ffff8800363fe240 ffffffff81862c50 ffff880283fb7ce8 ffff880023aa8440
 ffffffff81870000 ffff880023aa8400 ffff880141400000 ffff880141400008
Call Trace:
 [<ffffffff813d8e76>] blkfront_remove+0x4c/0xff
 [<ffffffff813772fa>] xenbus_dev_remove+0x76/0xb0
 [<ffffffff813bd611>] __device_release_driver+0x84/0xf8
 [<ffffffff813bd6a3>] device_release_driver+0x1e/0x2b
 [<ffffffff813bd1ef>] bus_remove_device+0x12c/0x141
 [<ffffffff813ba51d>] device_del+0x161/0x1e5
 [<ffffffff81375ef3>] ? xenbus_thread+0x239/0x239
 [<ffffffff813ba5e4>] device_unregister+0x43/0x4f
 [<ffffffff81377853>] xenbus_dev_changed+0x82/0x17f
 [<ffffffff81377566>] ? xenbus_otherend_changed+0xf0/0xff
 [<ffffffff81378d8d>] frontend_changed+0x43/0x48
 [<ffffffff81375fec>] xenwatch_thread+0xf9/0x127
 [<ffffffff81078fc4>] ? add_wait_queue+0x44/0x44
 [<ffffffff8106195b>] kthread+0xcd/0xd5
 [<ffffffff81060000>] ? alloc_pid+0xe8/0x492
 [<ffffffff8106188e>] ? kthread_freezable_should_stop+0x48/0x48
 [<ffffffff81533ee2>] ret_from_fork+0x42/0x70
 [<ffffffff8106188e>] ? kthread_freezable_should_stop+0x48/0x48
Code: 04 00 4c 8b 28 48 8d 78 e0 49 83 ed 20 eb 3d 48 8b 47 28 48 8b 57 20 48 be 00 01 10 00 00 00 ad de 49 b8 00 02 20 00 00 00 ad de <48> 89 42 08 48 89 10 48 89 77 20 4c 89 47 28 31 f6 e8 26 7d cf 
RIP  [<ffffffff813d577b>] blkif_free+0x162/0x5a9
 RSP <ffff880283fb7c48>
---[ end trace 5321d7f1ef8414d0 ]---
author
#5 lore raw

Re: [PATCH 2/3] xen-blkfront: rm BUG_ON(info->feature_persistent) in blkif_free

Quoted 52 lines · Bob Liu
Quoted 5 lines · Roger Pau Monné

El 21/07/15 a les 5.30, Bob Liu ha escrit:

Quoted 4 lines

This BUG_ON() in blkif_free() is incorrect, because indirect page can be added to list info->indirect_pages in blkif_completion() no matter feature_persistent is true or false.

Quoted 8 lines
Quoted 1 line

This was probably an oversight from when blkif_completion was changed to check for gnttab_query_foreign_access. It should be backported to stable trees.

Sorry, this patch is buggy and I haven't figure out why.

general protection fault: 0000 [#1] SMP 
Modules linked in:
CPU: 0 PID: 39 Comm: xenwatch Tainted: G        W       4.1.0-rc3-00003-g718cf80-dirty #67
Hardware name: Xen HVM domU, BIOS 4.5.0-rc 11/23/2014
task: ffff880283f4eca0 ti: ffff880283fb4000 task.ti: ffff880283fb4000
RIP: 0010:[<ffffffff813d577b>]  [<ffffffff813d577b>] blkif_free+0x162/0x5a9
RSP: 0018:ffff880283fb7c48  EFLAGS: 00010087
RAX: dead000000200200 RBX: ffff880141400000 RCX: 0000000000000000
RDX: dead000000100100 RSI: dead000000100100 RDI: ffff88028f418bb8
RBP: ffff880283fb7ca8 R08: dead000000200200 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000000 R12: ffff8801414481c8
R13: dead0000001000e0 R14: ffff8801414481b8 R15: ffffea0000000000
FS:  0000000000000000(0000) GS:ffff88028f400000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001582e08 CR3: 000000013345b000 CR4: 00000000001406f0
Stack:
 ffff880023aa8420 0000000000000286 ffff880283fb7cb7 ffff880023aa8420
 ffff8800363fe240 ffffffff81862c50 ffff880283fb7ce8 ffff880023aa8440
 ffffffff81870000 ffff880023aa8400 ffff880141400000 ffff880141400008
Call Trace:
 [<ffffffff813d8e76>] blkfront_remove+0x4c/0xff
 [<ffffffff813772fa>] xenbus_dev_remove+0x76/0xb0
 [<ffffffff813bd611>] __device_release_driver+0x84/0xf8
 [<ffffffff813bd6a3>] device_release_driver+0x1e/0x2b
 [<ffffffff813bd1ef>] bus_remove_device+0x12c/0x141
 [<ffffffff813ba51d>] device_del+0x161/0x1e5
 [<ffffffff81375ef3>] ? xenbus_thread+0x239/0x239
 [<ffffffff813ba5e4>] device_unregister+0x43/0x4f
 [<ffffffff81377853>] xenbus_dev_changed+0x82/0x17f
 [<ffffffff81377566>] ? xenbus_otherend_changed+0xf0/0xff
 [<ffffffff81378d8d>] frontend_changed+0x43/0x48
 [<ffffffff81375fec>] xenwatch_thread+0xf9/0x127
 [<ffffffff81078fc4>] ? add_wait_queue+0x44/0x44
 [<ffffffff8106195b>] kthread+0xcd/0xd5
Quoted 9 lines
 [<ffffffff81060000>] ? alloc_pid+0xe8/0x492
 [<ffffffff8106188e>] ? kthread_freezable_should_stop+0x48/0x48
 [<ffffffff81533ee2>] ret_from_fork+0x42/0x70
 [<ffffffff8106188e>] ? kthread_freezable_should_stop+0x48/0x48
Code: 04 00 4c 8b 28 48 8d 78 e0 49 83 ed 20 eb 3d 48 8b 47 28 48 8b 57 20 48 be 00 01 10 00 00 00 ad de 49 b8 00 02 20 00 00 00 ad de <48> 89 42 08 48 89 10 48 89 77 20 4c 89 47 28 31 f6 e8 26 7d cf 
RIP  [<ffffffff813d577b>] blkif_free+0x162/0x5a9
 RSP <ffff880283fb7c48>
---[ end trace 5321d7f1ef8414d0 ]---

The right fix should be:

drivers/block/xen-blkfront.c +4 -2
@@ -1124,8 +1124,10 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_info *info,
                                 * Add the used indirect page back to the list of
                                 * available pages for indirect grefs.
                                 */
-                               indirect_page = pfn_to_page(s->indirect_grants[i]->pfn);
-                               list_add(&indirect_page->lru, &info->indirect_pages);
+                               if (!info->feature_persistent) {
+                                       indirect_page = pfn_to_page(s->indirect_grants[i]->pfn);
+                                       list_add(&indirect_page->lru, &info->indirect_pages);
+                               }
                                s->indirect_grants[i]->gref = GRANT_INVALID_REF;
                                list_add_tail(&s->indirect_grants[i]->node, &info->grants);
                        }
author
#6 lore raw

Re: [PATCH 2/3] xen-blkfront: rm BUG_ON(info->feature_persistent) in blkif_free

In reply to Bob Liu
Quoted 52 lines · Bob Liu
Quoted 5 lines · Roger Pau Monné

El 21/07/15 a les 5.30, Bob Liu ha escrit:

Quoted 4 lines

This BUG_ON() in blkif_free() is incorrect, because indirect page can be added to list info->indirect_pages in blkif_completion() no matter feature_persistent is true or false.

Quoted 8 lines
Quoted 1 line

This was probably an oversight from when blkif_completion was changed to check for gnttab_query_foreign_access. It should be backported to stable trees.

Sorry, this patch is buggy and I haven't figure out why.

general protection fault: 0000 [#1] SMP 
Modules linked in:
CPU: 0 PID: 39 Comm: xenwatch Tainted: G        W       4.1.0-rc3-00003-g718cf80-dirty #67
Hardware name: Xen HVM domU, BIOS 4.5.0-rc 11/23/2014
task: ffff880283f4eca0 ti: ffff880283fb4000 task.ti: ffff880283fb4000
RIP: 0010:[<ffffffff813d577b>]  [<ffffffff813d577b>] blkif_free+0x162/0x5a9
RSP: 0018:ffff880283fb7c48  EFLAGS: 00010087
RAX: dead000000200200 RBX: ffff880141400000 RCX: 0000000000000000
RDX: dead000000100100 RSI: dead000000100100 RDI: ffff88028f418bb8
RBP: ffff880283fb7ca8 R08: dead000000200200 R09: 0000000000000001
R10: 0000000000000001 R11: 0000000000000000 R12: ffff8801414481c8
R13: dead0000001000e0 R14: ffff8801414481b8 R15: ffffea0000000000
FS:  0000000000000000(0000) GS:ffff88028f400000(0000) knlGS:0000000000000000
CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000000001582e08 CR3: 000000013345b000 CR4: 00000000001406f0
Stack:
 ffff880023aa8420 0000000000000286 ffff880283fb7cb7 ffff880023aa8420
 ffff8800363fe240 ffffffff81862c50 ffff880283fb7ce8 ffff880023aa8440
 ffffffff81870000 ffff880023aa8400 ffff880141400000 ffff880141400008
Call Trace:
 [<ffffffff813d8e76>] blkfront_remove+0x4c/0xff
 [<ffffffff813772fa>] xenbus_dev_remove+0x76/0xb0
 [<ffffffff813bd611>] __device_release_driver+0x84/0xf8
 [<ffffffff813bd6a3>] device_release_driver+0x1e/0x2b
 [<ffffffff813bd1ef>] bus_remove_device+0x12c/0x141
 [<ffffffff813ba51d>] device_del+0x161/0x1e5
 [<ffffffff81375ef3>] ? xenbus_thread+0x239/0x239
 [<ffffffff813ba5e4>] device_unregister+0x43/0x4f
 [<ffffffff81377853>] xenbus_dev_changed+0x82/0x17f
 [<ffffffff81377566>] ? xenbus_otherend_changed+0xf0/0xff
 [<ffffffff81378d8d>] frontend_changed+0x43/0x48
 [<ffffffff81375fec>] xenwatch_thread+0xf9/0x127
 [<ffffffff81078fc4>] ? add_wait_queue+0x44/0x44
 [<ffffffff8106195b>] kthread+0xcd/0xd5
Quoted 9 lines
 [<ffffffff81060000>] ? alloc_pid+0xe8/0x492
 [<ffffffff8106188e>] ? kthread_freezable_should_stop+0x48/0x48
 [<ffffffff81533ee2>] ret_from_fork+0x42/0x70
 [<ffffffff8106188e>] ? kthread_freezable_should_stop+0x48/0x48
Code: 04 00 4c 8b 28 48 8d 78 e0 49 83 ed 20 eb 3d 48 8b 47 28 48 8b 57 20 48 be 00 01 10 00 00 00 ad de 49 b8 00 02 20 00 00 00 ad de <48> 89 42 08 48 89 10 48 89 77 20 4c 89 47 28 31 f6 e8 26 7d cf 
RIP  [<ffffffff813d577b>] blkif_free+0x162/0x5a9
 RSP <ffff880283fb7c48>
---[ end trace 5321d7f1ef8414d0 ]---

The right fix should be:

drivers/block/xen-blkfront.c +4 -2
@@ -1124,8 +1124,10 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_info *info,
                                 * Add the used indirect page back to the list of
                                 * available pages for indirect grefs.
                                 */
-                               indirect_page = pfn_to_page(s->indirect_grants[i]->pfn);
-                               list_add(&indirect_page->lru, &info->indirect_pages);
+                               if (!info->feature_persistent) {
+                                       indirect_page = pfn_to_page(s->indirect_grants[i]->pfn);
+                                       list_add(&indirect_page->lru, &info->indirect_pages);
+                               }
                                s->indirect_grants[i]->gref = GRANT_INVALID_REF;
                                list_add_tail(&s->indirect_grants[i]->node, &info->grants);
                        }
Acked-by on v1 2/3
#7 lore raw

Re: [PATCH 2/3] xen-blkfront: rm BUG_ON(info->feature_persistent) in blkif_free

In reply to Bob Liu

El 21/07/15 a les 5.30, Bob Liu ha escrit:

Quoted 5 lines

This BUG_ON() in blkif_free() is incorrect, because indirect page can be added to list info->indirect_pages in blkif_completion() no matter feature_persistent is true or false.

This was probably an oversight from when blkif_completion was changed to check for gnttab_query_foreign_access. It should be backported to stable trees.

Roger.

author
#8 lore raw

PATCH 2/3 xen-blkfront: rm BUG_ON(info->feature_persistent) in blkif_free

In reply to Bob Liu

This BUG_ON() in blkif_free() is incorrect, because indirect page can be added to list info->indirect_pages in blkif_completion() no matter feature_persistent is true or false.

---
 drivers/block/xen-blkfront.c |    1 -
 1 file changed, 1 deletion(-)
drivers/block/xen-blkfront.c +0 -1
index e266d17..c98fcd0 100644
@@ -986,7 +986,6 @@ static void blkif_free(struct blkfront_info *info, int suspend)
 	if (!list_empty(&info->indirect_pages)) {
 		struct page *indirect_page, *n;
 
-		BUG_ON(info->feature_persistent);
 		list_for_each_entry_safe(indirect_page, n, &info->indirect_pages, lru) {
 			list_del(&indirect_page->lru);
 			__free_page(indirect_page);
Signature · 1 line
1.7.10.4
author
#9 lore raw

PATCH 3/3 xen-blkback: rm BUG_ON() in purge_persistent_gnt()

In reply to Bob Liu

This BUG_ON() will be triggered when previous purge work haven't finished. It's reasonable under pretty extreme load and should not panic the system.

---
 drivers/block/xen-blkback/blkback.c |    4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/block/xen-blkback/blkback.c +3 -1
index ced9677..b90ac8e 100644
@@ -394,7 +394,9 @@ static void purge_persistent_gnt(struct xen_blkif *blkif)
 
 	pr_debug("Going to purge %u persistent grants\n", num_clean);
 
-	BUG_ON(!list_empty(&blkif->persistent_purge_list));
+	if (!list_empty(&blkif->persistent_purge_list))
+		return;
+
 	root = &blkif->persistent_gnts;
 purge_list:
 	foreach_grant_safe(persistent_gnt, n, root, node) {
Signature · 1 line
1.7.10.4
author
#10 lore raw

PATCH 3/3 xen-blkback: rm BUG_ON() in purge_persistent_gnt()

In reply to Bob Liu

This BUG_ON() will be triggered when previous purge work haven't finished. It's reasonable under pretty extreme load and should not panic the system.

---
 drivers/block/xen-blkback/blkback.c |    4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/block/xen-blkback/blkback.c +3 -1
index ced9677..b90ac8e 100644
@@ -394,7 +394,9 @@ static void purge_persistent_gnt(struct xen_blkif *blkif)
 
 	pr_debug("Going to purge %u persistent grants\n", num_clean);
 
-	BUG_ON(!list_empty(&blkif->persistent_purge_list));
+	if (!list_empty(&blkif->persistent_purge_list))
+		return;
+
 	root = &blkif->persistent_gnts;
 purge_list:
 	foreach_grant_safe(persistent_gnt, n, root, node) {
Signature · 1 line
1.7.10.4
#11 lore raw

Re: [PATCH 3/3] xen-blkback: rm BUG_ON() in purge_persistent_gnt()

El 21/07/15 a les 5.30, Bob Liu ha escrit:

Quoted 13 lines

This BUG_ON() will be triggered when previous purge work haven't finished. It's reasonable under pretty extreme load and should not panic the system.

---
 drivers/block/xen-blkback/blkback.c |    4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/block/xen-blkback/blkback.c +0 -0
index ced9677..b90ac8e 100644

drivers/block/xen-blkback/blkback.c static void purge_persistent_gnt(struct xen_blkif *blkif) · line 399

Quoted 7 lines
drivers/block/xen-blkback/blkback.c +3 -1
@@ -394,7 +394,9 @@ static void purge_persistent_gnt(struct xen_blkif *blkif)
 
 	pr_debug("Going to purge %u persistent grants\n", num_clean);
 
-	BUG_ON(!list_empty(&blkif->persistent_purge_list));
+	if (!list_empty(&blkif->persistent_purge_list))
+		return;
+

I see the problem with this, there's a check for work_pending before this BUG_ON, but it doesn't account if the work is currently running. I would rather prefer to replace the work_pending check with work_busy instead, which will also take into account if the work is still running. The comment on work_busy however makes me nervous:

* Test whether @work is currently pending or running. There is no
* synchronization around this function and the test result is
* unreliable and only useful as advisory hints or for debugging.

AFAICT I think it should be safe because we don't have concurrent purge_persistent_gnt calls, but I'm no expert on Linux workqueues. It also makes me wonder why we have such a half-baked function in the Linux kernel.

Roger.

#12 lore raw

Re: [PATCH 3/3] xen-blkback: rm BUG_ON() in purge_persistent_gnt()

In reply to Bob Liu

El 21/07/15 a les 5.30, Bob Liu ha escrit:

Quoted 13 lines

This BUG_ON() will be triggered when previous purge work haven't finished. It's reasonable under pretty extreme load and should not panic the system.

---
 drivers/block/xen-blkback/blkback.c |    4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/block/xen-blkback/blkback.c +0 -0
index ced9677..b90ac8e 100644

drivers/block/xen-blkback/blkback.c static void purge_persistent_gnt(struct xen_blkif *blkif) · line 399

Quoted 7 lines
drivers/block/xen-blkback/blkback.c +3 -1
@@ -394,7 +394,9 @@ static void purge_persistent_gnt(struct xen_blkif *blkif)
 
 	pr_debug("Going to purge %u persistent grants\n", num_clean);
 
-	BUG_ON(!list_empty(&blkif->persistent_purge_list));
+	if (!list_empty(&blkif->persistent_purge_list))
+		return;
+

I see the problem with this, there's a check for work_pending before this BUG_ON, but it doesn't account if the work is currently running. I would rather prefer to replace the work_pending check with work_busy instead, which will also take into account if the work is still running. The comment on work_busy however makes me nervous:

* Test whether @work is currently pending or running. There is no
* synchronization around this function and the test result is
* unreliable and only useful as advisory hints or for debugging.

AFAICT I think it should be safe because we don't have concurrent purge_persistent_gnt calls, but I'm no expert on Linux workqueues. It also makes me wonder why we have such a half-baked function in the Linux kernel.

Roger.

author
#13 lore raw

Re: [PATCH 3/3] xen-blkback: rm BUG_ON() in purge_persistent_gnt()

Quoted 17 lines · Roger Pau Monné

El 21/07/15 a les 5.30, Bob Liu ha escrit:

Quoted 16 lines

This BUG_ON() will be triggered when previous purge work haven't finished. It's reasonable under pretty extreme load and should not panic the system.

---
 drivers/block/xen-blkback/blkback.c |    4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/block/xen-blkback/blkback.c +0 -0
index ced9677..b90ac8e 100644
@@ -394,7 +394,9 @@ static void purge_persistent_gnt(struct xen_blkif *blkif)
 
 	pr_debug("Going to purge %u persistent grants\n", num_clean);
 
Quoted 7 lines

drivers/block/xen-blkback/blkback.c static void purge_persistent_gnt(struct xen_blkif *blkif) · line 399

Quoted 4 lines
drivers/block/xen-blkback/blkback.c +3 -1
-	BUG_ON(!list_empty(&blkif->persistent_purge_list));
+	if (!list_empty(&blkif->persistent_purge_list))
+		return;
+

I see the problem with this, there's a check for work_pending before this BUG_ON, but it doesn't account if the work is currently running. I

Exactly.

Quoted 10 lines

would rather prefer to replace the work_pending check with work_busy instead, which will also take into account if the work is still running. The comment on work_busy however makes me nervous:

* Test whether @work is currently pending or running. There is no
* synchronization around this function and the test result is
* unreliable and only useful as advisory hints or for debugging.

AFAICT I think it should be safe because we don't have concurrent purge_persistent_gnt calls, but I'm no expert on Linux workqueues. It

Me neither, that's why I just replace this BUG_ON() with a simple return.

Quoted 3 lines

also makes me wonder why we have such a half-baked function in the Linux kernel.

author
#14 lore raw

Re: [PATCH 3/3] xen-blkback: rm BUG_ON() in purge_persistent_gnt()

In reply to Roger Pau Monné
Quoted 17 lines · Roger Pau Monné

El 21/07/15 a les 5.30, Bob Liu ha escrit:

Quoted 16 lines

This BUG_ON() will be triggered when previous purge work haven't finished. It's reasonable under pretty extreme load and should not panic the system.

---
 drivers/block/xen-blkback/blkback.c |    4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/block/xen-blkback/blkback.c +0 -0
index ced9677..b90ac8e 100644
@@ -394,7 +394,9 @@ static void purge_persistent_gnt(struct xen_blkif *blkif)
 
 	pr_debug("Going to purge %u persistent grants\n", num_clean);
 
Quoted 7 lines

drivers/block/xen-blkback/blkback.c static void purge_persistent_gnt(struct xen_blkif *blkif) · line 399

Quoted 4 lines
drivers/block/xen-blkback/blkback.c +3 -1
-	BUG_ON(!list_empty(&blkif->persistent_purge_list));
+	if (!list_empty(&blkif->persistent_purge_list))
+		return;
+

I see the problem with this, there's a check for work_pending before this BUG_ON, but it doesn't account if the work is currently running. I

Exactly.

Quoted 10 lines

would rather prefer to replace the work_pending check with work_busy instead, which will also take into account if the work is still running. The comment on work_busy however makes me nervous:

* Test whether @work is currently pending or running. There is no
* synchronization around this function and the test result is
* unreliable and only useful as advisory hints or for debugging.

AFAICT I think it should be safe because we don't have concurrent purge_persistent_gnt calls, but I'm no expert on Linux workqueues. It

Me neither, that's why I just replace this BUG_ON() with a simple return.

Quoted 3 lines

also makes me wonder why we have such a half-baked function in the Linux kernel.

#15 lore raw
In reply to Bob Liu

El 21/07/15 a les 5.30, Bob Liu ha escrit:

Quoted 12 lines

There is a bug when migrate from !feature-persistent host to feature-persistent host, because domU still think new host/backend don't support persistent. Dmesg like:
backed has not unmapped grant: 839
backed has not unmapped grant: 773
backed has not unmapped grant: 773
backed has not unmapped grant: 773
backed has not unmapped grant: 839

The fix is to recheck feature-persistent of new backend in blkif_recover().
See: https://lkml.org/lkml/2015/5/25/469

Quoted 6 lines

As Roger suggested, we can split the part of blkfront_connect that checks for optional features, like persistent grants, indirect descriptors and flush/barrier features to a separate function and call it from both blkfront_connect and blkif_recover

Thanks for taking care of this. The patch looks fine, just one minor nit.

Quoted 17 lines
---
 drivers/block/xen-blkfront.c |  118 +++++++++++++++++++++++-------------------
 1 file changed, 66 insertions(+), 52 deletions(-)
drivers/block/xen-blkfront.c +1 -0
index 5b45ee5..e266d17 100644
@@ -181,6 +181,7 @@ static DEFINE_SPINLOCK(minor_lock);
 	((_segs + SEGS_PER_INDIRECT_FRAME - 1)/SEGS_PER_INDIRECT_FRAME)
 
 static int blkfront_setup_indirect(struct blkfront_info *info);
+static void blkfront_gather_backend_features(struct blkfront_info *info);
 
 static int get_id_from_freelist(struct blkfront_info *info)
 {

drivers/block/xen-blkfront.c static int blkif_recover(struct blkfront_info *info) · line 1519

Quoted 5 lines
drivers/block/xen-blkfront.c +1 -0
@@ -1514,6 +1515,7 @@ static int blkif_recover(struct blkfront_info *info)
 	info->shadow_free = info->ring.req_prod_pvt;
 	info->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff;
 
+	blkfront_gather_backend_features(info);
 	rc = blkfront_setup_indirect(info);

AFAICT you can put the call to blkfront_setup_indirect inside of blkfront_gather_backend_features, like it's done for blkfront_setup_discard.

Roger.

#16 lore raw
In reply to Bob Liu

El 21/07/15 a les 5.30, Bob Liu ha escrit:

Quoted 12 lines

There is a bug when migrate from !feature-persistent host to feature-persistent host, because domU still think new host/backend don't support persistent. Dmesg like:
backed has not unmapped grant: 839
backed has not unmapped grant: 773
backed has not unmapped grant: 773
backed has not unmapped grant: 773
backed has not unmapped grant: 839

The fix is to recheck feature-persistent of new backend in blkif_recover().
See: https://lkml.org/lkml/2015/5/25/469

Quoted 6 lines

As Roger suggested, we can split the part of blkfront_connect that checks for optional features, like persistent grants, indirect descriptors and flush/barrier features to a separate function and call it from both blkfront_connect and blkif_recover

Thanks for taking care of this. The patch looks fine, just one minor nit.

Quoted 17 lines
---
 drivers/block/xen-blkfront.c |  118 +++++++++++++++++++++++-------------------
 1 file changed, 66 insertions(+), 52 deletions(-)
drivers/block/xen-blkfront.c +1 -0
index 5b45ee5..e266d17 100644
@@ -181,6 +181,7 @@ static DEFINE_SPINLOCK(minor_lock);
 	((_segs + SEGS_PER_INDIRECT_FRAME - 1)/SEGS_PER_INDIRECT_FRAME)
 
 static int blkfront_setup_indirect(struct blkfront_info *info);
+static void blkfront_gather_backend_features(struct blkfront_info *info);
 
 static int get_id_from_freelist(struct blkfront_info *info)
 {

drivers/block/xen-blkfront.c static int blkif_recover(struct blkfront_info *info) · line 1519

Quoted 5 lines
drivers/block/xen-blkfront.c +1 -0
@@ -1514,6 +1515,7 @@ static int blkif_recover(struct blkfront_info *info)
 	info->shadow_free = info->ring.req_prod_pvt;
 	info->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff;
 
+	blkfront_gather_backend_features(info);
 	rc = blkfront_setup_indirect(info);

AFAICT you can put the call to blkfront_setup_indirect inside of blkfront_gather_backend_features, like it's done for blkfront_setup_discard.

Roger.