mm, memory_hotplug: improve memory offlining failures debugging

From: Michal Hocko <[email protected]>

__dump_page prints the mapping pointer but that is quite unhelpful for many reports because the pointer itself only helps to distinguish anon/ksm mappings from other ones (because of lowest bits set). Sometimes it would be much more helpful to know what kind of mapping that is actually and if we know this is a file mapping then also try to resolve the dentry name.

...

mm/debug.c +2 -0

 
...

@@ -70,6 +71,18 @@ void __dump_page(struct page *page, const char *reason)
 	if (PageCompound(page))
 		pr_cont(" compound_mapcount: %d", compound_mapcount(page));
 	pr_cont("\n");
+	if (PageAnon(page))
+		pr_emerg("anon ");

Quoted 22 lines · On Wed, 7 Nov 2018 11:18:26 +0100 Michal Hocko <[email protected]> wrote:

From: Michal Hocko <[email protected]>

__dump_page prints the mapping pointer but that is quite unhelpful for many reports because the pointer itself only helps to distinguish anon/ksm mappings from other ones (because of lowest bits set). Sometimes it would be much more helpful to know what kind of mapping that is actually and if we know this is a file mapping then also try to resolve the dentry name.

...

mm/debug.c +2 -0

 
...

@@ -70,6 +71,18 @@ void __dump_page(struct page *page, const char *reason)
 	if (PageCompound(page))
 		pr_cont(" compound_mapcount: %d", compound_mapcount(page));
 	pr_cont("\n");
+	if (PageAnon(page))
+		pr_emerg("anon ");

mm/debug.c void __dump_page(struct page *page, const char *reason) · line 85

Quoted 10 lines

mm/debug.c +10 -0

+	else if (PageKsm(page))
+		pr_emerg("ksm ");
+	else if (mapping) {
+		pr_emerg("%ps ", mapping->a_ops);
+		if (mapping->host->i_dentry.first) {
+			struct dentry *dentry;
+			dentry = container_of(mapping->host->i_dentry.first, struct dentry, d_u.d_alias);
+			pr_emerg("name:\"%*s\" ", dentry->d_name.len, dentry->d_name.name);
+		}
+	}

There has to be a better way of printing the filename. It is so often needed.

The (poorly named and gleefully undocumented) take_dentry_name_snapshot() looks promising. However it's unclear that __dump_page() is always called from contexts where take_dentry_name_snapshot() and release_dentry_name_snapshot() can be safely called. Probably it's OK, but how to guarantee it?

From: Michal Hocko <[email protected]>

__dump_page messages use KERN_EMERG resp. KERN_ALERT loglevel (this is the case since 2004). Most callers of this function are really detecting a critical page state and BUG right after. On the other hand the function is called also from contexts which just want to inform about the page state and those would rather not disrupt logs that much (e.g. some systems route these messages to the normal console).

Reduce the loglevel to KERN_WARNING to make dump_page easier to reuse for other contexts while those messages will still make it to the kernel log in most setups. Even if the loglevel setup filters warnings away those paths that are really critical already print the more targeted error or panic and that should make it to the kernel log.

Signed-off-by Michal Hocko <[email protected]>

---
 mm/debug.c | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

mm/debug.c +1 -1

index a33177bfc856..d18c5cea3320 100644
@@ -54,7 +54,7 @@ void __dump_page(struct page *page, const char *reason)
 	 * dump_page() when detected.
 	 */
 	if (page_poisoned) {
-		pr_emerg("page:%px is uninitialized and poisoned", page);
+		pr_warn("page:%px is uninitialized and poisoned", page);
 		goto hex_only;
 	}
 

mm/debug.c +7 -7

 		  page, page_ref_count(page), mapcount,
 		  page->mapping, page_to_pgoff(page));
 	if (PageCompound(page))
 		pr_cont(" compound_mapcount: %d", compound_mapcount(page));
 	pr_cont("\n");
 	if (PageAnon(page))
-		pr_emerg("anon ");
+		pr_warn("anon ");
 	else if (PageKsm(page))
-		pr_emerg("ksm ");
+		pr_warn("ksm ");
 	else if (mapping) {
-		pr_emerg("%ps ", mapping->a_ops);
+		pr_warn("%ps ", mapping->a_ops);
 		if (mapping->host->i_dentry.first) {
 			struct dentry *dentry;
 			dentry = container_of(mapping->host->i_dentry.first, struct dentry, d_u.d_alias);
-			pr_emerg("name:\"%*s\" ", dentry->d_name.len, dentry->d_name.name);
+			pr_warn("name:\"%*s\" ", dentry->d_name.len, dentry->d_name.name);
 		}
 	}
 	BUILD_BUG_ON(ARRAY_SIZE(pageflag_names) != __NR_PAGEFLAGS + 1);
 
-	pr_emerg("flags: %#lx(%pGp)\n", page->flags, &page->flags);
+	pr_warn("flags: %#lx(%pGp)\n", page->flags, &page->flags);
 
 hex_only:
 	print_hex_dump(KERN_ALERT, "raw: ", DUMP_PREFIX_NONE, 32,

@@ -93,11 +93,11 @@ void __dump_page(struct page *page, const char *reason)
 			sizeof(struct page), false);
 
 	if (reason)
-		pr_alert("page dumped because: %s\n", reason);
+		pr_warn("page dumped because: %s\n", reason);
 
 #ifdef CONFIG_MEMCG
 	if (!page_poisoned && page->mem_cgroup)
-		pr_alert("page->mem_cgroup:%px\n", page->mem_cgroup);
+		pr_warn("page->mem_cgroup:%px\n", page->mem_cgroup);
 #endif
 }
 

Signature · 1 line

2.19.1

It seems one follow up fix got lost on my side. Andrew, could you fold this in please?

mm/debug.c +0 -0

index 68e9a9f2df16..4c916e1abedc 100644

From: Michal Hocko <[email protected]>

The memory offlining failure reporting is inconsistent and insufficient. Some error paths simply do not report the failure to the log at all. When we do report there are no details about the reason of the failure and there are several of them which makes memory offlining failures hard to debug.

Make sure that the

	memory offlining [mem %#010llx-%#010llx] failed

message is printed for all failures and also provide a short textual reason for the failure e.g.

[ 1984.506184] rac1 kernel: memory offlining [mem 0x82600000000-0x8267fffffff] failed due to signal backoff

this tells us that the offlining has failed because of a signal pending aka user intervention.

...

Fix:

mm/memory_hotplug.c renamed from mm/memory_hotplug.c~mm-memory_hotplug-print-reason-for-the-offlining-failure-fix +3 -3

@@ -1576,7 +1576,7 @@ static int __ref __offline_pages(unsigne
 				       MIGRATE_MOVABLE, true);
 	if (ret) {
 		mem_hotplug_done();
-		reason = "failed to isolate range";
+		reason = "failure to isolate range";
 		goto failed_removal
 	}
 

@@ -1587,7 +1587,7 @@ static int __ref __offline_pages(unsigne
 	ret = memory_notify(MEM_GOING_OFFLINE, &arg);
 	ret = notifier_to_errno(ret);
 	if (ret) {
-		reason = "notifiers failure";
+		reason = "notifier failure";
 		goto failed_removal_isolated;
 	}
 

@@ -1616,7 +1616,7 @@ repeat:
 	 */
 	ret = dissolve_free_huge_pages(start_pfn, end_pfn);
 	if (ret) {
-		reason = "fails to disolve hugetlb pages";
+		reason = "failure to dissolve huge pages";
 		goto failed_removal_isolated;
 	}
 	/* check again */

_

Quoted 13 lines · On Wed, 7 Nov 2018 11:18:29 +0100 Michal Hocko <[email protected]> wrote:

From: Michal Hocko <[email protected]>

The memory offlining failure reporting is inconsistent and insufficient. Some error paths simply do not report the failure to the log at all. When we do report there are no details about the reason of the failure and there are several of them which makes memory offlining failures hard to debug.

Make sure that the

	memory offlining [mem %#010llx-%#010llx] failed

message is printed for all failures and also provide a short textual reason for the failure e.g.

Quoted 6 lines

[ 1984.506184] rac1 kernel: memory offlining [mem 0x82600000000-0x8267fffffff] failed due to signal backoff

this tells us that the offlining has failed because of a signal pending aka user intervention.

...

Some of these messages will come out looking a bit odd.

Review static int __ref __offline_pages(unsigned long start_pfn, · line 1579

Quoted 6 lines

(quoted diff) +1 -1

@@ -1573,7 +1576,8 @@ static int __ref __offline_pages(unsigned long start_pfn,
 				       MIGRATE_MOVABLE, true);
 	if (ret) {
 		mem_hotplug_done();
-		return ret;
+		reason = "failed to isolate range";

"memory offlining [mem ...] failed due to failed to isolate range"

Review static int __ref __offline_pages(unsigned long start_pfn, · line 1590

Quoted 12 lines

(quoted diff) +3 -2

+		goto failed_removal
 	}
 
 	arg.start_pfn = start_pfn;

@@ -1582,15 +1586,19 @@ static int __ref __offline_pages(unsigned long start_pfn,
 
 	ret = memory_notify(MEM_GOING_OFFLINE, &arg);
 	ret = notifier_to_errno(ret);
-	if (ret)
-		goto failed_removal;
+	if (ret) {
+		reason = "notifiers failure";

"memory offlining [mem ...] failed due to notifiers failure"

Review static int __ref __offline_pages(unsigned long start_pfn, · line 1619

Quoted 8 lines

(quoted diff) +2 -2

@@ -1607,8 +1615,10 @@ static int __ref __offline_pages(unsigned long start_pfn,
 	 * actually in order to make hugetlbfs's object counting consistent.
 	 */
 	ret = dissolve_free_huge_pages(start_pfn, end_pfn);
-	if (ret)
-		goto failed_removal;
+	if (ret) {
+		reason = "fails to disolve hugetlb pages";

"memory offlining [mem ...] failed due to fails to disolve hugetlb pages"

Fix:

From: Michal Hocko <[email protected]>

The memory offlining failure reporting is inconsistent and insufficient. Some error paths simply do not report the failure to the log at all. When we do report there are no details about the reason of the failure and there are several of them which makes memory offlining failures hard to debug.

Make sure that the

	memory offlining [mem %#010llx-%#010llx] failed

message is printed for all failures and also provide a short textual reason for the failure e.g.

[ 1984.506184] rac1 kernel: memory offlining [mem 0x82600000000-0x8267fffffff] failed due to signal backoff

this tells us that the offlining has failed because of a signal pending aka user intervention.

Signed-off-by Michal Hocko <[email protected]>

Quoted 13 lines · Michal Hocko

From: Michal Hocko <[email protected]>

The memory offlining failure reporting is inconsistent and insufficient. Some error paths simply do not report the failure to the log at all. When we do report there are no details about the reason of the failure and there are several of them which makes memory offlining failures hard to debug.

Make sure that the

	memory offlining [mem %#010llx-%#010llx] failed

message is printed for all failures and also provide a short textual reason for the failure e.g.

Quoted 6 lines

[ 1984.506184] rac1 kernel: memory offlining [mem 0x82600000000-0x8267fffffff] failed due to signal backoff

this tells us that the offlining has failed because of a signal pending aka user intervention.

Signed-off-by Michal Hocko <[email protected]>

It might help to enumerate these failure reason strings and use macros.

From: Michal Hocko <[email protected]>

There is only very limited information printed when the memory offlining fails:

[ 1984.506184] rac1 kernel: memory offlining [mem 0x82600000000-0x8267fffffff] failed due to signal backoff

This tells us that the failure is triggered by the userspace intervention but it doesn't tell us much more about the underlying reason. It might be that the page migration failes repeatedly and the userspace timeout expires and send a signal or it might be some of the earlier steps (isolation, memory notifier) takes too long.

If the migration failes then it would be really helpful to see which page that and its state. The same applies to the isolation phase. If we fail to isolate a page from the allocator then knowing the state of the page would be helpful as well.

Dump the page state that fails to get isolated or migrated. This will tell us more about the failure and what to focus on during debugging.

Signed-off-by Michal Hocko <[email protected]>

---
 mm/memory_hotplug.c | 12 ++++++++----
 mm/page_alloc.c     |  1 +
 2 files changed, 9 insertions(+), 4 deletions(-)

mm/memory_hotplug.c +1 -3

index 1badac89c58e..bf214beccda3 100644
@@ -1388,10 +1388,8 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
 						    page_is_file_cache(page));
 
 		} else {
-#ifdef CONFIG_DEBUG_VM
-			pr_alert("failed to isolate pfn %lx\n", pfn);
+			pr_warn("failed to isolate pfn %lx\n", pfn)>  			dump_page(page, "isolation failed");
-#endif
 			put_page(page);
 			/* Because we don't have big zone->lock. we should
 			   check this again here. */

mm/memory_hotplug.c +1 -0

 			putback_movable_pages(&source);
+		}
 	}
 out:
 	return ret;

mm/page_alloc.c +0 -0

index a919ba5cb3c8..23267767bf98 100644

On 11/07/2018 03:48 PM, Michal Hocko wrote:

Review do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) · line 1415

Quoted 9 lines

(quoted diff) +4 -1

@@ -1411,8 +1409,14 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
 		/* Allocate a new page from the nearest neighbor node */
 		ret = migrate_pages(&source, new_node_page, NULL, 0,
 					MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
-		if (ret)
+		if (ret) {
+			list_for_each_entry(page, &source, lru) {
+				pr_warn("migrating pfn %lx failed ",
+				       page_to_pfn(page), ret);

Seems like pr_warn() needs to have %d in here to print 'ret'.

Though
dumping return code from migrate_pages() makes sense, wondering if it is required for each and every page which failed to migrate here or just one instance is enough.

Review do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) · line 1417

Quoted 2 lines

(quoted diff) +2 -0

+				dump_page(page, NULL);
+			}

s/NULL/failed to migrate/ for dump_page().

Quoted 11 lines

(quoted diff) +1 -0

 			putback_movable_pages(&source);
+		}
 	}
 out:
 	return ret;

mm/page_alloc.c +0 -0

index a919ba5cb3c8..23267767bf98 100644

mm/page_alloc.c bool has_unmovable_pages(struct zone *zone, struct page *page, int count, · line 7848

Quoted 4 lines

mm/page_alloc.c +1 -0

@@ -7845,6 +7845,7 @@ bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
 	return false;
 unmovable:
 	WARN_ON_ONCE(zone_idx(zone) == ZONE_MOVABLE);
+	dump_page(pfn_to_page(pfn+iter), "has_unmovable_pages");

s/has_unmovable_pages/is unmovable/

If we eally care about the function name, then dump_page() should be followed by dump_stack() like the case in some other instances.

mm/page_alloc.c bool has_unmovable_pages(struct zone *zone, struct page *page, int count, · line 7849

Quoted 1 line

mm/page_alloc.c +0 -0

 	return true;

This will be dumped from HugeTLB and CMA allocation paths as well through alloc_contig_range(). But it should be okay as those occurrences should be rare and dumping page state then will also help.

Quoted 3 lines · Anshuman Khandual

On 11/07/2018 03:48 PM, Michal Hocko wrote:

[...]

Quoted 11 lines

Review do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) · line 1415

Quoted 9 lines

(quoted diff) +4 -1

@@ -1411,8 +1409,14 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
 		/* Allocate a new page from the nearest neighbor node */
 		ret = migrate_pages(&source, new_node_page, NULL, 0,
 					MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
-		if (ret)
+		if (ret) {
+			list_for_each_entry(page, &source, lru) {
+				pr_warn("migrating pfn %lx failed ",
+				       page_to_pfn(page), ret);

Seems like pr_warn() needs to have %d in here to print 'ret'.

Dohh. Rebase hickup. You are right ret:%d got lost on the way.

Quoted 4 lines

Though
dumping return code from migrate_pages() makes sense, wondering if it is required for each and every page which failed to migrate here or just one instance is enough.

Does it matter enough to special case one printk?

Quoted 5 lines

Review do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) · line 1417

Quoted 3 lines

(quoted diff) +2 -0

+				dump_page(page, NULL);
+			}

s/NULL/failed to migrate/ for dump_page().

Yes, makes sense.

Quoted 17 lines

Quoted 11 lines

(quoted diff) +1 -0

 			putback_movable_pages(&source);
+		}
 	}
 out:
 	return ret;

mm/page_alloc.c +0 -0

index a919ba5cb3c8..23267767bf98 100644

mm/page_alloc.c bool has_unmovable_pages(struct zone *zone, struct page *page, int count, · line 7848

Quoted 4 lines

mm/page_alloc.c +1 -0

@@ -7845,6 +7845,7 @@ bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
 	return false;
 unmovable:
 	WARN_ON_ONCE(zone_idx(zone) == ZONE_MOVABLE);
+	dump_page(pfn_to_page(pfn+iter), "has_unmovable_pages");

s/has_unmovable_pages/is unmovable/

OK

Quoted 8 lines

If we eally care about the function name, then dump_page() should be followed by dump_stack() like the case in some other instances.

mm/page_alloc.c bool has_unmovable_pages(struct zone *zone, struct page *page, int count, · line 7849

Quoted 1 line

mm/page_alloc.c +0 -0

 	return true;

This will be dumped from HugeTLB and CMA allocation paths as well through alloc_contig_range(). But it should be okay as those occurrences should be rare and dumping page state then will also help.

yes

Thanks and here is the incremental fix:

mm/memory_hotplug.c +2 -2

index bf214beccda3..820397e18e59 100644
@@ -1411,9 +1411,9 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
 					MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
 		if (ret) {
 			list_for_each_entry(page, &source, lru) {
-				pr_warn("migrating pfn %lx failed ",
+				pr_warn("migrating pfn %lx failed ret:%d ",
 				       page_to_pfn(page), ret);
-				dump_page(page, NULL);
+				dump_page(page, "migration failure");
 			}
 			putback_movable_pages(&source);
 		}

mm/page_alloc.c +0 -0

index 23267767bf98..ec2c7916dc2d 100644

mm/memory_hotplug.c +2 -2

index bf214beccda3..820397e18e59 100644
@@ -1411,9 +1411,9 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
 					MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
 		if (ret) {
 			list_for_each_entry(page, &source, lru) {
-				pr_warn("migrating pfn %lx failed ",
+				pr_warn("migrating pfn %lx failed ret:%d ",
 				       page_to_pfn(page), ret);
-				dump_page(page, NULL);
+				dump_page(page, "migration failure");
 			}
 			putback_movable_pages(&source);
 		}

mm/page_alloc.c +1 -1

index 23267767bf98..ec2c7916dc2d 100644
@@ -7845,7 +7845,7 @@ bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
 	return false;
 unmovable:
 	WARN_ON_ONCE(zone_idx(zone) == ZONE_MOVABLE);
-	dump_page(pfn_to_page(pfn+iter), "has_unmovable_pages");
+	dump_page(pfn_to_page(pfn+iter), "unmovable page");
 	return true;
 }
 

Signature · 2 lines

Michal Hocko
SUSE Labs

From: Michal Hocko <[email protected]>

There is only very limited information printed when the memory offlining fails:

[ 1984.506184] rac1 kernel: memory offlining [mem 0x82600000000-0x8267fffffff] failed due to signal backoff

This tells us that the failure is triggered by the userspace intervention but it doesn't tell us much more about the underlying reason. It might be that the page migration failes repeatedly and the userspace timeout expires and send a signal or it might be some of the earlier steps (isolation, memory notifier) takes too long.

If the migration failes then it would be really helpful to see which page that and its state. The same applies to the isolation phase. If we fail to isolate a page from the allocator then knowing the state of the page would be helpful as well.

Dump the page state that fails to get isolated or migrated. This will tell us more about the failure and what to focus on during debugging.

...

mm/memory_hotplug.c +1 -3

@@ -1388,10 +1388,8 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
 						    page_is_file_cache(page));
 
 		} else {
-#ifdef CONFIG_DEBUG_VM
-			pr_alert("failed to isolate pfn %lx\n", pfn);
+			pr_warn("failed to isolate pfn %lx\n", pfn);
 			dump_page(page, "isolation failed");
-#endif
 			put_page(page);
 			/* Because we don't have big zone->lock. we should
 			   check this again here. */

Quoted 37 lines · On Wed, 7 Nov 2018 11:18:30 +0100 Michal Hocko <[email protected]> wrote:

From: Michal Hocko <[email protected]>

There is only very limited information printed when the memory offlining fails:

[ 1984.506184] rac1 kernel: memory offlining [mem 0x82600000000-0x8267fffffff] failed due to signal backoff

This tells us that the failure is triggered by the userspace intervention but it doesn't tell us much more about the underlying reason. It might be that the page migration failes repeatedly and the userspace timeout expires and send a signal or it might be some of the earlier steps (isolation, memory notifier) takes too long.

If the migration failes then it would be really helpful to see which page that and its state. The same applies to the isolation phase. If we fail to isolate a page from the allocator then knowing the state of the page would be helpful as well.

Dump the page state that fails to get isolated or migrated. This will tell us more about the failure and what to focus on during debugging.

...

mm/memory_hotplug.c +1 -3

@@ -1388,10 +1388,8 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
 						    page_is_file_cache(page));
 
 		} else {
-#ifdef CONFIG_DEBUG_VM
-			pr_alert("failed to isolate pfn %lx\n", pfn);
+			pr_warn("failed to isolate pfn %lx\n", pfn);
 			dump_page(page, "isolation failed");
-#endif
 			put_page(page);
 			/* Because we don't have big zone->lock. we should
 			   check this again here. */

mm/memory_hotplug.c do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) · line 1417

Quoted 10 lines

mm/memory_hotplug.c +6 -1

@@ -1411,8 +1409,14 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
 		/* Allocate a new page from the nearest neighbor node */
 		ret = migrate_pages(&source, new_node_page, NULL, 0,
 					MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
-		if (ret)
+		if (ret) {
+			list_for_each_entry(page, &source, lru) {
+				pr_warn("migrating pfn %lx failed ",
+				       page_to_pfn(page), ret);
+				dump_page(page, NULL);
+			}

./include/linux/kern_levels.h:5:18: warning: too many arguments for format [-Wformat-extra-args]

 #define KERN_SOH "\001"  /* ASCII Start Of Header */
                  ^
./include/linux/kern_levels.h:12:22: note: in expansion of macro a??KERN_SOHa??
 #define KERN_WARNING KERN_SOH "4" /* warning conditions */
                      ^~~~~~~~
./include/linux/printk.h:310:9: note: in expansion of macro a??KERN_WARNINGa??
  printk(KERN_WARNING pr_fmt(fmt), ##__VA_ARGS__)
         ^~~~~~~~~~~~
./include/linux/printk.h:311:17: note: in expansion of macro a??pr_warninga??
 #define pr_warn pr_warning
                 ^~~~~~~~~~

mm/memory_hotplug.c:1414:5: note: in expansion of macro a??pr_warna??
pr_warn("migrating pfn %lx failed ",
^~~~~~~

mm/memory_hotplug.c renamed from mm/memory_hotplug.c~mm-memory_hotplug-be-more-verbose-for-memory-offline-failures-fix +1 -1

@@ -1411,7 +1411,7 @@ do_migrate_range(unsigned long start_pfn
 					MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
 		if (ret) {
 			list_for_each_entry(page, &source, lru) {
-				pr_warn("migrating pfn %lx failed ",
+				pr_warn("migrating pfn %lx failed: %d",
 				       page_to_pfn(page), ret);
 				dump_page(page, NULL);
 			}