Skip to contentSkip to conversation

Fix crashkernel reservation on NUMA machines

PATCH Applied Bernhard Walle 7 participants 23 messages Jun 8, 2008 – Jun 10, 2008

Subsystems (5)
lore
Share & tools
lore
#0 lore raw

cover Fix crashkernel reservation on NUMA machines

This patch series fixes the crashkernel reservation on NUMA machine. The regression was discovered by Dave Anderson <[email protected]>.

The background is that on NUMA machines, reserve_bootmem_generic() is required instead of reserve_bootmem(). To achieve that, it's necessary to make a few API changes.

The patches are against latest linux-2.6 git. They should still go into 2.6.26 since it's only bug fixing. For 2.6.27, we should unify crashkernel reservation for i386 and x86-64.

Tested on both i386 and x86-64. Compilation was tested with both kexec disabled and enabled. The change is x86 only, so no need to test on other architectures.

Signature · 6 lines
Bernhard Walle, SUSE LINUX Products GmbH, Architecture Maintenance

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
#1 lore raw

PATCH 1/3 Add return value to reserve_bootmem_node()

This patch changes the function reserve_bootmem_node() from void to int, returning -ENOMEM if the allocation fails.

---
 include/linux/bootmem.h |    2 +-
 mm/bootmem.c            |    6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)
include/linux/bootmem.h +1 -1
@@ -94,7 +94,7 @@ extern unsigned long init_bootmem_node(p
 				       unsigned long freepfn,
 				       unsigned long startpfn,
 				       unsigned long endpfn);
-extern void reserve_bootmem_node(pg_data_t *pgdat,
+extern int reserve_bootmem_node(pg_data_t *pgdat,
 				 unsigned long physaddr,
 				 unsigned long size,
 				 int flags);
mm/bootmem.c +4 -2
@@ -442,15 +442,17 @@ unsigned long __init init_bootmem_node(p
 	return init_bootmem_core(pgdat, freepfn, startpfn, endpfn);
 }
 
-void __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
+int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
 				 unsigned long size, int flags)
 {
 	int ret;
 
 	ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
 	if (ret < 0)
-		return;
+		return -ENOMEM;
 	reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
+
+	return 0;
 }
 
 void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
#2 lore raw

PATCH 2/3 Add flags parameter to reserve_bootmem_generic()

In reply to Bernhard Walle

This patch adds a 'flags' parameter to reserve_bootmem_generic() like it already has been added in reserve_bootmem() with commit 72a7fe3967dbf86cb34e24fbf1d957fe24d2f246.

It also changes all users to use BOOTMEM_DEFAULT, which doesn't effectively change the behaviour. Since the change is x86-specific, I don't think it's necessary to add a new API for migration. There are only 4 users of that function.

The change is necessary for the next patch, using reserve_bootmem_generic() for crashkernel reservation.

---
 arch/x86/kernel/e820_64.c |    3 ++-
 arch/x86/kernel/efi_64.c  |    3 ++-
 arch/x86/kernel/mpparse.c |    5 +++--
 arch/x86/mm/init_64.c     |   17 ++++++++++++-----
 include/asm-x86/proto.h   |    2 +-
 5 files changed, 20 insertions(+), 10 deletions(-)
arch/x86/kernel/e820_64.c +2 -1
@@ -118,7 +118,8 @@ void __init early_res_to_bootmem(unsigne
 			continue;
 		printk(KERN_INFO "  early res: %d [%lx-%lx] %s\n", i,
 			final_start, final_end - 1, r->name);
-		reserve_bootmem_generic(final_start, final_end - final_start);
+		reserve_bootmem_generic(final_start, final_end - final_start,
+			BOOTMEM_DEFAULT);
 	}
 }
 
arch/x86/kernel/efi_64.c +2 -1
@@ -100,7 +100,8 @@ void __init efi_call_phys_epilog(void)
 void __init efi_reserve_bootmem(void)
 {
 	reserve_bootmem_generic((unsigned long)memmap.phys_map,
-				memmap.nr_map * memmap.desc_size);
+				memmap.nr_map * memmap.desc_size,
+				BOOTMEM_DEFAULT);
 }
 
 void __iomem * __init efi_ioremap(unsigned long phys_addr, unsigned long size)
arch/x86/kernel/mpparse.c +3 -2
@@ -729,10 +729,11 @@ static int __init smp_scan_config(unsign
 			if (!reserve)
 				return 1;
 
-			reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE);
+			reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE,
+				BOOTMEM_DEFAULT);
 			if (mpf->mpf_physptr)
 				reserve_bootmem_generic(mpf->mpf_physptr,
-							PAGE_SIZE);
+					PAGE_SIZE, BOOTMEM_DEFAULT);
 #endif
 		return 1;
 		}

Files a/arch/x86/kernel/setup_64.o and b/arch/x86/kernel/setup_64.o differ

arch/x86/mm/init_64.c +12 -5
@@ -798,12 +798,13 @@ void free_initrd_mem(unsigned long start
 }
 #endif
 
-void __init reserve_bootmem_generic(unsigned long phys, unsigned len)
+int __init reserve_bootmem_generic(unsigned long phys, unsigned len, int flags)
 {
 #ifdef CONFIG_NUMA
 	int nid, next_nid;
 #endif
 	unsigned long pfn = phys >> PAGE_SHIFT;
+	int ret;
 
 	if (pfn >= end_pfn) {
 		/*
@@ -811,11 +812,11 @@ void __init reserve_bootmem_generic(unsi
 		 * firmware tables:
 		 */
 		if (pfn < max_pfn_mapped)
-			return;
+			return -EFAULT;
 
 		printk(KERN_ERR "reserve_bootmem: illegal reserve %lx %u\n",
 				phys, len);
-		return;
+		return -EFAULT;
 	}
 
 	/* Should check here against the e820 map to avoid double free */
@@ -823,9 +824,13 @@ void __init reserve_bootmem_generic(unsi
 	nid = phys_to_nid(phys);
 	next_nid = phys_to_nid(phys + len - 1);
 	if (nid == next_nid)
-		reserve_bootmem_node(NODE_DATA(nid), phys, len, BOOTMEM_DEFAULT);
+		ret = reserve_bootmem_node(NODE_DATA(nid), phys, len, flags);
 	else
-		reserve_bootmem(phys, len, BOOTMEM_DEFAULT);
+		ret = reserve_bootmem(phys, len, flags);
+
+	if (ret != 0)
+		return ret;
+
 #else
 	reserve_bootmem(phys, len, BOOTMEM_DEFAULT);
 #endif
@@ -834,6 +839,8 @@ void __init reserve_bootmem_generic(unsi
 		dma_reserve += len / PAGE_SIZE;
 		set_dma_reserve(dma_reserve);
 	}
+
+	return 0;
 }
 
 int kern_addr_valid(unsigned long addr)

Files a/arch/x86/mm/init_64.o and b/arch/x86/mm/init_64.o differ

include/asm-x86/proto.h +1 -1
@@ -14,7 +14,7 @@ extern void ia32_syscall(void);
 extern void ia32_cstar_target(void);
 extern void ia32_sysenter_target(void);
 
-extern void reserve_bootmem_generic(unsigned long phys, unsigned len);
+extern int reserve_bootmem_generic(unsigned long phys, unsigned len, int flags);
 
 extern void syscall32_cpu_init(void);
 

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#3 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

Quoted 38 lines · Bernhard Walle

This patch adds a 'flags' parameter to reserve_bootmem_generic() like it already has been added in reserve_bootmem() with commit 72a7fe3967dbf86cb34e24fbf1d957fe24d2f246.

It also changes all users to use BOOTMEM_DEFAULT, which doesn't effectively change the behaviour. Since the change is x86-specific, I don't think it's necessary to add a new API for migration. There are only 4 users of that function.

The change is necessary for the next patch, using reserve_bootmem_generic() for crashkernel reservation.

---

arch/x86/kernel/e820_64.c |    3 ++-
arch/x86/kernel/efi_64.c  |    3 ++-
arch/x86/kernel/mpparse.c |    5 +++--
arch/x86/mm/init_64.c     |   17 ++++++++++++-----
include/asm-x86/proto.h   |    2 +-
5 files changed, 20 insertions(+), 10 deletions(-)
arch/x86/kernel/e820_64.c +2 -1
@@ -118,7 +118,8 @@ void __init early_res_to_bootmem(unsigne
 			continue;
 		printk(KERN_INFO "  early res: %d [%lx-%lx] %s\n", i,
 			final_start, final_end - 1, r->name);
-		reserve_bootmem_generic(final_start, final_end - final_start);
+		reserve_bootmem_generic(final_start, final_end - final_start,
+			BOOTMEM_DEFAULT);
 	}
 }
 

arch/x86/kernel/efi_64.c void __init efi_call_phys_epilog(void) · line 105

Quoted 7 lines
arch/x86/kernel/efi_64.c +2 -1
@@ -100,7 +100,8 @@ void __init efi_call_phys_epilog(void)
 void __init efi_reserve_bootmem(void)
 {
 	reserve_bootmem_generic((unsigned long)memmap.phys_map,
-				memmap.nr_map * memmap.desc_size);
+				memmap.nr_map * memmap.desc_size,
+				BOOTMEM_DEFAULT);
 }

Just one comment.

Since 'reserve_bootmem_generic' is changed from 'void' to 'int', we should check its return value for failure when possible, right?

Signature · 7 lines
Hi, I'm a .signature virus, please copy/paste me to help me spread
all over the world.

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
#4 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

Quoted 3 lines · WANG Cong

Since 'reserve_bootmem_generic' is changed from 'void' to 'int', we should check its return value for failure when possible, right?

That may make sense here, but that's unrelated to my change.

Just because the error *can* be caught by checking the return value doesn't mean that it *must* be caught always. It was silently ignored before in the efi_reserve_bootmem() function before, and so is it now. No behaviour change.

Bernhard

Signature · 6 lines
Bernhard Walle, SUSE LINUX Products GmbH, Architecture Maintenance

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
#5 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

In reply to Bernhard Walle

Hi,

Quoted 18 lines · Bernhard Walle
arch/x86/mm/init_64.c +2 -1
@@ -798,12 +798,13 @@ void free_initrd_mem(unsigned long start
 }
 #endif
 
-void __init reserve_bootmem_generic(unsigned long phys, unsigned len)
+int __init reserve_bootmem_generic(unsigned long phys, unsigned len, int flags)
 {
 #ifdef CONFIG_NUMA
 	int nid, next_nid;
 #endif
 	unsigned long pfn = phys >> PAGE_SHIFT;
+	int ret;
 
 	if (pfn >= end_pfn) {
 		/*

arch/x86/mm/init_64.c void __init reserve_bootmem_generic(unsi · line 815

Quoted 5 lines
arch/x86/mm/init_64.c +1 -1
@@ -811,11 +812,11 @@ void __init reserve_bootmem_generic(unsi
 		 * firmware tables:
 		 */
 		if (pfn < max_pfn_mapped)
-			return;
+			return -EFAULT;

This seemed to be `just do nothing' behaviour. Wouldn't 0 be more correct here? Or something else so there is a difference between the path that does not print a warning (the one below) and the path that does?

arch/x86/mm/init_64.c void __init reserve_bootmem_generic(unsi · line 820

Quoted 6 lines
arch/x86/mm/init_64.c +1 -1
 		printk(KERN_ERR "reserve_bootmem: illegal reserve %lx %u\n",
 				phys, len);
-		return;
+		return -EFAULT;
 	}

Hannes

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#6 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

Quoted 19 lines · Johannes Weiner

Hi,

Quoted 15 lines · Bernhard Walle
arch/x86/mm/init_64.c +2 -1
@@ -798,12 +798,13 @@ void free_initrd_mem(unsigned long start
 }
 #endif
 
-void __init reserve_bootmem_generic(unsigned long phys, unsigned len)
+int __init reserve_bootmem_generic(unsigned long phys, unsigned len, int flags)
 {
 #ifdef CONFIG_NUMA
 	int nid, next_nid;
 #endif
 	unsigned long pfn = phys >> PAGE_SHIFT;
+	int ret;
 
Quoted 13 lines

arch/x86/mm/init_64.c void __init reserve_bootmem_generic(unsi · line 815

Quoted 8 lines
arch/x86/mm/init_64.c +1 -1
 	if (pfn >= end_pfn) {
 		/*
@@ -811,11 +812,11 @@ void __init reserve_bootmem_generic(unsi
 		 * firmware tables:
 		 */
 		if (pfn < max_pfn_mapped)
-			return;
+			return -EFAULT;

This seemed to be `just do nothing' behaviour. Wouldn't 0 be more correct here? Or something else so there is a difference between the path that does not print a warning (the one below) and the path that does?

Bernard,

This is interesting. IIUC, end_pfn represents end of physical RAM and max_pfn_mapped represents, end of other tables like ACPI which are mapped in higher regions.

Kdump first kernel always tries to reserve just physical RAM and nothing else. So I am not sure what does above code do. Try to reserve a memory which is not RAM but is in the region less than highest mapped entity and in that case return silently without any warning. In what case do we exercise this path?

Thanks
Vivek

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#7 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

Quoted 6 lines · Vivek Goyal

Kdump first kernel always tries to reserve just physical RAM and nothing else. So I am not sure what does above code do. Try to reserve a memory which is not RAM but is in the region less than highest mapped entity and in that case return silently without any warning. In what case do we exercise this path?

I don't know. That code has been introduced in
http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git;a=commitdiff;h=5e58a02a8f6a7a1c9ae41f39286bcd3aea0d6f24

Ccing Andi.

IMO we should not print any warning in that function, leaving the error handling to the caller.

Bernhard

Signature · 6 lines
Bernhard Walle, SUSE LINUX Products GmbH, Architecture Maintenance

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
#8 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

Quoted 7 lines · Bernhard Walle
Quoted 5 lines · Vivek Goyal

Kdump first kernel always tries to reserve just physical RAM and nothing else. So I am not sure what does above code do. Try to reserve a memory which is not RAM but is in the region less than highest mapped entity and in that case return silently without any warning. In what case do we exercise this path?

Quoted 7 lines

I don't know. That code has been introduced in
http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git;a=commitdiff;h=5e58a02a8f6a7a1c9ae41f39286bcd3aea0d6f24

Ccing Andi.

IMO we should not print any warning in that function, leaving the error handling to the caller.

Don't remember the details. Perhaps Amul does (cc'ed)

-Andi

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#9 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

Quoted 11 lines · Andi Kleen
Quoted 10 lines · Bernhard Walle
Quoted 5 lines · Vivek Goyal

Kdump first kernel always tries to reserve just physical RAM and nothing else. So I am not sure what does above code do. Try to reserve a memory which is not RAM but is in the region less than highest mapped entity and in that case return silently without any warning. In what case do we exercise this path?

I don't know. That code has been introduced in
http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git;a=commitdiff;h=5e58a02a8f6a7a1c9ae41f39286bcd3aea0d6f24

Quoted 9 lines
Quoted 4 lines

Ccing Andi.

IMO we should not print any warning in that function, leaving the error handling to the caller.

Don't remember the details. Perhaps Amul does (cc'ed)

-Andi

The short story is that the kexec kernel was panicking when trying to reserve the MP tables. The panic occurs because the MP tables resided in a reserved memory area above the highest address (80MB phys at that time) in the user defined E820 map used by the kexec kernel.

I had placed my code to affect only MP table reservation (see patch below) because it is unique to just that code path. Andi decided a generalized approach would be better in case other vendors had similar issues.

Vivek asked if I was using a user defined memory map for the kexec kernel. I was using one, but the top of memory was being defined as 80MB physical (end_pfn). The "exactmap" option parsing is clobbering the variable end_pfn_map. I suggested using the saved_pfn_map variable.

In the end Andi's patch was the best, so it stuck.

I took a quick look at the current code base and it would still panic when reserving the MP table. The function smp_scan_config does the reservation. I did not track down how the BUG_ON in reserve_bootmem_core corresponds to end_pfn.

thanks,
Amul

Here is my original email (http://lkml.org/lkml/2006/11/2/285):

The kdump crash kernel panics when it tries to reserve the MP Config tables on an ES7000.

The MP Config table is located above 1MB of physical memory in a reserved memory area. It is located outside the first 1MB area because the tables are too large, 240k.

The crash kernel is given a user defined memory map with E820 reserved and ACPI areas passed in by kexec tools and a usable area from 16MB physical to 80MB physical. This user defined map causes the top of memory to be set as 80MB.

The ACPI tables and MP Tables reside higher in memory. When reserving memory with reserve_bootmem_generic, the function has a BUG panic if the memory location to reserve is above the top of memory. The MP table is above the top of memory in a user defined memory map.

This patch will ignore reserving the MP tables if the MP table resides in an area already reserved in the E820.

I have two alternate patches that accomplish the same effect if this patch is not acceptable.
1. avoid reserving the MP tables if a user defined memory map or if a user defined memory limit ("mem=") is used.
2. avoid reserving the MP tables if a kernel parameter is passed in to ignore MP table reservation.

arch/x86_64/kernel/e820.c 2006-11-02 17:56:01.000000000 -0500 renamed from arch/x86_64/kernel/e820.c 2006-10-31 17:38:41.000000000 -0500 +47 -0
diff -Naur linux-2.6.19-rc4/arch/x86_64/kernel/e820.c linux-2.6.19-rc4-az/arch/x86_64/kernel/e820.c
@@ -351,6 +351,53 @@
        }
 }
 
+int __init e820_reserved(unsigned long target_phys)
+{
+       int i;
+       unsigned long section_begin_phys, section_end_phys;
+
+       for (i = 0; i < e820.nr_map; i++) {
+               // if it is usable memory, ignore it
+               if (e820.map[i].type == E820_RAM )
+                       continue;
+
+               section_begin_phys = e820.map[i].addr;
+               section_end_phys = e820.map[i].addr + e820.map[i].size;
+
+               // if its NOT within the memory range, ignore it
+               if (!(section_begin_phys < target_phys &&
+                     target_phys < section_end_phys))
+                       continue;
+
+               printk(KERN_DEBUG "MP Tables at %lx in %016lx - %016lx",
+                      target_phys, section_begin_phys, section_end_phys);
+               
+               switch (e820.map[i].type) {
+               case E820_RESERVED:
+                       printk(KERN_DEBUG "(reserved)\n");
+                       break;
+               case E820_ACPI:
+                       printk(KERN_DEBUG "(ACPI data)\n");
+                       printk(KERN_DEBUG "WARNING: MP Tables located in");
+                       printk(KERN_DEBUG "ACPI Data Area\n");
+                       break;
+               case E820_NVS:
+                       printk(KERN_DEBUG "(ACPI NVS)\n");
+                       printk(KERN_DEBUG "WARNING: MP Tables located in");
+                       printk(KERN_DEBUG "ACPI NVS Area\n");
+                       break;
+               default:        
+                       printk(KERN_DEBUG "(type %u)\n", e820.map[i].type);
+                       printk(KERN_ERR "WARNING: MP Tables located in");
+                       printk(KERN_ERR "Unkown Memory Area!\n");
+                       printk(KERN_ERR "Reservations are disallowed.\n");
+                       return 0;
+               }
+               return 1;
+       }
+       return 0;
+}
+
 /*
  * Sanitize the BIOS e820 map.
  *
arch/x86_64/kernel/mpparse.c 2006-11-02 17:25:10.000000000 -0500 renamed from arch/x86_64/kernel/mpparse.c 2006-10-31 17:38:41.000000000 -0500 +2 -1
diff -Naur linux-2.6.19-rc4/arch/x86_64/kernel/mpparse.c linux-2.6.19-rc4-az/arch/x86_64/kernel/mpparse.c
@@ -23,6 +23,7 @@
 #include <linux/acpi.h>
 #include <linux/module.h>
 
+#include <asm/e820.h>
 #include <asm/smp.h>
 #include <asm/mtrr.h>
 #include <asm/mpspec.h>
@@ -543,7 +544,7 @@
 
                        smp_found_config = 1;
                        reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE);
-                       if (mpf->mpf_physptr)
+                       if (mpf->mpf_physptr && e820_reserved(mpf->mpf_physptr))
                                reserve_bootmem_generic(mpf->mpf_physptr, PAGE_SIZE);
                        mpf_found = mpf;
                        return 1;
include/asm-x86_64/e820.h 2006-11-02 17:25:10.000000000 -0500 renamed from include/asm-x86_64/e820.h 2006-10-31 17:39:24.000000000 -0500 +1 -0
diff -Naur linux-2.6.19-rc4/include/asm-x86_64/e820.h linux-2.6.19-rc4-az/include/asm-x86_64/e820.h
@@ -44,6 +44,7 @@
 extern void e820_reserve_resources(void);
 extern void e820_mark_nosave_regions(void);
 extern void e820_print_map(char *who);
+extern int e820_reserved(unsigned long target_phys);
 extern int e820_any_mapped(unsigned long start, unsigned long end, unsigned type);
 extern int e820_all_mapped(unsigned long start, unsigned long end, unsigned type);
 

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#10 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

Quoted 11 lines · Amul Shah
Quoted 5 lines

Don't remember the details. Perhaps Amul does (cc'ed)

-Andi

The short story is that the kexec kernel was panicking when trying to reserve the MP tables. The panic occurs because the MP tables resided in a reserved memory area above the highest address (80MB phys at that time) in the user defined E820 map used by the kexec kernel.

Quoted 4 lines

I had placed my code to affect only MP table reservation (see patch below) because it is unique to just that code path. Andi decided a generalized approach would be better in case other vendors had similar issues.

Ok, in that case it makes indeed sense to just return success here. Here's my third version of that patch:

-------------------------------------------------------------------------
From: Bernhard Walle <[email protected]>
Subject: Add 'flags' parameter to reserve_bootmem_generic()

This patch adds a 'flags' parameter to reserve_bootmem_generic() like it already has been added in reserve_bootmem() with commit 72a7fe3967dbf86cb34e24fbf1d957fe24d2f246.

It also changes all users to use BOOTMEM_DEFAULT, which doesn't effectively change the behaviour. Since the change is x86-specific, I don't think it's necessary to add a new API for migration. There are only 4 users of that function.

The change is necessary for the next patch, using reserve_bootmem_generic() for crashkernel reservation.

---
 arch/x86/kernel/e820_64.c |    3 ++-
 arch/x86/kernel/efi_64.c  |    3 ++-
 arch/x86/kernel/mpparse.c |    5 +++--
 arch/x86/mm/init_64.c     |   19 +++++++++++++------
 include/asm-x86/proto.h   |    2 +-
 5 files changed, 21 insertions(+), 11 deletions(-)
arch/x86/kernel/e820_64.c +2 -1
@@ -118,7 +118,8 @@ void __init early_res_to_bootmem(unsigne
 			continue;
 		printk(KERN_INFO "  early res: %d [%lx-%lx] %s\n", i,
 			final_start, final_end - 1, r->name);
-		reserve_bootmem_generic(final_start, final_end - final_start);
+		reserve_bootmem_generic(final_start, final_end - final_start,
+			BOOTMEM_DEFAULT);
 	}
 }
 
arch/x86/kernel/efi_64.c +2 -1
@@ -100,7 +100,8 @@ void __init efi_call_phys_epilog(void)
 void __init efi_reserve_bootmem(void)
 {
 	reserve_bootmem_generic((unsigned long)memmap.phys_map,
-				memmap.nr_map * memmap.desc_size);
+				memmap.nr_map * memmap.desc_size,
+				BOOTMEM_DEFAULT);
 }
 
 void __iomem * __init efi_ioremap(unsigned long phys_addr, unsigned long size)
arch/x86/kernel/mpparse.c +3 -2
@@ -729,10 +729,11 @@ static int __init smp_scan_config(unsign
 			if (!reserve)
 				return 1;
 
-			reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE);
+			reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE,
+				BOOTMEM_DEFAULT);
 			if (mpf->mpf_physptr)
 				reserve_bootmem_generic(mpf->mpf_physptr,
-							PAGE_SIZE);
+					PAGE_SIZE, BOOTMEM_DEFAULT);
 #endif
 		return 1;
 		}
arch/x86/mm/init_64.c +13 -6
@@ -798,12 +798,13 @@ void free_initrd_mem(unsigned long start
 }
 #endif
 
-void __init reserve_bootmem_generic(unsigned long phys, unsigned len)
+int __init reserve_bootmem_generic(unsigned long phys, unsigned len, int flags)
 {
 #ifdef CONFIG_NUMA
 	int nid, next_nid;
 #endif
 	unsigned long pfn = phys >> PAGE_SHIFT;
+	int ret;
 
 	if (pfn >= end_pfn) {
 		/*
@@ -811,11 +812,11 @@ void __init reserve_bootmem_generic(unsi
 		 * firmware tables:
 		 */
 		if (pfn < max_pfn_mapped)
-			return;
+			return 0;
 
 		printk(KERN_ERR "reserve_bootmem: illegal reserve %lx %u\n",
 				phys, len);
-		return;
+		return -EFAULT;
 	}
 
 	/* Should check here against the e820 map to avoid double free */
@@ -823,17 +824,23 @@ void __init reserve_bootmem_generic(unsi
 	nid = phys_to_nid(phys);
 	next_nid = phys_to_nid(phys + len - 1);
 	if (nid == next_nid)
-		reserve_bootmem_node(NODE_DATA(nid), phys, len, BOOTMEM_DEFAULT);
+		ret = reserve_bootmem_node(NODE_DATA(nid), phys, len, flags);
 	else
-		reserve_bootmem(phys, len, BOOTMEM_DEFAULT);
+		ret = reserve_bootmem(phys, len, flags);
+
 #else
-	reserve_bootmem(phys, len, BOOTMEM_DEFAULT);
+	ret = reserve_bootmem(phys, len, flags);
 #endif
 
+	if (ret != 0)
+		return ret;
+
 	if (phys+len <= MAX_DMA_PFN*PAGE_SIZE) {
 		dma_reserve += len / PAGE_SIZE;
 		set_dma_reserve(dma_reserve);
 	}
+
+	return 0;
 }
 
 int kern_addr_valid(unsigned long addr)
include/asm-x86/proto.h +1 -1
@@ -14,7 +14,7 @@ extern void ia32_syscall(void);
 extern void ia32_cstar_target(void);
 extern void ia32_sysenter_target(void);
 
-extern void reserve_bootmem_generic(unsigned long phys, unsigned len);
+extern int reserve_bootmem_generic(unsigned long phys, unsigned len, int flags);
 
 extern void syscall32_cpu_init(void);
 

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#11 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

Quoted 12 lines · Bernhard Walle
Quoted 11 lines · Amul Shah
Quoted 5 lines

Don't remember the details. Perhaps Amul does (cc'ed)

-Andi

The short story is that the kexec kernel was panicking when trying to reserve the MP tables. The panic occurs because the MP tables resided in a reserved memory area above the highest address (80MB phys at that time) in the user defined E820 map used by the kexec kernel.

Quoted 7 lines
Quoted 4 lines

I had placed my code to affect only MP table reservation (see patch below) because it is unique to just that code path. Andi decided a generalized approach would be better in case other vendors had similar issues.

Ok, in that case it makes indeed sense to just return success here. Here's my third version of that patch:

Hi Bernhard,

[..]

Quoted 12 lines
arch/x86/mm/init_64.c +2 -1
-void __init reserve_bootmem_generic(unsigned long phys, unsigned len)
+int __init reserve_bootmem_generic(unsigned long phys, unsigned len, int flags)
 {
 #ifdef CONFIG_NUMA
 	int nid, next_nid;
 #endif
 	unsigned long pfn = phys >> PAGE_SHIFT;
+	int ret;
 
 	if (pfn >= end_pfn) {
 		/*

arch/x86/mm/init_64.c void __init reserve_bootmem_generic(unsi · line 816

Quoted 6 lines
arch/x86/mm/init_64.c +1 -1
@@ -811,11 +812,11 @@ void __init reserve_bootmem_generic(unsi
 		 * firmware tables:
 		 */
 		if (pfn < max_pfn_mapped)
-			return;
+			return 0;
 

Can you please put some more explanation comment here to explain that why it is ok to return with success, despite the fact that we never reserved any memory.

One comment is already there, but it would be nice if we could expand that a bit to give more context. Like during normal boot there we need to make sure "MP tables" memory is reserved but once kdump kernel is booted, same "MP tables" memory might be beyond "end_pfn" and reserving code would not know about this special case. So it is ok to return with success. (Something similar).

Thanks
Vivek

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#12 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

Hi Vivek,

Quoted 4 lines · Vivek Goyal

Can you please put some more explanation comment here to explain that why it is ok to return with success, despite the fact that we never reserved any memory.

Do you think that's ok?

When booting the kdump kernel, the MP tables (for example, or other firmware-reserved memory) of the BIOS are beyond end_pfn. (kexec-tools adds exactmap parameters to the kernel so that the E820 map is no longer used.) Therefore, it's ok to return "success" here. For normal boot, the MP tables must be reserved normally.

Bernhard

------------------------------------------------------------------------------
From: Bernhard Walle <[email protected]>
Subject: Add 'flags' parameter to reserve_bootmem_generic()

This patch adds a 'flags' parameter to reserve_bootmem_generic() like it already has been added in reserve_bootmem() with commit 72a7fe3967dbf86cb34e24fbf1d957fe24d2f246.

It also changes all users to use BOOTMEM_DEFAULT, which doesn't effectively change the behaviour. Since the change is x86-specific, I don't think it's necessary to add a new API for migration. There are only 4 users of that function.

The change is necessary for the next patch, using reserve_bootmem_generic() for crashkernel reservation.

---
 arch/x86/kernel/e820_64.c |    3 ++-
 arch/x86/kernel/efi_64.c  |    3 ++-
 arch/x86/kernel/mpparse.c |    5 +++--
 arch/x86/mm/init_64.c     |   27 +++++++++++++++++++--------
 include/asm-x86/proto.h   |    2 +-
 5 files changed, 27 insertions(+), 13 deletions(-)
arch/x86/kernel/e820_64.c +2 -1
@@ -118,7 +118,8 @@ void __init early_res_to_bootmem(unsigne
 			continue;
 		printk(KERN_INFO "  early res: %d [%lx-%lx] %s\n", i,
 			final_start, final_end - 1, r->name);
-		reserve_bootmem_generic(final_start, final_end - final_start);
+		reserve_bootmem_generic(final_start, final_end - final_start,
+			BOOTMEM_DEFAULT);
 	}
 }
 
arch/x86/kernel/efi_64.c +2 -1
@@ -100,7 +100,8 @@ void __init efi_call_phys_epilog(void)
 void __init efi_reserve_bootmem(void)
 {
 	reserve_bootmem_generic((unsigned long)memmap.phys_map,
-				memmap.nr_map * memmap.desc_size);
+				memmap.nr_map * memmap.desc_size,
+				BOOTMEM_DEFAULT);
 }
 
 void __iomem * __init efi_ioremap(unsigned long phys_addr, unsigned long size)
arch/x86/kernel/mpparse.c +3 -2
@@ -729,10 +729,11 @@ static int __init smp_scan_config(unsign
 			if (!reserve)
 				return 1;
 
-			reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE);
+			reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE,
+				BOOTMEM_DEFAULT);
 			if (mpf->mpf_physptr)
 				reserve_bootmem_generic(mpf->mpf_physptr,
-							PAGE_SIZE);
+					PAGE_SIZE, BOOTMEM_DEFAULT);
 #endif
 		return 1;
 		}
arch/x86/mm/init_64.c +19 -8
@@ -798,24 +798,29 @@ void free_initrd_mem(unsigned long start
 }
 #endif
 
-void __init reserve_bootmem_generic(unsigned long phys, unsigned len)
+int __init reserve_bootmem_generic(unsigned long phys, unsigned len, int flags)
 {
 #ifdef CONFIG_NUMA
 	int nid, next_nid;
 #endif
 	unsigned long pfn = phys >> PAGE_SHIFT;
+	int ret;
 
 	if (pfn >= end_pfn) {
 		/*
-		 * This can happen with kdump kernels when accessing
-		 * firmware tables:
+		 * When booting the kdump kernel, the MP tables (for example,
+		 * or other firmware-reserved memory) of the BIOS are beyond
+		 * end_pfn. (kexec-tools adds exactmap parameters to the kernel
+		 * so that the E820 map is no longer used.) Therefore, it's ok
+		 * to return "success" here. For normal boot, the MP tables
+		 * must be reserved normally.
 		 */
 		if (pfn < max_pfn_mapped)
-			return;
+			return 0;
 
 		printk(KERN_ERR "reserve_bootmem: illegal reserve %lx %u\n",
 				phys, len);
-		return;
+		return -EFAULT;
 	}
 
 	/* Should check here against the e820 map to avoid double free */
@@ -823,17 +828,23 @@ void __init reserve_bootmem_generic(unsi
 	nid = phys_to_nid(phys);
 	next_nid = phys_to_nid(phys + len - 1);
 	if (nid == next_nid)
-		reserve_bootmem_node(NODE_DATA(nid), phys, len, BOOTMEM_DEFAULT);
+		ret = reserve_bootmem_node(NODE_DATA(nid), phys, len, flags);
 	else
-		reserve_bootmem(phys, len, BOOTMEM_DEFAULT);
+		ret = reserve_bootmem(phys, len, flags);
+
 #else
-	reserve_bootmem(phys, len, BOOTMEM_DEFAULT);
+	ret = reserve_bootmem(phys, len, flags);
 #endif
 
+	if (ret != 0)
+		return ret;
+
 	if (phys+len <= MAX_DMA_PFN*PAGE_SIZE) {
 		dma_reserve += len / PAGE_SIZE;
 		set_dma_reserve(dma_reserve);
 	}
+
+	return 0;
 }
 
 int kern_addr_valid(unsigned long addr)
include/asm-x86/proto.h +1 -1
@@ -14,7 +14,7 @@ extern void ia32_syscall(void);
 extern void ia32_cstar_target(void);
 extern void ia32_sysenter_target(void);
 
-extern void reserve_bootmem_generic(unsigned long phys, unsigned len);
+extern int reserve_bootmem_generic(unsigned long phys, unsigned len, int flags);
 
 extern void syscall32_cpu_init(void);
 

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#13 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

Quoted 8 lines · Bernhard Walle

Hi Vivek,

Quoted 4 lines · Vivek Goyal

Can you please put some more explanation comment here to explain that why it is ok to return with success, despite the fact that we never reserved any memory.

Quoted 9 lines

Do you think that's ok?

When booting the kdump kernel, the MP tables (for example, or other firmware-reserved memory) of the BIOS are beyond end_pfn. (kexec-tools adds exactmap parameters to the kernel so that the E820 map is no longer used.) Therefore, it's ok to return "success" here. For normal boot, the MP tables must be reserved normally.

Looks good. Thanks

Vivek

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#14 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

Quoted 20 lines · Vivek Goyal
Quoted 8 lines · Bernhard Walle

Hi Vivek,

Quoted 4 lines · Vivek Goyal

Can you please put some more explanation comment here to explain that why it is ok to return with success, despite the fact that we never reserved any memory.

Quoted 9 lines

Do you think that's ok?

When booting the kdump kernel, the MP tables (for example, or other firmware-reserved memory) of the BIOS are beyond end_pfn. (kexec-tools adds exactmap parameters to the kernel so that the E820 map is no longer used.) Therefore, it's ok to return "success" here. For normal boot, the MP tables must be reserved normally.

Quoted 1 line

Looks good. Thanks

Should I resend the whole patch series to Linus for inclusion?

Bernhard

Signature · 6 lines
Bernhard Walle, SUSE LINUX Products GmbH, Architecture Maintenance

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
#15 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

Quoted 21 lines · Bernhard Walle
Quoted 20 lines · Vivek Goyal
Quoted 17 lines · Bernhard Walle

Hi Vivek,

Quoted 4 lines · Vivek Goyal

Can you please put some more explanation comment here to explain that why it is ok to return with success, despite the fact that we never reserved any memory.

Do you think that's ok?

When booting the kdump kernel, the MP tables (for example, or other firmware-reserved memory) of the BIOS are beyond end_pfn. (kexec-tools adds exactmap parameters to the kernel so that the E820 map is no longer used.) Therefore, it's ok to return "success" here. For normal boot, the MP tables must be reserved normally.

Quoted 4 lines
Quoted 1 line

Looks good. Thanks

Should I resend the whole patch series to Linus for inclusion?

I am really not sure. :-). In the past, Andrew picked up the patches, kept in his tree for couple of days and if something is important, he will push it out to Linus.

Thanks
Vivek

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#16 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

Quoted 4 lines · Vivek Goyal

I am really not sure. :-). In the past, Andrew picked up the patches, kept in his tree for couple of days and if something is important, he will push it out to Linus.

Ok, I'm a bit confused about linux-next, and the new x86 tree from Ingo/Thomas and how to merge patches. So, I'll wait a few days and then resend the patches with Andrew in Cc.

Bernhard

Signature · 6 lines
Bernhard Walle, SUSE LINUX Products GmbH, Architecture Maintenance

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
#17 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

In reply to Johannes Weiner
Quoted 13 lines · Johannes Weiner

arch/x86/mm/init_64.c void __init reserve_bootmem_generic(unsi · line 815

Quoted 8 lines
arch/x86/mm/init_64.c +1 -1
 		/*
@@ -811,11 +812,11 @@ void __init reserve_bootmem_generic(unsi
 		 * firmware tables:
 		 */
 		if (pfn < max_pfn_mapped)
-			return;
+			return -EFAULT;

This seemed to be `just do nothing' behaviour. Wouldn't 0 be more correct here? Or something else so there is a difference between the path that does not print a warning (the one below) and the path that does?

Well, I don't think that we should return success when memory allocation fails. For kdump, I think if the memory has not been reserved, then the function should failed, for whatever reason it failed. Because we cannot load the crashkernel.

So IMO the code should look like

[...]

	int ret;

	if (pfn >= end_pfn)
		return -EFAULT;

	/* Should check here against the e820 map to avoid double free */
#ifdef CONFIG_NUMA
	nid = phys_to_nid(phys);

[...]

Bernhard

Signature · 6 lines
Bernhard Walle, SUSE LINUX Products GmbH, Architecture Maintenance

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec
#18 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

In reply to Bernhard Walle

Hi,

Quoted 100 lines · Bernhard Walle

This patch adds a 'flags' parameter to reserve_bootmem_generic() like it already has been added in reserve_bootmem() with commit 72a7fe3967dbf86cb34e24fbf1d957fe24d2f246.

It also changes all users to use BOOTMEM_DEFAULT, which doesn't effectively change the behaviour. Since the change is x86-specific, I don't think it's necessary to add a new API for migration. There are only 4 users of that function.

The change is necessary for the next patch, using reserve_bootmem_generic() for crashkernel reservation.

---
 arch/x86/kernel/e820_64.c |    3 ++-
 arch/x86/kernel/efi_64.c  |    3 ++-
 arch/x86/kernel/mpparse.c |    5 +++--
 arch/x86/mm/init_64.c     |   17 ++++++++++++-----
 include/asm-x86/proto.h   |    2 +-
 5 files changed, 20 insertions(+), 10 deletions(-)
arch/x86/kernel/e820_64.c +2 -1
@@ -118,7 +118,8 @@ void __init early_res_to_bootmem(unsigne
 			continue;
 		printk(KERN_INFO "  early res: %d [%lx-%lx] %s\n", i,
 			final_start, final_end - 1, r->name);
-		reserve_bootmem_generic(final_start, final_end - final_start);
+		reserve_bootmem_generic(final_start, final_end - final_start,
+			BOOTMEM_DEFAULT);
 	}
 }
 
arch/x86/kernel/efi_64.c +2 -1
@@ -100,7 +100,8 @@ void __init efi_call_phys_epilog(void)
 void __init efi_reserve_bootmem(void)
 {
 	reserve_bootmem_generic((unsigned long)memmap.phys_map,
-				memmap.nr_map * memmap.desc_size);
+				memmap.nr_map * memmap.desc_size,
+				BOOTMEM_DEFAULT);
 }
 
 void __iomem * __init efi_ioremap(unsigned long phys_addr, unsigned long size)
arch/x86/kernel/mpparse.c +3 -2
@@ -729,10 +729,11 @@ static int __init smp_scan_config(unsign
 			if (!reserve)
 				return 1;
 
-			reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE);
+			reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE,
+				BOOTMEM_DEFAULT);
 			if (mpf->mpf_physptr)
 				reserve_bootmem_generic(mpf->mpf_physptr,
-							PAGE_SIZE);
+					PAGE_SIZE, BOOTMEM_DEFAULT);
 #endif
 		return 1;
 		}

Files a/arch/x86/kernel/setup_64.o and b/arch/x86/kernel/setup_64.o differ

arch/x86/mm/init_64.c +4 -4
@@ -798,12 +798,13 @@ void free_initrd_mem(unsigned long start
 }
 #endif
 
-void __init reserve_bootmem_generic(unsigned long phys, unsigned len)
+int __init reserve_bootmem_generic(unsigned long phys, unsigned len, int flags)
 {
 #ifdef CONFIG_NUMA
 	int nid, next_nid;
 #endif
 	unsigned long pfn = phys >> PAGE_SHIFT;
+	int ret;
 
 	if (pfn >= end_pfn) {
 		/*
@@ -811,11 +812,11 @@ void __init reserve_bootmem_generic(unsi
 		 * firmware tables:
 		 */
 		if (pfn < max_pfn_mapped)
-			return;
+			return -EFAULT;
 
 		printk(KERN_ERR "reserve_bootmem: illegal reserve %lx %u\n",
 				phys, len);
-		return;
+		return -EFAULT;
 	}
 
 	/* Should check here against the e820 map to avoid double free */
@@ -823,9 +824,13 @@ void __init reserve_bootmem_generic(unsi
 	nid = phys_to_nid(phys);
 	next_nid = phys_to_nid(phys + len - 1);
 	if (nid == next_nid)
-		reserve_bootmem_node(NODE_DATA(nid), phys, len, BOOTMEM_DEFAULT);

arch/x86/mm/init_64.c void __init reserve_bootmem_generic(unsi · line 835

Quoted 10 lines
arch/x86/mm/init_64.c +6 -1
+		ret = reserve_bootmem_node(NODE_DATA(nid), phys, len, flags);
 	else
-		reserve_bootmem(phys, len, BOOTMEM_DEFAULT);
+		ret = reserve_bootmem(phys, len, flags);
+
+	if (ret != 0)
+		return ret;
+
 #else
 	reserve_bootmem(phys, len, BOOTMEM_DEFAULT);
                                   ^^^^^^^^^^^^^^^   flags?

And you ignore the return value here.

Hannes

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#19 lore raw

Re: [patch 2/3] Add flags parameter to reserve_bootmem_generic()

Quoted 6 lines · Johannes Weiner

Review

Quoted 4 lines
(quoted diff) +1 -0
+
 #else
 	reserve_bootmem(phys, len, BOOTMEM_DEFAULT);  
                                   ^^^^^^^^^^^^^^^   flags?

And you ignore the return value here.

Thanks for catching that. Updated patch:

------------------------------------------------------------------
From: Bernhard Walle <[email protected]>
Subject: Add 'flags' parameter to reserve_bootmem_generic()

This patch adds a 'flags' parameter to reserve_bootmem_generic() like it already has been added in reserve_bootmem() with commit 72a7fe3967dbf86cb34e24fbf1d957fe24d2f246.

It also changes all users to use BOOTMEM_DEFAULT, which doesn't effectively change the behaviour. Since the change is x86-specific, I don't think it's necessary to add a new API for migration. There are only 4 users of that function.

The change is necessary for the next patch, using reserve_bootmem_generic() for crashkernel reservation.

---
 arch/x86/kernel/e820_64.c |    3 ++-
 arch/x86/kernel/efi_64.c  |    3 ++-
 arch/x86/kernel/mpparse.c |    5 +++--
 arch/x86/mm/init_64.c     |   19 +++++++++++++------
 include/asm-x86/proto.h   |    2 +-
 5 files changed, 21 insertions(+), 11 deletions(-)
arch/x86/kernel/e820_64.c +2 -1
@@ -118,7 +118,8 @@ void __init early_res_to_bootmem(unsigne
 			continue;
 		printk(KERN_INFO "  early res: %d [%lx-%lx] %s\n", i,
 			final_start, final_end - 1, r->name);
-		reserve_bootmem_generic(final_start, final_end - final_start);
+		reserve_bootmem_generic(final_start, final_end - final_start,
+			BOOTMEM_DEFAULT);
 	}
 }
 
arch/x86/kernel/efi_64.c +2 -1
@@ -100,7 +100,8 @@ void __init efi_call_phys_epilog(void)
 void __init efi_reserve_bootmem(void)
 {
 	reserve_bootmem_generic((unsigned long)memmap.phys_map,
-				memmap.nr_map * memmap.desc_size);
+				memmap.nr_map * memmap.desc_size,
+				BOOTMEM_DEFAULT);
 }
 
 void __iomem * __init efi_ioremap(unsigned long phys_addr, unsigned long size)
arch/x86/kernel/mpparse.c +3 -2
@@ -729,10 +729,11 @@ static int __init smp_scan_config(unsign
 			if (!reserve)
 				return 1;
 
-			reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE);
+			reserve_bootmem_generic(virt_to_phys(mpf), PAGE_SIZE,
+				BOOTMEM_DEFAULT);
 			if (mpf->mpf_physptr)
 				reserve_bootmem_generic(mpf->mpf_physptr,
-							PAGE_SIZE);
+					PAGE_SIZE, BOOTMEM_DEFAULT);
 #endif
 		return 1;
 		}
arch/x86/mm/init_64.c +13 -6
@@ -798,12 +798,13 @@ void free_initrd_mem(unsigned long start
 }
 #endif
 
-void __init reserve_bootmem_generic(unsigned long phys, unsigned len)
+int __init reserve_bootmem_generic(unsigned long phys, unsigned len, int flags)
 {
 #ifdef CONFIG_NUMA
 	int nid, next_nid;
 #endif
 	unsigned long pfn = phys >> PAGE_SHIFT;
+	int ret;
 
 	if (pfn >= end_pfn) {
 		/*
@@ -811,11 +812,11 @@ void __init reserve_bootmem_generic(unsi
 		 * firmware tables:
 		 */
 		if (pfn < max_pfn_mapped)
-			return;
+			return -EFAULT;
 
 		printk(KERN_ERR "reserve_bootmem: illegal reserve %lx %u\n",
 				phys, len);
-		return;
+		return -EFAULT;
 	}
 
 	/* Should check here against the e820 map to avoid double free */
@@ -823,17 +824,23 @@ void __init reserve_bootmem_generic(unsi
 	nid = phys_to_nid(phys);
 	next_nid = phys_to_nid(phys + len - 1);
 	if (nid == next_nid)
-		reserve_bootmem_node(NODE_DATA(nid), phys, len, BOOTMEM_DEFAULT);
+		ret = reserve_bootmem_node(NODE_DATA(nid), phys, len, flags);
 	else
-		reserve_bootmem(phys, len, BOOTMEM_DEFAULT);
+		ret = reserve_bootmem(phys, len, flags);
+
 #else
-	reserve_bootmem(phys, len, BOOTMEM_DEFAULT);
+	ret = reserve_bootmem(phys, len, flags);
 #endif
 
+	if (ret != 0)
+		return ret;
+
 	if (phys+len <= MAX_DMA_PFN*PAGE_SIZE) {
 		dma_reserve += len / PAGE_SIZE;
 		set_dma_reserve(dma_reserve);
 	}
+
+	return 0;
 }
 
 int kern_addr_valid(unsigned long addr)
include/asm-x86/proto.h +1 -1
@@ -14,7 +14,7 @@ extern void ia32_syscall(void);
 extern void ia32_cstar_target(void);
 extern void ia32_sysenter_target(void);
 
-extern void reserve_bootmem_generic(unsigned long phys, unsigned len);
+extern int reserve_bootmem_generic(unsigned long phys, unsigned len, int flags);
 
 extern void syscall32_cpu_init(void);
 

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

#20 lore raw

PATCH 3/3 Use reserve_bootmem_generic() to reserve crashkernel memory on x86_64

In reply to Bernhard Walle

This patch uses reserve_bootmem_generic() instead of reserve_bootmem() to reserve the crashkernel memory on x86_64. That's necessary for NUMA machines, see 00212fef814612245ed0261cbac8426d0c9a31a5:

[PATCH] Fix kdump Crash Kernel boot memory reservation for NUMA machines

This patch will fix a boot memory reservation bug that trashes memory on the ES7000 when loading the kdump crash kernel.

The code in arch/x86_64/kernel/setup.c to reserve boot memory for the crash kernel uses the non-numa aware "reserve_bootmem" function instead of the NUMA aware "reserve_bootmem_generic". I checked to make sure that no other function was using "reserve_bootmem" and found none, except the ones that had NUMA ifdef'ed out.

I have tested this patch only on an ES7000 with NUMA on and off (numa=off) in a single (non-NUMA) and multi-cell (NUMA) configurations.

Looks-good-to: Vivek Goyal <[email protected]>

The switch-back to reserve_bootmem() was accidentally introduced in 5c3391f9f749023a49c64d607da4fb49263690eb when adding the BOOTMEM_EXCLUSIVE parameter.

---
 arch/x86/kernel/setup_64.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/kernel/setup_64.c +1 -1
@@ -243,7 +243,7 @@ static void __init reserve_crashkernel(v
 			return;
 		}
 
-		if (reserve_bootmem(crash_base, crash_size,
+		if (reserve_bootmem_generic(crash_base, crash_size,
 					BOOTMEM_EXCLUSIVE) < 0) {
 			printk(KERN_INFO "crashkernel reservation failed - "
 					"memory is in use\n");

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

Acked-by on the v1 series
#21 lore raw
In reply to Bernhard Walle
Quoted 11 lines · Bernhard Walle

This patch series fixes the crashkernel reservation on NUMA machine. The regression was discovered by Dave Anderson <[email protected]>.

The background is that on NUMA machines, reserve_bootmem_generic() is required instead of reserve_bootmem(). To achieve that, it's necessary to make a few API changes.

The patches are against latest linux-2.6 git. They should still go into 2.6.26 since it's only bug fixing. For 2.6.27, we should unify crashkernel reservation for i386 and x86-64.

Quoted 5 lines

Tested on both i386 and x86-64. Compilation was tested with both kexec disabled and enabled. The change is x86 only, so no need to test on other architectures.

Yes we should be using reserve_bootmem_generic() for reserving memory on NUMA machines. Recently myself and Dave A. ran into crash while reserving memory using resreve_bootmem() on a NUMA machine.

Reason for crash? reserve_bootmem() always assumes node id to be zero, and that was not the case.

Thanks
Vivek

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec

Applied
#22 lore raw
In reply to Bernhard Walle
Quoted 7 lines · Bernhard Walle

This patch series fixes the crashkernel reservation on NUMA machine. The regression was discovered by Dave Anderson <[email protected]>.

The background is that on NUMA machines, reserve_bootmem_generic() is required instead of reserve_bootmem(). To achieve that, it's necessary to make a few API changes.

Quoted 7 lines

The patches are against latest linux-2.6 git. They should still go into 2.6.26 since it's only bug fixing. For 2.6.27, we should unify crashkernel reservation for i386 and x86-64.

Tested on both i386 and x86-64. Compilation was tested with both kexec disabled and enabled. The change is x86 only, so no need to test on other architectures.

applied to tip/x86/numa - thanks Bernhard.

Ingo

_______________________________________________
kexec mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/kexec