x86: add max_addr boot option

Currently, the boot option max_addr is only supported on ia64 platform. We also need it on x86 platform.
For example:
There are two nodes:
 NODE#0  address range 0x00000000 00000000 - 0x00010000 00000000 NODE#1  address range 0x00010000 00000000 - 0x00020000 00000000 If we only want to use node0, we can specify the max_addr. The boot option "mem=" can do the same thing now. But the boot option "mem=" means the total memory used by the system. If we tell the user that the boot option "mem=" can do this, it will confuse the user. So we need an new boot option "max_addr" on x86 platform.

Signed-off-by Wen Congyang <[email protected]>

---
 Documentation/kernel-parameters.txt |    2 +- arch/x86/kernel/e820.c              |   36 +++++++++++++++++++++++++++++++++++

 2 files changed, 37 insertions(+), 1 deletions(-)

Documentation/kernel-parameters.txt +0 -0

index a92c5eb..034609d 100644
@@ -1441,7 +1441,7 @@ bytes respectively. Such letter suffixes can also be entirely omitted.

                        yeeloong laptop.
                       Example: machtype=lemote-yeeloong-2f-7inch

-       max_addr=nn[KMG]        [KNL,BOOT,ia64] All physical memory greater
+       max_addr=nn[KMG]        [KNL,BOOT,ia64,X86] All physical memory greater than or equal to this physical address is ignored.

       maxcpus=        [SMP] Maximum number of processors that an SMP kernel

arch/x86/kernel/e820.c +36 -1

index 4185797..cd07226 100644
@@ -47,6 +47,7 @@ unsigned long pci_mem_start = 0xaeedbabe;
  #ifdef CONFIG_PCI
  EXPORT_SYMBOL(pci_mem_start);
  #endif
+static u64 max_addr = ~0ULL;
 
  /*
  * This function checks if any part of the range <start,end> is mapped

@@ -119,6 +120,20 @@ static void __init __e820_add_region(struct e820map *e820x, u64 start, u64 size,
                return;
        }
 
+       if (start >= max_addr) {
+               printk(KERN_ERR "e820: ignoring [mem %#010llx-%#010llx]\n",
+                      (unsigned long long)start,
+                      (unsigned long long)(start + size - 1));
+               return;
+       }
+
+       if (max_addr - start < size) {
+               printk(KERN_ERR "e820: ignoring [mem %#010llx-%#010llx]\n",
+                      (unsigned long long)max_addr,
+                      (unsigned long long)(start + size - 1));
+               size = max_addr - start;
+       }
+
        e820x->map[x].addr = start;
        e820x->map[x].size = size;
        e820x->map[x].type = type;

@@ -835,6 +850,22 @@ static int __init parse_memopt(char *p)
  }
  early_param("mem", parse_memopt);
 
+static int __init parse_memmax_opt(char *p)
+{
+       char *oldp;
+
+       if (!p)
+               return -EINVAL;
+
+       oldp = p;
+       max_addr = memparse(p, &p);
+       if (p == oldp)
+               return -EINVAL;
+
+       return 0;
+}
+early_param("max_addr", parse_memmax_opt);
+
  static int __init parse_memmap_opt(char *p)
  {
        char *oldp;

@@ -881,6 +912,11 @@ early_param("memmap", parse_memmap_opt);
 
  void __init finish_e820_parsing(void)
  {
+       if (max_addr != ~0ULL) {
+               userdef = 1;
+               e820_remove_range(max_addr, ULLONG_MAX - max_addr, E820_RAM, 1);
+       }
+
        if (userdef) {
                u32 nr = e820.nr_map;
 
--

1.7.1

Quoted 11 lines

Currently, the boot option max_addr is only supported on ia64 platform. We also need it on x86 platform.
For example:
There are two nodes:
NODE#0 address range 0x00000000 00000000 - 0x00010000 00000000 NODE#1 address range 0x00010000 00000000 - 0x00020000 00000000
If we only want to use node0, we can specify the max_addr. The boot option "mem=" can do the same thing now. But the boot option "mem=" means the total memory used by the system. If we tell the user that the boot option "mem=" can do this, it will confuse the user. So we need an new boot option "max_addr" on x86 platform.

I don't object to this patch (and thanks for tweaking the mem range printk).

I don't know what your use case is, but from a user interface perspective, the "max_addr=" option feels like a bit of a hack. If you're trying to avoid use of other nodes, "max_addr" is an awkward way to do it. It requires the user to know the physical address -> node mappings, and it doesn't affect the CPUs and I/O resources on other nodes. You could implement a "numa_node=" or similar parameter that would allow you to ignore remote memory, CPUs, and I/O.

Quoted 89 lines

Signed-off-by Wen Congyang <[email protected]>

---
 Documentation/kernel-parameters.txt |    2 +-
 arch/x86/kernel/e820.c              |   36 +++++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletions(-)

Documentation/kernel-parameters.txt +1 -1

index a92c5eb..034609d 100644
@@ -1441,7 +1441,7 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
                        yeeloong laptop.
                       Example: machtype=lemote-yeeloong-2f-7inch
 
-       max_addr=nn[KMG]        [KNL,BOOT,ia64] All physical memory greater
+       max_addr=nn[KMG]        [KNL,BOOT,ia64,X86] All physical memory greater
                       than or equal to this physical address is ignored.
 
       maxcpus=        [SMP] Maximum number of processors that an SMP kernel

arch/x86/kernel/e820.c +36 -1

index 4185797..cd07226 100644
@@ -47,6 +47,7 @@ unsigned long pci_mem_start = 0xaeedbabe;
 #ifdef CONFIG_PCI
 EXPORT_SYMBOL(pci_mem_start);
 #endif
+static u64 max_addr = ~0ULL;
 
 /*
 * This function checks if any part of the range <start,end> is mapped

@@ -119,6 +120,20 @@ static void __init __e820_add_region(struct e820map *e820x, u64 start, u64 size,
               return;
       }
 
+       if (start >= max_addr) {
+               printk(KERN_ERR "e820: ignoring [mem %#010llx-%#010llx]\n",
+                      (unsigned long long)start,
+                      (unsigned long long)(start + size - 1));
+               return;
+       }
+
+       if (max_addr - start < size) {
+               printk(KERN_ERR "e820: ignoring [mem %#010llx-%#010llx]\n",
+                      (unsigned long long)max_addr,
+                      (unsigned long long)(start + size - 1));
+               size = max_addr - start;
+       }
+
       e820x->map[x].addr = start;
       e820x->map[x].size = size;
       e820x->map[x].type = type;

@@ -835,6 +850,22 @@ static int __init parse_memopt(char *p)
 }
 early_param("mem", parse_memopt);
 
+static int __init parse_memmax_opt(char *p)
+{
+       char *oldp;
+
+       if (!p)
+               return -EINVAL;
+
+       oldp = p;
+       max_addr = memparse(p, &p);
+       if (p == oldp)
+               return -EINVAL;
+
+       return 0;
+}
+early_param("max_addr", parse_memmax_opt);
+
 static int __init parse_memmap_opt(char *p)
 {
       char *oldp;

@@ -881,6 +912,11 @@ early_param("memmap", parse_memmap_opt);
 
 void __init finish_e820_parsing(void)
 {
+       if (max_addr != ~0ULL) {
+               userdef = 1;
+               e820_remove_range(max_addr, ULLONG_MAX - max_addr, E820_RAM, 1);
+       }
+
       if (userdef) {
               u32 nr = e820.nr_map;
 
--

Quoted 2 lines

1.7.1

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

At 06/12/2012 01:35 AM, Bjorn Helgaas Wrote:

Quoted 15 lines · Wen Congyang

Quoted 11 lines

Currently, the boot option max_addr is only supported on ia64 platform. We also need it on x86 platform.
For example:
There are two nodes:
 NODE#0  address range 0x00000000 00000000 - 0x00010000 00000000 NODE#1  address range 0x00010000 00000000 - 0x00020000 00000000 If we only want to use node0, we can specify the max_addr. The boot option "mem=" can do the same thing now. But the boot option "mem=" means the total memory used by the system. If we tell the user that the boot option "mem=" can do this, it will confuse the user. So we need an new boot option "max_addr" on x86 platform.

I don't object to this patch (and thanks for tweaking the mem range printk).

Quoted 7 lines

I don't know what your use case is, but from a user interface perspective, the "max_addr=" option feels like a bit of a hack.  If you're trying to avoid use of other nodes, "max_addr" is an awkward way to do it.  It requires the user to know the physical address -> node mappings, and it doesn't affect the CPUs and I/O resources on other nodes.  You could implement a "numa_node=" or similar parameter that would allow you to ignore remote memory, CPUs, and I/O.

Currently, I only need to ignore the memory. If we need to ignore a node, "numa_node=" or similar parameter is a better choice.

Quoted 96 lines

Quoted 88 lines

Signed-off-by Wen Congyang <[email protected]>

---
 Documentation/kernel-parameters.txt |    2 +- arch/x86/kernel/e820.c              |   36 +++++++++++++++++++++++++++++++++++

 2 files changed, 37 insertions(+), 1 deletions(-)

Documentation/kernel-parameters.txt +0 -0

index a92c5eb..034609d 100644
@@ -1441,7 +1441,7 @@ bytes respectively. Such letter suffixes can also be entirely omitted.

                        yeeloong laptop.
                       Example: machtype=lemote-yeeloong-2f-7inch

-       max_addr=nn[KMG]        [KNL,BOOT,ia64] All physical memory greater
+       max_addr=nn[KMG]        [KNL,BOOT,ia64,X86] All physical memory greater than or equal to this physical address is ignored.

       maxcpus=        [SMP] Maximum number of processors that an SMP kernel

arch/x86/kernel/e820.c +36 -1

index 4185797..cd07226 100644
@@ -47,6 +47,7 @@ unsigned long pci_mem_start = 0xaeedbabe;
  #ifdef CONFIG_PCI
  EXPORT_SYMBOL(pci_mem_start);
  #endif
+static u64 max_addr = ~0ULL;
 
  /*
  * This function checks if any part of the range <start,end> is mapped

@@ -119,6 +120,20 @@ static void __init __e820_add_region(struct e820map *e820x, u64 start, u64 size,
                return;
        }
 
+       if (start >= max_addr) {
+               printk(KERN_ERR "e820: ignoring [mem %#010llx-%#010llx]\n",
+                      (unsigned long long)start,
+                      (unsigned long long)(start + size - 1));
+               return;
+       }
+
+       if (max_addr - start < size) {
+               printk(KERN_ERR "e820: ignoring [mem %#010llx-%#010llx]\n",
+                      (unsigned long long)max_addr,
+                      (unsigned long long)(start + size - 1));
+               size = max_addr - start;
+       }
+
        e820x->map[x].addr = start;
        e820x->map[x].size = size;
        e820x->map[x].type = type;

@@ -835,6 +850,22 @@ static int __init parse_memopt(char *p)
  }
  early_param("mem", parse_memopt);
 
+static int __init parse_memmax_opt(char *p)
+{
+       char *oldp;
+
+       if (!p)
+               return -EINVAL;
+
+       oldp = p;
+       max_addr = memparse(p, &p);
+       if (p == oldp)
+               return -EINVAL;
+
+       return 0;
+}
+early_param("max_addr", parse_memmax_opt);
+
  static int __init parse_memmap_opt(char *p)
  {
        char *oldp;

@@ -881,6 +912,11 @@ early_param("memmap", parse_memmap_opt);
 
  void __init finish_e820_parsing(void)
  {
+       if (max_addr != ~0ULL) {
+               userdef = 1;
+               e820_remove_range(max_addr, ULLONG_MAX - max_addr, E820_RAM, 1);
+       }
+
        if (userdef) {
                u32 nr = e820.nr_map;
 
--

Quoted 2 lines

1.7.1

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/

Quoted 24 lines · Wen Congyang

At 06/12/2012 01:35 AM, Bjorn Helgaas Wrote:

Quoted 22 lines · Wen Congyang

Quoted 11 lines

Currently, the boot option max_addr is only supported on ia64 platform. We also need it on x86 platform.
For example:
There are two nodes:
NODE#0 address range 0x00000000 00000000 - 0x00010000 00000000 NODE#1 address range 0x00010000 00000000 - 0x00020000 00000000
If we only want to use node0, we can specify the max_addr. The boot option "mem=" can do the same thing now. But the boot option "mem=" means the total memory used by the system. If we tell the user that the boot option "mem=" can do this, it will confuse the user. So we need an new boot option "max_addr" on x86 platform.

I don't object to this patch (and thanks for tweaking the mem range printk).

I don't know what your use case is, but from a user interface perspective, the "max_addr=" option feels like a bit of a hack. If you're trying to avoid use of other nodes, "max_addr" is an awkward way to do it. It requires the user to know the physical address -> node mappings, and it doesn't affect the CPUs and I/O resources on other nodes. You could implement a "numa_node=" or similar parameter that would allow you to ignore remote memory, CPUs, and I/O.

Quoted 2 lines

Currently, I only need to ignore the memory. If we need to ignore a node, "numa_node=" or similar parameter is a better choice.

Doesn't the end user have to know the memory map of the system to use "max_addr="? How do you know what value to supply? Do you have to attempt a boot once to discover the highest address on node 0? What if node 0 and node 1 memory are interleaved, so there's some node 1 memory below the highest node 0 address?

Quoted 9 lines

Quoted 3 lines

Currently, I only need to ignore the memory. If we need to ignore a node,
"numa_node=" or similar parameter is a better choice.

Doesn't the end user have to know the memory map of the system to use "max_addr="? How do you know what value to supply? Do you have to attempt a boot once to discover the highest address on node 0? What if node 0 and node 1 memory are interleaved, so there's some node 1 memory below the highest node 0 address?

Current our plan is to avoid asking end-user to fix their boot option by hand even if memory size per node is changed. We'll ship a hardware, which has _fixed_ physical address range per each node regardless of equipped memory size.

Problem happens => reboot (disable some DIMM) => remove memmap= option for avoiding
trouble => check memory layout again =>fix mem_map= => reboot again. This reboot takes much time because the system which have Dynamic-partitioning tends to
be big....so, we'd like to have some _relaxed_ way to specify the region of memory.

Problem happens => reboot (disable some DIMM) => no changes required (because we have enough memory hole between Node0 and Node1.)

BTW, how do you think about mem= boot option which works as max_addr=, now ?
This caused troubles some times on our support-desk, saying
Q. I specified mem=8G boot option but it seems the system has only 7GB.... A. it's because of PCI configuration area on 3G-4G address range...

Even if our requirement can be covered current mem= option, I'd like to have max_addr= option and make mem= option to be sane as ia64.

Quoted 10 lines · Kamezawa Hiroyuki

Quoted 9 lines

Quoted 3 lines

Currently, I only need to ignore the memory. If we need to ignore a node,
"numa_node=" or similar parameter is a better choice.

Doesn't the end user have to know the memory map of the system to use "max_addr="? How do you know what value to supply? Do you have to attempt a boot once to discover the highest address on node 0? What if node 0 and node 1 memory are interleaved, so there's some node 1 memory below the highest node 0 address?

Quoted 4 lines

Current our plan is to avoid asking end-user to fix their boot option by hand even if memory size per node is changed. We'll ship a hardware, which has _fixed_ physical address range per each node regardless of equipped memory size.

I.E. you'll be configuring this yourself when you ship hardware.

You're adding an option because you consider it less confusing for your end users who are digging into kernel parameters, but you will set this new option for your users because they haven't got the information to set it themselves?

Quoted 10 lines

Problem happens => reboot (disable some DIMM) => remove memmap= option for avoiding
trouble => check memory layout again =>fix mem_map= => reboot again. This reboot takes much time because the system which have Dynamic-partitioning tends to
be big....so, we'd like to have some _relaxed_ way to specify the region of memory.

Problem happens => reboot (disable some DIMM) => no changes required (because we have enough memory hole between Node0 and Node1.)

I'm guessing the above means "or you'll be providing some tool that does it when they install/remove memory in the hardware"...

Quoted 5 lines

BTW, how do you think about mem= boot option which works as max_addr=, now ?
This caused troubles some times on our support-desk, saying
Q. I specified mem=8G boot option but it seems the system has only 7GB.... A. it's because of PCI configuration area on 3G-4G address range...

So you're saying there are already two ways to do this, but you want to add a third to be less confusing for end users who are modifying the linux kernel boot parameters by hand using information only you can supply to them?

I'm confused...

Quoted 5 lines

You're adding an option because you consider it less confusing for your end users who are digging into kernel parameters, but you will set this new option for your users because they haven't got the information to set it themselves?

My users don't need to know about hardware settings and the meaning of kernel params. They'll just do as we ask to do.

Quoted 5 lines

So you're saying there are already two ways to do this, but you want to add a third to be less confusing for end users who are modifying the linux kernel boot parameters by hand using information only you can supply to them?

Quoted 2 lines

I'm confused...

I'm just saying current mem= implemenation seems buggy because spec. and impl. doesn't match. So, we're just afraid that someone other than us will fix it and break our assumption how mem= works. It's dangerous to build a production on a feature where spec. and impl. doesn't match. So, we proposed to add max_addr= option for avoiding that situation.

Currently, the boot option max_addr is only supported on ia64 platform. We also need it on x86 platform.
For example:
There are two nodes:
NODE#0 address range 0x00000000 00000000 - 0x00010000 00000000 NODE#1 address range 0x00010000 00000000 - 0x00020000 00000000
If we only want to use node0, we can specify the max_addr. The boot option "mem=" can do the same thing now. But the boot option "mem=" means the total memory used by the system. If we tell the user that the boot option "mem=" can do this, it will confuse the user. So we need an new boot option "max_addr" on x86 platform.

Quoted 12 lines

Currently, the boot option max_addr is only supported on ia64 platform. We also need it on x86 platform.
For example:
There are two nodes:
NODE#0 address range 0x00000000 00000000 - 0x00010000 00000000 NODE#1 address range 0x00010000 00000000 - 0x00020000 00000000
If we only want to use node0, we can specify the max_addr. The boot option "mem=" can do the same thing now. But the boot option "mem=" means the total memory used by the system. If we tell the user that the boot option "mem=" can do this, it will confuse the user. So we need an new boot option "max_addr" on x86 platform.

I fail to see what this does that cannot be done with the since-long-existing memmap= option. Could you address why memmap= doesn't match your needs?

-hpa

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

At 06/12/2012 05:15 AM, H. Peter Anvin Wrote:

Quoted 2 lines · Wen Congyang

Quoted 1 line

Currently, the boot option max_addr is only supported on ia64

platform.

Quoted 12 lines

Quoted 11 lines

We also need it on x86 platform.
For example:
There are two nodes:
NODE#0 address range 0x00000000 00000000 - 0x00010000 00000000 NODE#1 address range 0x00010000 00000000 - 0x00020000 00000000
If we only want to use node0, we can specify the max_addr. The boot option "mem=" can do the same thing now. But the boot option "mem=" means the total memory used by the system. If we tell the user that the boot option "mem=" can do this, it will confuse the user. So we need an new boot option "max_addr" on x86 platform.

Quoted 3 lines

I fail to see what this does that cannot be done with the since-long-existing memmap= option. Could you address why memmap= doesn't match your needs?

The memmap= option is very diffcult to use. The end user should know the memory
map in the system. The end user can get the max address of NODE#0, but he
may not know the memory map for NODE#0. If the end user give the wrong memory
map, the kernel can not boot. For example: I add memmap=16G@0 in the kernel
parameter, and the kernel cannot boot. The max_addr is more easier to use.

Thanks
Wen Congyang

Quoted 5 lines

-hpa

--
To unsubscribe from this list: send the line "unsubscribe

linux-kernel" in

Quoted 4 lines

the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

If what you care about is nodes, why not have an option to specify a map?

But now, we know mem= boot option is buggy....it acts as max_addr= option, we have concerns that 'someone may fix mem= option as sane as ia64. because it's buggy".

We'd like to fix mem= boot option by ourselves and preserve old behavior with max_addr= boot option, which ia64 has.

Quoted 8 lines · Kamezawa Hiroyuki

But now, we know mem= boot option is buggy....it acts as max_addr= option, we have concerns that 'someone may fix mem= option as sane as ia64. because it's buggy".

We'd like to fix mem= boot option by ourselves and preserve old behavior with max_addr= boot option, which ia64 has.

Now I'm *really* confused.

Realistically, there is no point in the old mem= behavior of assuming a contiguous chunk of memory up to that point; it simply doesn't match how modern hardware is constructed. Your notion that ia64 is "sane" is probably more of "outdated" in my opinion.

As such, the current behavior for mem= seems like the right thing and the change was intentional (not to mention has been in place since kernel 2.5.65, back in 2003); it also solves your requirements. If you are concerned about it, it would make more sense to make sure it is documented as intentional.

In fact, it looks like IA64 introduced a divergence when the max_addr= patch was introduced in 2004. You're basically proposing the same divergence for x86 now; talk about having the tail wag the dog.

Sorry. NAK.

Quoted 8 lines

But now, we know mem= boot option is buggy....it acts as max_addr= option, we have concerns that 'someone may fix mem= option as sane as ia64. because it's buggy".

We'd like to fix mem= boot option by ourselves and preserve old behavior with max_addr= boot option, which ia64 has.

Now I'm *really* confused.

Realistically, there is no point in the old mem= behavior of assuming a contiguous chunk of memory up to that point; it simply doesn't match how modern hardware is constructed. Your notion that ia64 is "sane" is probably more of "outdated" in my opinion.

As such, the current behavior for mem= seems like the right thing and the change was intentional (not to mention has been in place since kernel 2.5.65, back in 2003); it also solves your requirements. If you are concerned about it, it would make more sense to make sure it is documented as intentional.

In fact, it looks like IA64 introduced a divergence when the max_addr= patch was introduced in 2004. You're basically proposing the same divergence for x86 now; talk about having the tail wag the dog.

Sorry. NAK.

-hpa