x86: add brk allocation for very, very early allocations
Subsystems (3)
RFC PATCH 1/1 x86: add brk allocation for very, very early allocations
[
I'd like to add a mechanism like this so I can dynamically allocate some Xen-related structures, rather than statically allocating them in the bss, both so that Xen has less overhead when it isn't being used, and so I can scale better to things like memory size.
I think this is more widely useful; it would supplant dmi_alloc_data[], for example, and I'm sure there's other cases.
This is fundimentally the same as head_32.S's extension of the bss to build the initial kernel mapping, but 64-bit doesn't currently do anything analogous to this.
Unfortunately when I use this code as-is I'm getting crashes when the slab allocator starts up. I think this is all correct, but I'm wondering if there's something I'm overlooking which is broken in principle.
So, what am I missing?
Thanks,
J
]
Add a brk()-like allocator which effectively extends the bss in order to allow very early code to do dynamic allocations. This is better than using statically allocated arrays for data in subsystems which may never get used.
The amount of space available depends on how much the initial kernel mappings have covered, and so is fairly limited.
arch/x86/include/asm/setup.h +0 -0
@@ -99,6 +99,11 @@ extern struct boot_params boot_params;*/
#define LOWMEMSIZE() (0x9f000)
+/* exceedingly early brk-like allocator */
+extern unsigned long _brk_start, _brk_end;
+void init_brk(unsigned long start);
+void *extend_brk(size_t size, size_t align);
+
#ifdef __i386__
void __init i386_start_kernel(void);
arch/x86/kernel/head32.c +0 -0
@@ -34,6 +34,8 @@ void __init i386_start_kernel(void)
reserve_ebda_region();
+ init_brk((unsigned long)__va(init_pg_tables_end));
+
/*
* At this point everything still needed from the boot loader
* or BIOS or kernel text should be early reserved or marked not
arch/x86/kernel/head64.c +0 -0
@@ -91,6 +91,8 @@ void __init x86_64_start_kernel(char * real_mode_data)
if (console_loglevel == 10)
early_printk("Kernel alive\n");+ init_brk((unsigned long)&_end);
+
x86_64_start_reservations(real_mode_data); }
arch/x86/kernel/setup.c +0 -0
@@ -113,6 +113,7 @@
#endif
unsigned int boot_cpu_id __read_mostly;
+__initdata unsigned long _brk_start, _brk_end;
#ifdef CONFIG_X86_64
int default_cpu_present_to_apicid(int mps_cpu)
@@ -335,6 +336,26 @@ static void __init relocate_initrd(void) }
#endif
+void __init init_brk(unsigned long brk)
+{
+ _brk_start = _brk_end = brk;
+}
+
+void * __init extend_brk(size_t size, size_t align) +{
+ size_t mask = align - 1;
+ void *ret;
+
+ BUG_ON(align & mask);
+
+ _brk_end = (_brk_end + mask) & ~mask;
+
+ ret = (void *)_brk_end;
+ _brk_end += size;
+
+ return ret;
+}
+
static void __init reserve_initrd(void)
{
u64 ramdisk_image = boot_params.hdr.ramdisk_image;
@@ -727,11 +748,7 @@ void __init setup_arch(char **cmdline_p)
init_mm.start_code = (unsigned long) _text;
init_mm.end_code = (unsigned long) _etext;
init_mm.end_data = (unsigned long) _edata;-#ifdef CONFIG_X86_32
- init_mm.brk = init_pg_tables_end + PAGE_OFFSET;
-#else
- init_mm.brk = (unsigned long) &_end;
-#endif
+ init_mm.brk = _brk_end;
code_resource.start = virt_to_phys(_text); code_resource.end = virt_to_phys(_etext)-1; @@ -897,6 +914,9 @@ void __init setup_arch(char **cmdline_p) acpi_numa_init();
#endif
+ if (_brk_end > _brk_start)
+ reserve_early(__pa(_brk_start), __pa(_brk_end), "BRK");
+
initmem_init(0, max_pfn);
#ifdef CONFIG_ACPI_SLEEP
arch/x86/mm/pageattr.c +0 -0
@@ -16,6 +16,7 @@
#include <asm/processor.h>
#include <asm/tlbflush.h>
#include <asm/sections.h>
+#include <asm/setup.h>
#include <asm/uaccess.h>
#include <asm/pgalloc.h>
#include <asm/proto.h>
@@ -95,7 +96,7 @@ static inline unsigned long highmap_start_pfn(void)
static inline unsigned long highmap_end_pfn(void)
{
- return __pa(roundup((unsigned long)_end, PMD_SIZE)) >> PAGE_SHIFT;
+ return __pa(roundup(_brk_end, PMD_SIZE)) >> PAGE_SHIFT;
}
#endif
@@ -707,7 +708,7 @@ static int cpa_process_alias(struct cpa_data *cpa)
* No need to redo, when the primary call touched the high
* mapping already:
*/
- if (within(vaddr, (unsigned long) _text, (unsigned long) _end))
+ if (within(vaddr, (unsigned long) _text, _brk_end))
return 0; /*
arch/x86/xen/enlighten.c +0 -0
@@ -975,6 +975,9 @@ asmlinkage void __init xen_start_kernel(void)
init_mm.pgd = pgd;
+ /* Set up very early brk allocator after Xen pagetables */
+ init_brk(xen_start_info->pt_base + xen_start_info->nr_pt_frames * PAGE_SIZE);
+
/* keep using Xen gdt for now; no urgent need to change it */ pv_info.kernel_rpl = 1;
H. Peter Anvin
reviewerQuoted 8 lines · Jeremy Fitzhardinge
I'd like to add a mechanism like this so I can dynamically allocate some Xen-related structures, rather than statically allocating them in the bss, both so that Xen has less overhead when it isn't being used, and so I can scale better to things like memory size.
I think this is more widely useful; it would supplant dmi_alloc_data[], for example, and I'm sure there's other cases.
Quoted 8 lines
This is fundimentally the same as head_32.S's extension of the bss to build the initial kernel mapping, but 64-bit doesn't currently do anything analogous
to this.Unfortunately when I use this code as-is I'm getting crashes when the slab allocator starts up. I think this is all correct, but I'm wondering if there's something I'm overlooking which is broken in principle.
First of all, I like the concept. As far as getting crashes, I suspect what you're finding is some use of this type of extended memory space that just isn't documented. I would try this in a simulator, setting a watchpoint on _end to see if you get any hits before the slab allocator starts.
-hpa
Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center I work for Intel. I don't speak on their behalf.
H. Peter Anvin
reviewerQuoted 4 lines · Jeremy Fitzhardinge
unsigned int
boot_cpu_id__read_mostly;
+__initdata unsigned long_brk_start,_brk_end;
Better yet, initialize _brk_start and _brk_end statically:
extern const char _end[];__initdata unsigned long _brk_start = (unsigned long)&_end;__initdata unsigned long _brk_end = (unsigned long)&_end;
That way it's available from the first instruction, and we can fully remove the x86-32 init_pg_tables_end and replace it with _brk_end (thus putting the initial page tables in the brk.)
-hpa
Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center I work for Intel. I don't speak on their behalf.
Quoted 9 lines · H. Peter Anvin
Quoted 4 lines · Jeremy Fitzhardinge
unsigned int
boot_cpu_id__read_mostly;
+__initdata unsigned long_brk_start,_brk_end;Better yet, initialize
_brk_startand_brk_endstatically:
Quoted 7 lines
extern const char _end[];
__initdataunsigned long_brk_start= (unsigned long)&_end;__initdataunsigned long_brk_end= (unsigned long)&_end;That way it's available from the first instruction, and we can fully remove the x86-32
init_pg_tables_endand replace it with_brk_end(thus putting the initial page tables in the brk.)
Yes. It doesn't work for Xen (the domain builder puts the initrd and Xen-built pagetable immediately after the kernel's bss), but its no problem to move the brk in that case.
J
Quoted 130 lines · Jeremy Fitzhardinge
[
I'd like to add a mechanism like this so I can dynamically allocate some Xen-related structures, rather than statically allocating them in the bss, both so that Xen has less overhead when it isn't being used, and so I can scale better to things like memory size.I think this is more widely useful; it would supplant dmi_alloc_data[], for example, and I'm sure there's other cases.
This is fundimentally the same as head_32.S's extension of the bss to build the initial kernel mapping, but 64-bit doesn't currently do anything analogous
to this.Unfortunately when I use this code as-is I'm getting crashes when the slab allocator starts up. I think this is all correct, but I'm wondering if there's something I'm overlooking which is broken in principle.
So, what am I missing?
Thanks,
J
]Add a brk()-like allocator which effectively extends the bss in order to allow very early code to do dynamic allocations. This is better than using statically allocated arrays for data in subsystems which may never get used.
The amount of space available depends on how much the initial kernel mappings have covered, and so is fairly limited.
Not-Signed-off-by Jeremy Fitzhardinge <[email protected]>arch/x86/include/asm/setup.h +5
-0@@ -99,6 +99,11 @@ extern struct boot_params boot_params;*/ #define LOWMEMSIZE () (0x9f000) +/* exceedingly early brk-like allocator */ +extern unsigned long _brk_start, _brk_end; +void init_brk (unsigned long start); +void *extend_brk (size_t size,size_t align); +#ifdef __i386__void __init i386_start_kernel (void );arch/x86/kernel/head32.c +2
-0@@ -34,6 +34,8 @@ void __init i386_start_kernel(void)reserve_ebda_region (); +init_brk ((unsigned long )__va (init_pg_tables_end)); +/* * At this point everything still needed from the boot loader * or BIOS or kernel text should be early reserved or marked not arch/x86/kernel/head64.c +2
-0@@ -91,6 +91,8 @@ void __init x86_64_start_kernel(char * real_mode_data)if (console_loglevel ==10 )early_printk ("Kernel alive\n" ); +init_brk ((unsigned long )&_end); +x86_64_start_reservations (real_mode_data); }arch/x86/kernel/setup.c +22
-5@@ -113,6 +113,7 @@#endif unsigned int boot_cpu_id__read_mostly ; +__initdata unsigned long _brk_start, _brk_end;#ifdef CONFIG_X86_64 int default_cpu_present_to_apicid (int mps_cpu)@@ -335,6 +336,26 @@ static void __init relocate_initrd(void) }#endif +void __init init_brk (unsigned long brk) +{ + _brk_start = _brk_end = brk; +} + +void *__init extend_brk (size_t size,size_t align) +{ +size_t mask = align -1 ; +void *ret; + +BUG_ON (align & mask); + + _brk_end = (_brk_end + mask) & ~mask; + + ret = (void *)_brk_end; + _brk_end += size; + +return ret; +} +static void __init reserve_initrd (void ) {u64 ramdisk_image = boot_params.hdr.ramdisk_image;@@ -727,11 +748,7 @@ void __init setup_arch(char **cmdline_p) init_mm.start_code = (unsigned long ) _text; init_mm.end_code = (unsigned long ) _etext; init_mm.end_data = (unsigned long ) _edata; -#ifdef CONFIG_X86_32 - init_mm.brk = init_pg_tables_end +PAGE_OFFSET ; -#else - init_mm.brk = (unsigned long ) &_end; -#endif + init_mm.brk = _brk_end; code_resource.start =virt_to_phys (_text); code_resource.end =virt_to_phys (_etext)-1 ;
arch/x86/kernel/setup.c void __init setup_arch(char **cmdline_p) · line 920
Quoted 7 lines
arch/x86/kernel/setup.c +3
-0@@ -897,6 +914,9 @@ void __init setup_arch(char **cmdline_p)acpi_numa_init ();#endif +if (_brk_end > _brk_start) +reserve_early (__pa (_brk_start),__pa (_brk_end),"BRK" ); +initmem_init (0 , max_pfn);
it seems reserve _brk_end is some late?
init_memory_mapping(0,...) could get some for direct mapping page table. and it could start from _end...
YH
Quoted 4 lines · Yinghai Lu
it seems reserve
_brk_endis some late?
init_memory_mapping(0,...)could get some for direct mapping page table. and it could start from_end...
Ah, thanks! I was confused by the use of reserve_early() in initmem_init(), but they're OK because they allocate out of the e820 map at that point (and init_memory_mapping() reserves everything it allocates).
I'll give it a whirl.
J
H. Peter Anvin
reviewerQuoted 5 lines · Yinghai Lu
it seems reserve
_brk_endis some late?
init_memory_mapping(0,...)could get some for direct mapping page table. and it could start from_end...
That would make sense. init_memory_mapping() needs to know what not to step on, and that would include the brk at this stage. For 64-bit mode, Xen is the only user of !PSE, and so may be particularly vulnerable to this issue (because of massively larger direct mapping tables.)
This reminds me... is your intent that the BRK is permanent (unless explicitly freed on a page by page basis) or part of the init memory that is flushed?
-hpa
Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center I work for Intel. I don't speak on their behalf.
Quoted 5 lines · H. Peter Anvin
That would make sense.
init_memory_mapping()needs to know what not to step on, and that would include the brk at this stage. For 64-bit mode, Xen is the only user of !PSE, and so may be particularly vulnerable to this issue (because of massively larger direct mapping tables.)
Yep.
Quoted 4 lines
This reminds me... is your intent that the BRK is permanent (unless explicitly freed on a page by page basis) or part of the init memory that is flushed?
Permanent. I'm using it to allocate things which are the moral equivalent of the kernel pagetables.
J
H. Peter Anvin
reviewerQuoted 8 lines · Jeremy Fitzhardinge
Quoted 4 lines
This reminds me... is your intent that the BRK is permanent (unless explicitly freed on a page by page basis) or part of the init memory that is flushed?
Permanent. I'm using it to allocate things which are the moral equivalent of the kernel pagetables.
Cool; just wanted it noted what the rules were. Even more so the reason to fold as many possible uses as possible into this new mechanism.
-hpa
P.S. Could you send me the updates to the dom0 patches you said you had?
Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center I work for Intel. I don't speak on their behalf.
Quoted 3 lines · H. Peter Anvin
Cool; just wanted it noted what the rules were. Even more so the reason to fold as many possible uses as possible into this new mechanism.
Yes, I just converted i386 pagetable construction and dmi_alloc(). Boots under qemu.
Quoted 2 lines
P.S. Could you send me the updates to the dom0 patches you said you had?
Was just tidying them up for posting but got slightly sidetracked. (The TSS/io-bitmap changes are a prelude to the dom0 patches though.)
J