Skip to contentSkip to conversation

x86: add brk allocation for very, very early allocations

RFC PATCH In review Jeremy Fitzhardinge 3 participants 10 messages Feb 27, 2009

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 x86: add brk allocation for very, very early allocations

[
I'd like to add a mechanism like this so I can dynamically allocate some Xen-related structures, rather than statically allocating them in the bss, both so that Xen has less overhead when it isn't being used, and so I can scale better to things like memory size.

I think this is more widely useful; it would supplant dmi_alloc_data[], for example, and I'm sure there's other cases.

This is fundimentally the same as head_32.S's extension of the bss to build the initial kernel mapping, but 64-bit doesn't currently do anything analogous to this.

Unfortunately when I use this code as-is I'm getting crashes when the slab allocator starts up. I think this is all correct, but I'm wondering if there's something I'm overlooking which is broken in principle.

So, what am I missing?

Thanks,
J
]

Add a brk()-like allocator which effectively extends the bss in order to allow very early code to do dynamic allocations. This is better than using statically allocated arrays for data in subsystems which may never get used.

The amount of space available depends on how much the initial kernel mappings have covered, and so is fairly limited.

arch/x86/include/asm/setup.h +0 -0
index 66801cb..e6b754b 100644
@@ -99,6 +99,11 @@ extern struct boot_params boot_params;
 */

#define LOWMEMSIZE() (0x9f000)

+/* exceedingly early brk-like allocator */
+extern unsigned long _brk_start, _brk_end;
+void init_brk(unsigned long start);
+void *extend_brk(size_t size, size_t align);
+
#ifdef __i386__

void __init i386_start_kernel(void);

arch/x86/kernel/head32.c +0 -0
index ac108d1..fa9ae31 100644
@@ -34,6 +34,8 @@ void __init i386_start_kernel(void)
 
	reserve_ebda_region();

+ init_brk((unsigned long)__va(init_pg_tables_end));
+
/*
* At this point everything still needed from the boot loader
* or BIOS or kernel text should be early reserved or marked not

arch/x86/kernel/head64.c +0 -0
index f5b2722..4b29802 100644
@@ -91,6 +91,8 @@ void __init x86_64_start_kernel(char * real_mode_data)
	if (console_loglevel == 10)
		early_printk("Kernel alive\n");

+ init_brk((unsigned long)&_end);
+

	x86_64_start_reservations(real_mode_data);
}
arch/x86/kernel/setup.c +0 -0
index 0d051b4..8899cfa 100644
@@ -113,6 +113,7 @@

#endif

unsigned int boot_cpu_id __read_mostly;
+__initdata unsigned long _brk_start, _brk_end;

#ifdef CONFIG_X86_64
int default_cpu_present_to_apicid(int mps_cpu)
@@ -335,6 +336,26 @@ static void __init relocate_initrd(void) }
#endif

+void __init init_brk(unsigned long brk)
+{
+ _brk_start = _brk_end = brk;
+}
+
+void * __init extend_brk(size_t size, size_t align) +{
+ size_t mask = align - 1;
+ void *ret;
+
+ BUG_ON(align & mask);
+
+ _brk_end = (_brk_end + mask) & ~mask;
+
+ ret = (void *)_brk_end;
+ _brk_end += size;
+
+ return ret;
+}
+
static void __init reserve_initrd(void)

{
	u64 ramdisk_image = boot_params.hdr.ramdisk_image;
@@ -727,11 +748,7 @@ void __init setup_arch(char **cmdline_p)
	init_mm.start_code = (unsigned long) _text;
	init_mm.end_code = (unsigned long) _etext;
	init_mm.end_data = (unsigned long) _edata;

-#ifdef CONFIG_X86_32
- init_mm.brk = init_pg_tables_end + PAGE_OFFSET;
-#else
- init_mm.brk = (unsigned long) &_end;
-#endif
+ init_mm.brk = _brk_end;

	code_resource.start = virt_to_phys(_text);
	code_resource.end = virt_to_phys(_etext)-1;
@@ -897,6 +914,9 @@ void __init setup_arch(char **cmdline_p)
	acpi_numa_init();

#endif

+ if (_brk_end > _brk_start)
+ reserve_early(__pa(_brk_start), __pa(_brk_end), "BRK");
+

	initmem_init(0, max_pfn);

#ifdef CONFIG_ACPI_SLEEP

arch/x86/mm/pageattr.c +0 -0
index dd5df65..4222ed6 100644
@@ -16,6 +16,7 @@

#include <asm/processor.h>
#include <asm/tlbflush.h>
#include <asm/sections.h>
+#include <asm/setup.h>
#include <asm/uaccess.h>
#include <asm/pgalloc.h>
#include <asm/proto.h>
@@ -95,7 +96,7 @@ static inline unsigned long highmap_start_pfn(void)

static inline unsigned long highmap_end_pfn(void)
{
- return __pa(roundup((unsigned long)_end, PMD_SIZE)) >> PAGE_SHIFT;
+ return __pa(roundup(_brk_end, PMD_SIZE)) >> PAGE_SHIFT;
}

#endif
@@ -707,7 +708,7 @@ static int cpa_process_alias(struct cpa_data *cpa)
* No need to redo, when the primary call touched the high
* mapping already:
*/
- if (within(vaddr, (unsigned long) _text, (unsigned long) _end))
+ if (within(vaddr, (unsigned long) _text, _brk_end))

		return 0;

	/*
arch/x86/xen/enlighten.c +0 -0
index 1815b8a..a3cc9ff 100644
@@ -975,6 +975,9 @@ asmlinkage void __init xen_start_kernel(void)
 
	init_mm.pgd = pgd;

+ /* Set up very early brk allocator after Xen pagetables */
+ init_brk(xen_start_info->pt_base + xen_start_info->nr_pt_frames * PAGE_SIZE);
+

	/* keep using Xen gdt for now; no urgent need to change it */

	pv_info.kernel_rpl = 1;
#1 lore raw
Quoted 8 lines · Jeremy Fitzhardinge

I'd like to add a mechanism like this so I can dynamically allocate some Xen-related structures, rather than statically allocating them in the bss, both so that Xen has less overhead when it isn't being used, and so I can scale better to things like memory size.

I think this is more widely useful; it would supplant dmi_alloc_data[], for example, and I'm sure there's other cases.

Quoted 8 lines

This is fundimentally the same as head_32.S's extension of the bss to build the initial kernel mapping, but 64-bit doesn't currently do anything analogous
to this.

Unfortunately when I use this code as-is I'm getting crashes when the slab allocator starts up. I think this is all correct, but I'm wondering if there's something I'm overlooking which is broken in principle.

First of all, I like the concept. As far as getting crashes, I suspect what you're finding is some use of this type of extended memory space that just isn't documented. I would try this in a simulator, setting a watchpoint on _end to see if you get any hits before the slab allocator starts.

-hpa

Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.
#2 lore raw
In reply to Jeremy Fitzhardinge
Quoted 4 lines · Jeremy Fitzhardinge

unsigned int boot_cpu_id __read_mostly;
+__initdata unsigned long _brk_start, _brk_end;

Better yet, initialize _brk_start and _brk_end statically:

extern const char _end[];
__initdata unsigned long _brk_start = (unsigned long)&_end;
__initdata unsigned long _brk_end = (unsigned long)&_end;

That way it's available from the first instruction, and we can fully remove the x86-32 init_pg_tables_end and replace it with _brk_end (thus putting the initial page tables in the brk.)

-hpa

Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.
#3 lore raw
Quoted 9 lines · H. Peter Anvin
Quoted 4 lines · Jeremy Fitzhardinge

unsigned int boot_cpu_id __read_mostly;
+__initdata unsigned long _brk_start, _brk_end;

Better yet, initialize _brk_start and _brk_end statically:

Quoted 7 lines

extern const char _end[];
__initdata unsigned long _brk_start = (unsigned long)&_end;
__initdata unsigned long _brk_end = (unsigned long)&_end;

That way it's available from the first instruction, and we can fully remove the x86-32 init_pg_tables_end and replace it with _brk_end (thus putting the initial page tables in the brk.)

Yes. It doesn't work for Xen (the domain builder puts the initrd and Xen-built pagetable immediately after the kernel's bss), but its no problem to move the brk in that case.

J

#4 lore raw
In reply to Jeremy Fitzhardinge
Quoted 130 lines · Jeremy Fitzhardinge

[
I'd like to add a mechanism like this so I can dynamically allocate some Xen-related structures, rather than statically allocating them in the bss, both so that Xen has less overhead when it isn't being used, and so I can scale better to things like memory size.

I think this is more widely useful; it would supplant dmi_alloc_data[], for example, and I'm sure there's other cases.

This is fundimentally the same as head_32.S's extension of the bss to build the initial kernel mapping, but 64-bit doesn't currently do anything analogous
to this.

Unfortunately when I use this code as-is I'm getting crashes when the slab allocator starts up. I think this is all correct, but I'm wondering if there's something I'm overlooking which is broken in principle.

So, what am I missing?

Thanks,
J
]

Add a brk()-like allocator which effectively extends the bss in order to allow very early code to do dynamic allocations. This is better than using statically allocated arrays for data in subsystems which may never get used.

The amount of space available depends on how much the initial kernel mappings have covered, and so is fairly limited.

arch/x86/include/asm/setup.h +5 -0
index 66801cb..e6b754b 100644
@@ -99,6 +99,11 @@ extern struct boot_params boot_params;
 */
 #define LOWMEMSIZE()    (0x9f000)
 
+/* exceedingly early brk-like allocator */
+extern unsigned long _brk_start, _brk_end;
+void init_brk(unsigned long start);
+void *extend_brk(size_t size, size_t align);
+
 #ifdef __i386__
 
 void __init i386_start_kernel(void);
arch/x86/kernel/head32.c +2 -0
index ac108d1..fa9ae31 100644
@@ -34,6 +34,8 @@ void __init i386_start_kernel(void)
 
    reserve_ebda_region();
 
+    init_brk((unsigned long)__va(init_pg_tables_end));
+
    /*
     * At this point everything still needed from the boot loader
     * or BIOS or kernel text should be early reserved or marked not
arch/x86/kernel/head64.c +2 -0
index f5b2722..4b29802 100644
@@ -91,6 +91,8 @@ void __init x86_64_start_kernel(char * real_mode_data)
    if (console_loglevel == 10)
        early_printk("Kernel alive\n");
 
+    init_brk((unsigned long)&_end);
+
    x86_64_start_reservations(real_mode_data);
 }
 
arch/x86/kernel/setup.c +22 -5
index 0d051b4..8899cfa 100644
@@ -113,6 +113,7 @@
 #endif
 
 unsigned int boot_cpu_id __read_mostly;
+__initdata unsigned long _brk_start, _brk_end;
 
 #ifdef CONFIG_X86_64
 int default_cpu_present_to_apicid(int mps_cpu)
@@ -335,6 +336,26 @@ static void __init relocate_initrd(void)
 }
 #endif
 
+void __init init_brk(unsigned long brk)
+{
+    _brk_start = _brk_end = brk;
+}
+
+void * __init extend_brk(size_t size, size_t align)
+{
+    size_t mask = align - 1;
+    void *ret;
+
+    BUG_ON(align & mask);
+
+    _brk_end = (_brk_end + mask) & ~mask;
+
+    ret = (void *)_brk_end;
+    _brk_end += size;
+
+    return ret;
+}
+
 static void __init reserve_initrd(void)
 {
    u64 ramdisk_image = boot_params.hdr.ramdisk_image;
@@ -727,11 +748,7 @@ void __init setup_arch(char **cmdline_p)
    init_mm.start_code = (unsigned long) _text;
    init_mm.end_code = (unsigned long) _etext;
    init_mm.end_data = (unsigned long) _edata;
-#ifdef CONFIG_X86_32
-    init_mm.brk = init_pg_tables_end + PAGE_OFFSET;
-#else
-    init_mm.brk = (unsigned long) &_end;
-#endif
+    init_mm.brk = _brk_end;
 
    code_resource.start = virt_to_phys(_text);
    code_resource.end = virt_to_phys(_etext)-1;

arch/x86/kernel/setup.c void __init setup_arch(char **cmdline_p) · line 920

Quoted 7 lines
arch/x86/kernel/setup.c +3 -0
@@ -897,6 +914,9 @@ void __init setup_arch(char **cmdline_p)
    acpi_numa_init();
 #endif
 
+    if (_brk_end > _brk_start)
+        reserve_early(__pa(_brk_start), __pa(_brk_end), "BRK");
+
    initmem_init(0, max_pfn);

it seems reserve _brk_end is some late?

init_memory_mapping(0,...) could get some for direct mapping page table. and it could start from _end...

YH

#5 lore raw
Quoted 4 lines · Yinghai Lu

it seems reserve _brk_end is some late?

init_memory_mapping(0,...) could get some for direct mapping page table. and it could start from _end...

Ah, thanks! I was confused by the use of reserve_early() in initmem_init(), but they're OK because they allocate out of the e820 map at that point (and init_memory_mapping() reserves everything it allocates).

I'll give it a whirl.

J

#6 lore raw
In reply to Yinghai Lu
Quoted 5 lines · Yinghai Lu

it seems reserve _brk_end is some late?

init_memory_mapping(0,...) could get some for direct mapping page table. and it could start from _end...

That would make sense. init_memory_mapping() needs to know what not to step on, and that would include the brk at this stage. For 64-bit mode, Xen is the only user of !PSE, and so may be particularly vulnerable to this issue (because of massively larger direct mapping tables.)

This reminds me... is your intent that the BRK is permanent (unless explicitly freed on a page by page basis) or part of the init memory that is flushed?

-hpa

Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.
#7 lore raw
Quoted 5 lines · H. Peter Anvin

That would make sense. init_memory_mapping() needs to know what not to step on, and that would include the brk at this stage. For 64-bit mode, Xen is the only user of !PSE, and so may be particularly vulnerable to this issue (because of massively larger direct mapping tables.)

Yep.

Quoted 4 lines

This reminds me... is your intent that the BRK is permanent (unless explicitly freed on a page by page basis) or part of the init memory that is flushed?

Permanent. I'm using it to allocate things which are the moral equivalent of the kernel pagetables.

J

#8 lore raw
Quoted 8 lines · Jeremy Fitzhardinge
Quoted 4 lines

This reminds me... is your intent that the BRK is permanent (unless explicitly freed on a page by page basis) or part of the init memory that is flushed?

Permanent. I'm using it to allocate things which are the moral equivalent of the kernel pagetables.

Cool; just wanted it noted what the rules were. Even more so the reason to fold as many possible uses as possible into this new mechanism.

-hpa

P.S. Could you send me the updates to the dom0 patches you said you had?

Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.
#9 lore raw
Quoted 3 lines · H. Peter Anvin

Cool; just wanted it noted what the rules were. Even more so the reason to fold as many possible uses as possible into this new mechanism.

Yes, I just converted i386 pagetable construction and dmi_alloc(). Boots under qemu.

Quoted 2 lines

P.S. Could you send me the updates to the dom0 patches you said you had?

Was just tidying them up for posting but got slightly sidetracked. (The TSS/io-bitmap changes are a prelude to the dom0 patches though.)

J