scheduler fixes
Subsystems (8)
Linus,
Please pull the latest sched-fixes-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip.git sched-fixes-for-linus
Thanks,
Ingo
------------------>
Ron (1):
sched: Fix fallback sched_clock()'s offset when using jiffies
Rusty Russell (1):
sched: avoid flexible array member inside struct (gcc extension)
kernel/sched.c | 28 +++++++++++++++------------- kernel/sched_clock.c | 3 ++- 2 files changed, 17 insertions(+), 14 deletions(-)
kernel/sched.c +15 -13
@@ -7756,22 +7756,24 @@ int sched_smt_power_savings = 0, sched_mc_power_savings = 0;* FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space * for nr_cpu_ids < CONFIG_NR_CPUS. */ -struct static_sched_group { +union static_sched_group {struct sched_group sg; - DECLARE_BITMAP(cpus, CONFIG_NR_CPUS); +char _sg_and_cpus[sizeof (struct sched_group ) + +BITS_TO_LONGS (CONFIG_NR_CPUS ) *sizeof (long )]; }; -struct static_sched_domain { +union static_sched_domain {struct sched_domain sd; - DECLARE_BITMAP(span, CONFIG_NR_CPUS); +char _sd_and_cpus[sizeof (struct sched_domain ) + +BITS_TO_LONGS (CONFIG_NR_CPUS ) *sizeof (long )]; };/* * SMT sched-domains: */ #ifdef CONFIG_SCHED_SMT -static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains); -static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus); +static DEFINE_PER_CPU(union static_sched_domain, cpu_domains); +static DEFINE_PER_CPU(union static_sched_group, sched_group_cpus); static int cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
@@ -7787,8 +7789,8 @@ cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,* multi-core sched-domains: */ #ifdef CONFIG_SCHED_MC -static DEFINE_PER_CPU(struct static_sched_domain, core_domains); -static DEFINE_PER_CPU(struct static_sched_group, sched_group_core); +static DEFINE_PER_CPU(union static_sched_domain, core_domains); +static DEFINE_PER_CPU(union static_sched_group, sched_group_core);#endif /* CONFIG_SCHED_MC */ #if defined(CONFIG_SCHED_MC ) && defined(CONFIG_SCHED_SMT )
@@ -7815,8 +7817,8 @@ cpu_to_core_group(int cpu, const struct cpumask *cpu_map, }#endif -static DEFINE_PER_CPU(struct static_sched_domain, phys_domains); -static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys); +static DEFINE_PER_CPU(union static_sched_domain, phys_domains); +static DEFINE_PER_CPU(union static_sched_group, sched_group_phys); static int cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
@@ -7843,11 +7845,11 @@ cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,* groups, so roll our own. Now each node has its own list of groups which * gets dynamically allocated. */ -static DEFINE_PER_CPU(struct static_sched_domain, node_domains); +static DEFINE_PER_CPU(union static_sched_domain, node_domains);static struct sched_group ***sched_group_nodes_bycpu; -static DEFINE_PER_CPU(struct static_sched_domain, allnodes_domains); -static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes); +static DEFINE_PER_CPU(union static_sched_domain, allnodes_domains); +static DEFINE_PER_CPU(union static_sched_group, sched_group_allnodes); static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map, struct sched_group **sg,
kernel/sched_clock.c +2 -1
@@ -38,7 +38,8 @@*/ unsigned long long __attribute__((weak))sched_clock (void ) { -return (unsigned long long )jiffies * (NSEC_PER_SEC /HZ ); +return (unsigned long long )(jiffies -INITIAL_JIFFIES ) + * (NSEC_PER_SEC /HZ ); }static __read_mostly int sched_clock_running;
Quoted 3 lines · Ingo Molnar
Rusty Russell (1):
sched: avoid flexible array member inside struct (gcc extension)
I'm not pulling this one either.
It makes no sense what-so-ever. It's uglier code, so calling it a cleanup is just wrong.
Now apart from being ugly and pointless, it is also FUNDAMENTALLY INCORRECT.
It is in no way true that "a union is the Right Way to do this", especially not the way THAT PIECE OF UTTER CRAP does it.
This is the original data structure:
struct static_sched_group {
struct sched_group sg;
DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
};and it is fine (the fact that "sg" isn't fine is a totally different issue).
The new one:
union static_sched_group {
struct sched_group sg;
char _sg_and_cpus[sizeof(struct sched_group) +
BITS_TO_LONGS(CONFIG_NR_CPUS) * sizeof(long)];
};claimed to be a "cleanup" (hah - what a f*cking joke! Anybody looking at it for half a second would see that that is a clear lie) is just a horrible bug waiting to happen.
You can't do that. Doing a character array with "sizeof" IS NOT VALID. It doesn't take things like different alignment into account. It might _work_, but it is still UTTER SH*T.
Yes, I'm upset. It's -rc6 and now two "please pull" requests have been totally unacceptable in very fundamental and obvious forms.
I'm also upset becasue that obvious PIECE OF CRAP got two Acked-by's from people who should know better.
If you wan tto fix this up, then just fix "struct sched_group sg" instead. Here's a suggested better fix, but I'd suggest somebody also write a honking big COMMENT in addition to this.
But note how simple this attached patch is? Note how it's not adding totally ugly and horrible code? THIS is a fix (and yes, zero-sized arrays are a gcc extensiontoo, but we've used them a lot)
I'm sure there are other ways to fix it too, and I'm open to them, but that union with character arrays etc I'm not open to.
Linus
--- include/linux/sched.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)
include/linux/sched.h +1 -1
@@ -838,7 +838,7 @@ struct sched_group {*/ u32 reciprocal_cpu_power; -unsigned long cpumask[]; +unsigned long cpumask[0 ]; };static inline struct cpumask *sched_group_cpus (struct sched_group *sg)
Quoted 9 lines · Linus Torvalds
Quoted 3 lines · Ingo Molnar
Rusty Russell (1):
sched: avoid flexible array member inside struct (gcc extension)I'm not pulling this one either.
It makes no sense what-so-ever. It's uglier code, so calling it a cleanup is just wrong.
hm - i've Cc:-ed Jeff & Viro. The background is that Sparse and LLVM barfed on the current construct and Al strongly advocated this solution, see:
"[RFC PATCH 2/2] kernel/sched.c: VLA in middle of struct"
See that particular reply below.
Ingo
----- Forwarded message from Al Viro <[email protected]> -----
Date: Tue, 12 May 2009 15:03:44 +0100
From: Al Viro <[email protected]>
To: Rusty Russell <[email protected]>
Subject: Re: [RFC PATCH 2/2] kernel/sched.c: VLA in middle of struct
Cc: Jeff Garzik <[email protected]>, Ingo Molnar <[email protected]>,
Peter Zijlstra <[email protected]>,
Mike Travis <[email protected]>, LKML <[email protected]>, Andrew Morton <[email protected]>, [email protected]
Quoted 19 lines · Rusty Russell
Quoted 10 lines · Jeff Garzik
Quoted 8 lines · Rusty Russell
Yeah, it's kinda nasty. Generally,
sched_groupis dynamically allocated, so we just allocatesizeof(struct sched_group)+ size ofnr_cpu_idsbits.These ones are static, and it was easier to put this hack in than make them dynamic. There's nothing wrong with it, until we really want
NR_CPUS== bignum, or we want to get rid ofNR_CPUSaltogether forCONFIG_CPUMASKS_OFFSTACK(which would be very clean, but not clearly worthwhile).Quoted 7 lines
Nothing wrong with it, except
- C99 only defines variable-length automatic arrays
- VLA in the middle of a struct are difficult to optimize
- gcc's VLA handling WILL change, as gcc docs state
- other compilers -- and sparse -- puke all over VLAs, making static analysis impossible for all code with this weirdism
Quoted 5 lines
Jeff, you seem confused. In my copy of the standard, you'd know this is called a "flexible array member"; it's not a variable length array. The only GCC specific issue I can find here is that you're not normally allowed to embed structs with them in another struct (according to the gcc docs; I can't actually find this clearly stated in the standard).
6.7.2.1p2. It's a separate issue from revolting gcc extension that *do* allow VLA-in-the-middle-of-struct. And I mean real VLA, not flex array member :-/
Quoted 3 lines
Anyway, since [] is C99, I thought it preferable to [0] which is a gcc extension. However, if C99 is really so braindead as to disallow this fairly standard trick, so I'm happy to go with the gcc extension.[1]
No. There's a standard way to do that in C99; you can put a struct with that thing into a union. So correct way to force enought storage for such an object is
union {
struct has_flex_array_member foo;
char [how_much_space_do_I_want];
} bar;Unions with overlapping members are fine. Structures are not.
----- End forwarded message -----
Quoted 4 lines · Ingo Molnar
hm - i've Cc:-ed Jeff & Viro. The background is that Sparse and LLVM barfed on the current construct and Al strongly advocated this solution, see:
I know the background.
Did you read my email?
Did you see my one-line patch that fixes the same problem WITHOUT THE INSANITY?
Yes, Al is mostly right. In this case he is wrong. (Ab-)Using unions for something like this is crazy, since the code doesn't want a union of overlapping data, it wants two consecutive data structures.
And I'm not saying that my one-liner is necessarily the only way to fix it. But it's a _better_ way than the crazy way you merged.
Linus
Quoted 6 lines · Linus Torvalds
Quoted 4 lines · Ingo Molnar
hm - i've Cc:-ed Jeff & Viro. The background is that Sparse and LLVM barfed on the current construct and Al strongly advocated this solution, see:
Quoted 6 lines
I know the background.
Did you read my email?
Did you see my one-line patch that fixes the same problem WITHOUT THE INSANITY?
Yes, i even proposed that very patch in the original discussion, in my first reply:
http://lkml.org/lkml/2009/5/8/378
but then got sidetracked by the 'this is C-correct' claim. Should have noticed the bogosity, sorry about that.
Ingo
Quoted 12 lines · Ingo Molnar
Quoted 10 lines · Linus Torvalds
Quoted 4 lines · Ingo Molnar
hm - i've Cc:-ed Jeff & Viro. The background is that Sparse and LLVM barfed on the current construct and Al strongly advocated this solution, see:
I know the background.
Did you read my email?
Quoted 7 lines
Quoted 2 lines
Did you see my one-line patch that fixes the same problem WITHOUT THE INSANITY?
Yes, i even proposed that very patch in the original discussion, in my first reply:
http://lkml.org/lkml/2009/5/8/378
Something like the patch below. It also fixes ->span[] which has a similar problem.
But ... i think this needs further clean-ups really. Either go fully static, or go fully dynamic.
I'd suggest we go fully dynamic: the static structures are never used directly anyway, they are pointer-ized during sched domain setup.
The reason for this duality is allocation bootstrap: there's no generic early-capable allocator that allocates something and switches from bootmem to kmalloc transparently once the SLAB has been set up.
Would be nice if bootmem_alloc() was extended with such properties - if SLAB is up (and bootmem is down) it would return kmalloc(GFP_KERNEL) memory buffers.
Ingo
-------------->
Subject: sched: properly define the sched_group::cpumask and sched_domain::span fields
From: Ingo Molnar <[email protected]>
Properly document the variable-size structure tricks we are doing wrt. struct sched_group and sched_domain, and use the field[0] GCC extension instead of defining a vla array.
Dont use unions for this, as pointed out by Linus.
This also un-confuses Sparse and LLVM.
LKML-Reference: <[email protected]>
--- include/linux/sched.h | 25 ++++++++++++++++++++++--- kernel/sched.c | 5 +++-- 2 files changed, 25 insertions(+), 5 deletions(-)
include/linux/sched.h +22 -3
@@ -846,7 +846,17 @@ struct sched_group {*/ u32 reciprocal_cpu_power; -unsigned long cpumask[]; +/* +* The CPUs this group covers. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +* +* It is also be embedded into static data structures at build +* time. (See 'struct static_sched_group' in kernel/sched.c) +*/ +unsigned long cpumask[0 ]; };static inline struct cpumask *sched_group_cpus (struct sched_group *sg)
@@ -932,8 +942,17 @@ struct sched_domain {char *name;#endif -/* span of all CPUs in this domain */ -unsigned long span[]; +/* +* Span of all CPUs in this domain. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +* +* It is also be embedded into static data structures at build +* time. (See 'struct static_sched_domain' in kernel/sched.c) +*/ +unsigned long span[0 ]; };static inline struct cpumask *sched_domain_span (struct sched_domain *sd)
kernel/sched.c +3 -2
@@ -8049,8 +8049,9 @@ int sched_smt_power_savings = 0, sched_m/* * The cpus mask in sched_group and sched_domain hangs off the end. -* FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space -* for nr_cpu_ids < CONFIG_NR_CPUS. +* +* ( See the the comments in include/linux/sched.h:struct sched_group +* and struct sched_domain. ) */ struct static_sched_group {struct sched_group sg;
Quoted 3 lines · Ingo Molnar
Something like the patch below. It also fixes ->span[] which has a similar problem.
Patch looks good to me.
Quoted 2 lines
But ... i think this needs further clean-ups really. Either go fully static, or go fully dynamic.
I do agree that it would probably be good to try to avoid this static allocation, and allocate these data structures dynamically. However, if we end up having to use two different allocators anyway (one for bootup, and one for regular uptimes), then I think that would be an overall loss (compared to just the simplicity of statically doing this in a couple of places), rather than an overall win.
Quoted 3 lines
Would be nice if
bootmem_alloc()was extended with such properties - if SLAB is up (and bootmem is down) it would returnkmalloc(GFP_KERNEL)memory buffers.
I would rather say the other way around: no "bootmem_alloc()" at all, but just have a regular alloc() that ends up working like the "SMP alternatives" code, but instead of being about SMP, it would be about how early in the boot sequence it is.
That said, if there are just a couple of places like this that care, I don't think it's worth it. The static allocation isn't that horrible. I'd rather have a few ugly static allocations with comments about _why_ they look the way they do, than try to over-design things to look "clean".
Simplicity is a good thing - even if it can then end up meaning special cases like this.
That said, if we could move the kmalloc initialization up some more (and get at least the "boot node" data structures set up, and avoid any bootmem alloc issues _entirely_, then that would be good.
I hate that stupid bootmem allocator. I suspect we seriously over-use it, and that we _should_ be able to do the SL*B init earlier.
Linus
Quoted 6 lines · Linus Torvalds
Quoted 3 lines · Ingo Molnar
Something like the patch below. It also fixes ->span[] which has a similar problem.
Patch looks good to me.
ok. I've queued it up for .31, with your Acked-by. (which i assume your reply implies?)
Quoted 29 lines
Quoted 2 lines
But ... i think this needs further clean-ups really. Either go fully static, or go fully dynamic.
I do agree that it would probably be good to try to avoid this static allocation, and allocate these data structures dynamically. However, if we end up having to use two different allocators anyway (one for bootup, and one for regular uptimes), then I think that would be an overall loss (compared to just the simplicity of statically doing this in a couple of places), rather than an overall win.
Quoted 3 lines
Would be nice if
bootmem_alloc()was extended with such properties - if SLAB is up (and bootmem is down) it would returnkmalloc(GFP_KERNEL)memory buffers.I would rather say the other way around: no "
bootmem_alloc()" at all, but just have a regularalloc()that ends up working like the "SMP alternatives" code, but instead of being about SMP, it would be about how early in the boot sequence it is.That said, if there are just a couple of places like this that care, I don't think it's worth it. The static allocation isn't that horrible. I'd rather have a few ugly static allocations with comments about _why_ they look the way they do, than try to over-design things to look "clean".
Simplicity is a good thing - even if it can then end up meaning special cases like this.
Quoted 8 lines
That said, if we could move the kmalloc initialization up some more (and get at least the "boot node" data structures set up, and avoid any bootmem alloc issues _entirely_, then that would be good.
I hate that stupid bootmem allocator. I suspect we seriously over-use it, and that we _should_ be able to do the SL*B init earlier.
Hm, tempting thought - not sure how to pull it off though.
One of the biggest user of bootmem is the mem_map[] hierarchies and the page allocator bitmaps. Not sure we can get rid of bootmem there
- those areas are really large, physical memory is often fragmented and we need a good NUMA sense for them as well.
We might also have a 22-architectures-to-fix problem as well, before we can get rid of bootmem:
$ git grep alloc_bootmem arch/ | wc -l 168
On x86 we recently switched some (but not all) early-pagetable allocations to the 'early brk' method (which is an utterly simple early linear allocator, for limited early dynamic allocations), but even with that we still have ugly bootmem use - for example see the after_bootmem hacks in arch/x86/mm/init_64.c.
So we have these increasingly more complete layers of allocators, which bootstrap each other gradually:
- static, build-time allocations
- early-brk (see extend_brk(), RESERVE_BRK and direct use of _brk_end in assembly code)
- e820 based early allocator (reserve_early()) to bootstrap bootmem
- bootmem - to bootstrap the page allocator [NUMA aware]
- page allocator - to bootstrap SLAB
- SLAB
that's 5 layers until we get to SLAB. Each layer has to be aware of its own limits, has to interact with pagetable setup and has to end up with a NUMA-aware dynamic allocations as early as possible.
And all this complexity definitely _feels_ utterly wrong, as we really know it pretty early on what kind of memory we have, how it's laid out amongst nodes. In the end we really just want to have the page allocator and SL[AOQU]B.
Looks daunting.
Ingo
Quoted 3 lines · Ingo Molnar
ok. I've queued it up for .31, with your Acked-by. (which i assume your reply implies?)
Yes.
Quoted 5 lines
Quoted 3 lines
I hate that stupid bootmem allocator. I suspect we seriously over-use it, and that we _should_ be able to do the SL*B init earlier.
Hm, tempting thought - not sure how to pull it off though.
As far as I can recall, one of the things that historically made us want to use the bootmem allocator even relatively late was that the real SLAB allocator had to wait until all the node information etc was initialized.
That's pretty damn late. And I wonder if SLUB (and SLOB) might not need a lot less initialization, and work much earlier. Something like that might be the final nail in the coffin for SLAB, and convince me to just say 'we don't support it any more".
That said, for the case of things like 'static_sched_group' and 'static_sched_domain', the problem might well be not just about the allocation itself, but simply about the use of those variables. Maybe they themselves are needed before we've done all the CPU setup?
Quoted 4 lines
One of the biggest user of bootmem is the mem_map[] hierarchies and the page allocator bitmaps. Not sure we can get rid of bootmem there
- those areas are really large, physical memory is often fragmented and we need a good NUMA sense for them as well.
I think that's the _valid_ kind of use of a bootmem allocator.
But for something like the scheduler data structures? Not so much.
Linus
Quoted 8 lines · Linus Torvalds
Quoted 4 lines
One of the biggest user of bootmem is the mem_map[] hierarchies and the page allocator bitmaps. Not sure we can get rid of bootmem there
- those areas are really large, physical memory is often fragmented and we need a good NUMA sense for them as well.I think that's the _valid_ kind of use of a bootmem allocator.
But for something like the scheduler data structures? Not so much.
Yeah, and it feels dirty to use slab_is_available() to figure if the code should kmalloc or alloc_bootmem.
Ideally kmalloc/kfree would "always work". But at least we could get closer.
Thanks,
Rusty.
Quoted 6 lines · Linus Torvalds
Quoted 5 lines
Quoted 3 lines
I hate that stupid bootmem allocator. I suspect we seriously over-use it, and that we _should_ be able to do the SL*B init earlier.
Hm, tempting thought - not sure how to pull it off though.
Quoted 8 lines
As far as I can recall, one of the things that historically made us want to use the bootmem allocator even relatively late was that the real SLAB allocator had to wait until all the node information etc was initialized.
That's pretty damn late. And I wonder if SLUB (and SLOB) might not need a lot less initialization, and work much earlier. Something like that might be the final nail in the coffin for SLAB, and convince me to just say 'we don't support it any more".
Ingo, here's a patch that boots UMA+SMP+SLUB x86-64 kernel on qemu all the way to userspace. It probably breaks bunch of things for now but something for you to play with if you want.
Pekka
init/main.c +16 -12
@@ -575,6 +575,22 @@ asmlinkage void __init start_kernel(void)setup_nr_cpu_ids ();smp_prepare_boot_cpu ();/* arch-specific boot-cpu hooks */ +build_all_zonelists (); +page_alloc_init (); + +printk (KERN_NOTICE "Kernel command line: %s\n" , boot_command_line); +parse_early_param (); +parse_args ("Booting kernel" , static_command_line, __start___param, + __stop___param - __start___param, + &unknown_bootoption); +/* +* Setup kernel memory allocators +*/ +pidhash_init (); +vmalloc_init (); +vfs_caches_init_early (); +mem_init (); +kmem_cache_init ();/* * Set up the scheduler prior starting any interrupts (such as the * timer interrupt). Full topology setup happens at smp_init()
@@ -586,13 +602,6 @@ asmlinkage void __init start_kernel(void)* fragile until we cpu_idle() for the first time. */ preempt_disable (); -build_all_zonelists (); -page_alloc_init (); -printk (KERN_NOTICE "Kernel command line: %s\n" , boot_command_line); -parse_early_param (); -parse_args ("Booting kernel" , static_command_line, __start___param, - __stop___param - __start___param, - &unknown_bootoption);if (!irqs_disabled ()) {printk (KERN_WARNING "start_kernel(): bug: interrupts were " "enabled *very* early, fixing it\n" );
@@ -604,7 +613,6 @@ asmlinkage void __init start_kernel(void)/* init some links before init_ISA_irqs() */ early_irq_init ();init_IRQ (); -pidhash_init ();init_timers ();hrtimers_init ();softirq_init ();
@@ -646,14 +654,10 @@ asmlinkage void __init start_kernel(void) initrd_start = 0; }#endif -vmalloc_init (); -vfs_caches_init_early ();cpuset_init_early ();page_cgroup_init (); -mem_init ();enable_debug_pagealloc ();cpu_hotplug_init (); -kmem_cache_init ();kmemtrace_init ();debug_objects_mem_init ();idr_init_cache ();
kernel/irq/handle.c +5 -5
@@ -18,7 +18,7 @@#include <linux/rculist.h> #include <linux/hash.h> #include <trace/irq.h> -#include <linux/bootmem.h> +#include <linux/slab.h> #include "internals.h"
@@ -44,7 +44,7 @@ void handle_bad_irq(unsigned int irq, struct irq_desc *desc)#if defined(CONFIG_SMP ) && defined(CONFIG_GENERIC_HARDIRQS )static void __init init_irq_default_affinity (void ) { -alloc_bootmem_cpumask_var (&irq_default_affinity); +alloc_cpumask_var (&irq_default_affinity,GFP_NOWAIT );cpumask_setall (irq_default_affinity); }#else
@@ -158,12 +158,12 @@ int __init early_irq_init(void) legacy_count =ARRAY_SIZE (irq_desc_legacy);/* allocate irq_desc_ptrs array based on nr_irqs */ - irq_desc_ptrs =alloc_bootmem (nr_irqs *sizeof (void *)); + irq_desc_ptrs =kcalloc (nr_irqs,sizeof (void *),GFP_NOWAIT );/* allocate based on nr_cpu_ids */ /* FIXME: invert kstat_irgs, and it'd be a per_cpu_alloc'd thing */ - kstat_irqs_legacy =alloc_bootmem (NR_IRQS_LEGACY * nr_cpu_ids * -sizeof (int )); + kstat_irqs_legacy =kzalloc (NR_IRQS_LEGACY * nr_cpu_ids * +sizeof (int ),GFP_NOWAIT );for (i =0 ; i < legacy_count; i++) { desc[i].irq = i;
kernel/sched.c +11 -19
@@ -68,7 +68,6 @@#include <linux/pagemap.h> #include <linux/hrtimer.h> #include <linux/tick.h> -#include <linux/bootmem.h> #include <linux/debugfs.h> #include <linux/ctype.h> #include <linux/ftrace.h>
@@ -7525,21 +7524,18 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd) static int __init_refok init_rootdomain(struct root_domain *rd, bool bootmem) { +gfp_t gfp =GFP_KERNEL ; +memset (rd,0 ,sizeof (*rd)); -if (bootmem) { -alloc_bootmem_cpumask_var (&def_root_domain.span); -alloc_bootmem_cpumask_var (&def_root_domain.online); -alloc_bootmem_cpumask_var (&def_root_domain.rto_mask); -cpupri_init (&rd->cpupri, true); -return 0 ; - } +if (bootmem) + gfp =GFP_NOWAIT ; -if (!alloc_cpumask_var (&rd->span,GFP_KERNEL )) +if (!alloc_cpumask_var (&rd->span, gfp)) goto out; -if (!alloc_cpumask_var (&rd->online,GFP_KERNEL )) +if (!alloc_cpumask_var (&rd->online, gfp)) goto free_span; -if (!alloc_cpumask_var (&rd->rto_mask,GFP_KERNEL )) +if (!alloc_cpumask_var (&rd->rto_mask, gfp)) goto free_online;if (cpupri_init (&rd->cpupri, false) !=0 )
@@ -8860,12 +8856,8 @@ void __init sched_init(void)#ifdef CONFIG_CPUMASK_OFFSTACK alloc_size +=num_possible_cpus () *cpumask_size ();#endif -/* -* As sched_init() is called before page_alloc is setup, -* we use alloc_bootmem(). -*/ if (alloc_size) { - ptr = (unsigned long )alloc_bootmem (alloc_size); + ptr = (unsigned long )kzalloc (alloc_size,GFP_NOWAIT );#ifdef CONFIG_FAIR_GROUP_SCHED init_task_group.se = (struct sched_entity **)ptr;
@@ -9051,12 +9043,12 @@ void __init sched_init(void) current->sched_class = &fair_sched_class;/* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */ -alloc_bootmem_cpumask_var (&nohz_cpu_mask); +alloc_cpumask_var (&nohz_cpu_mask,GFP_NOWAIT );#ifdef CONFIG_SMP #ifdef CONFIG_NO_HZ -alloc_bootmem_cpumask_var (&nohz.cpu_mask); +alloc_cpumask_var (&nohz.cpu_mask,GFP_NOWAIT );#endif -alloc_bootmem_cpumask_var (&cpu_isolated_map); +alloc_cpumask_var (&cpu_isolated_map,GFP_NOWAIT );#endif /* SMP */ scheduler_running =1 ;
mm/slub.c +10 -7
@@ -2557,13 +2557,16 @@ static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s,if (gfp_flags &SLUB_DMA ) flags =SLAB_CACHE_DMA ; -down_write (&slub_lock); +/* +* This function is called with IRQs disabled during early-boot on +* single CPU so there's no need to take slub_lock here. +*/ if (!kmem_cache_open (s, gfp_flags, name, size,ARCH_KMALLOC_MINALIGN , flags,NULL )) goto panic;list_add (&s->list, &slab_caches); -up_write (&slub_lock); +if (sysfs_slab_add (s)) goto panic;return s;
@@ -3021,7 +3024,7 @@ void __init kmem_cache_init(void)* kmem_cache_open for slab_state == DOWN. */ create_kmalloc_cache (&kmalloc_caches[0 ],"kmem_cache_node" , -sizeof (struct kmem_cache_node ),GFP_KERNEL ); +sizeof (struct kmem_cache_node ),GFP_NOWAIT ); kmalloc_caches[0 ].refcount =-1 ; caches++;
@@ -3034,16 +3037,16 @@ void __init kmem_cache_init(void)/* Caches that are not of the two-to-the-power-of size */ if (KMALLOC_MIN_SIZE <=64 ) {create_kmalloc_cache (&kmalloc_caches[1 ], -"kmalloc-96" ,96 ,GFP_KERNEL ); +"kmalloc-96" ,96 ,GFP_NOWAIT ); caches++;create_kmalloc_cache (&kmalloc_caches[2 ], -"kmalloc-192" ,192 ,GFP_KERNEL ); +"kmalloc-192" ,192 ,GFP_NOWAIT ); caches++; }for (i =KMALLOC_SHIFT_LOW ; i <SLUB_PAGE_SHIFT ; i++) {create_kmalloc_cache (&kmalloc_caches[i], -"kmalloc" ,1 << i,GFP_KERNEL ); +"kmalloc" ,1 << i,GFP_NOWAIT ); caches++; }
@@ -3080,7 +3083,7 @@ void __init kmem_cache_init(void)/* Provide the correct kmalloc names now that the caches are up */ for (i =KMALLOC_SHIFT_LOW ; i <SLUB_PAGE_SHIFT ; i++) kmalloc_caches[i]. name = -kasprintf (GFP_KERNEL ,"kmalloc-%d" ,1 << i); +kasprintf (GFP_NOWAIT ,"kmalloc-%d" ,1 << i);#ifdef CONFIG_SMP register_cpu_notifier (&slab_notifier);
Quoted 4 lines · Pekka J Enberg
Ingo, here's a patch that boots UMA+SMP+SLUB x86-64 kernel on qemu all the way to userspace. It probably breaks bunch of things for now but something for you to play with if you want.
Just looking at the patch (not actually trying it out in any way or looking at some bigger context), this absolutely looks like the right direction to go in.
Our order in init/main.c is largely totally historical, and I think it makes tons of sense to move the memory allocation initialization up much earlier.
In fact, it would be nice to perhaps try to move it even earlier. Now you moved it to before the scheduler init (good!), but I do wonder if it could be moved up to even before the setup_per_cpu_areas() etc crud.
I realize that the allocator wants to use the per-CPU area, but if we have just the boot CPU area set up statically at that point, since it's only the boot CPU running, maybe we could do those per-cpu area allocations without the bootmem allocator too?
But even just getting bootmem out of the scheduler setup is a big improvement, I think. So this patch looks very promising as is.
Did you test whether the other allocators were ok with this too?
Linus
Hi Linus,
Quoted 3 lines · Pekka J Enberg
Quoted 3 lines
Ingo, here's a patch that boots UMA+SMP+SLUB x86-64 kernel on qemu all the way to userspace. It probably breaks bunch of things for now but something for you to play with if you want.
Quoted 3 lines · Linus Torvalds
In fact, it would be nice to perhaps try to move it even earlier. Now you moved it to before the scheduler init (good!), but I do wonder if it could be moved up to even before the
setup_per_cpu_areas()etc crud.
Oh, sure, we can look into that. I just wanted to take the conservative approach because I worry about breaking bunch of configurations I cannot test. I suspect it's going to get pretty hairy if we do kmem_cache_init() even earlier. Furthermore, SLUB does sysfs setup in kmem_cache_init() so we probably need to split slab initialization in two stages.
Quoted 4 lines · Linus Torvalds
I realize that the allocator wants to use the per-CPU area, but if we have just the boot CPU area set up statically at that point, since it's only the boot CPU running, maybe we could do those per-cpu area allocations without the bootmem allocator too?
We probably can. I don't see any fundamental reason why slab allocators can't bootstrap early in the boot sequence after we've set up the page allocator.
Quoted 4 lines · Linus Torvalds
But even just getting bootmem out of the scheduler setup is a big improvement, I think. So this patch looks very promising as is.
Did you test whether the other allocators were ok with this too?
SLUB and SLOB are fine but SLAB explodes. I didn't investigate it yet but it's probably because SLAB expects interrupts to be enabled when kmem_cache_init() is called.
Pekka
Quoted 8 lines · Linus Torvalds
In fact, it would be nice to perhaps try to move it even earlier. Now you moved it to before the scheduler init (good!), but I do wonder if it could be moved up to even before the
setup_per_cpu_areas()etc crud.I realize that the allocator wants to use the per-CPU area, but if we have just the boot CPU area set up statically at that point, since it's only the boot CPU running, maybe we could do those per-cpu area allocations without the bootmem allocator too?
Well, we want at least node information since we want per-cpu areas to be allocated on the right node etc...
But then, bootmem has them, so we should be able to feed them off to SL*B early.
One thing I'm wondering... Most archs I see have their own allocator for before bootmem is available even. On PowerPC and Sparc, we call it LMB and it's in fact in generic code now. x86 seems to have several layers but thew e820 early allocator seems to fit a similar bill.
I wonder if we could try to shoot bootmem that way.
With a blend of Pekka's approach which can drastically reduce how much we need bootmem, for the remaining bits such as the SL*B own data structures and the mem_map, the arch is responsible to provide a simple API to provide node local allocations that is roughly equivalent to whatever bits of bootmem remain and are needed.
That API wraps on top of whatever the arch already has for early boot stuff.
Finally, we can keep bootmem around in lib/ or such for archs that don't want to convert or don't have an existing suitable early allocator.
Cheers,
Ben.
Quoted 9 lines · Pekka J Enberg
Quoted 8 lines · Linus Torvalds
Quoted 4 lines
Quoted 3 lines
I hate that stupid bootmem allocator. I suspect we seriously over-use it, and that we _should_ be able to do the SL*B init earlier.
Hm, tempting thought - not sure how to pull it off though.
As far as I can recall, one of the things that historically made us want to use the bootmem allocator even relatively late was that the real SLAB allocator had to wait until all the node information etc was initialized.
Quoted 9 lines
Quoted 4 lines
That's pretty damn late. And I wonder if SLUB (and SLOB) might not need a lot less initialization, and work much earlier. Something like that might be the final nail in the coffin for SLAB, and convince me to just say 'we don't support it any more".
Ingo, here's a patch that boots UMA+SMP+SLUB x86-64 kernel on qemu all the way to userspace. It probably breaks bunch of things for now but something for you to play with if you want.
updated with tip/master. also add change to cpupri_init otherwise will get
[ 0.000000] Memory: 523096612k/537526272k available (10461k kernel code, 656156k absent, 13773504k reserved, 7186k data, 2548k init) [ 0.000000] SLUB: Genslabs=14, HWalign=64, Order=0-3, MinObjects=0, CPUs=32, Nodes=8 [ 0.000000] ------------[ cut here ]------------ [ 0.000000] WARNING: at kernel/lockdep.c:2282 lockdep_trace_alloc+0xaf/0xee() [ 0.000000] Hardware name: Sun Fire X4600 M2 [ 0.000000] Modules linked in: [ 0.000000] Pid: 0, comm: swapper Not tainted 2.6.30-rc6-tip-01778-g0afdd0f-dirty #259 [ 0.000000] Call Trace: [ 0.000000] [<ffffffff810a0274>] ? lockdep_trace_alloc+0xaf/0xee [ 0.000000] [<ffffffff81075ab0>] warn_slowpath_common+0x88/0xcb [ 0.000000] [<ffffffff81075b15>] warn_slowpath_null+0x22/0x38 [ 0.000000] [<ffffffff810a0274>] lockdep_trace_alloc+0xaf/0xee [ 0.000000] [<ffffffff8110301b>] kmem_cache_alloc_node+0x38/0x14d [ 0.000000] [<ffffffff813ec548>] ? alloc_cpumask_var_node+0x4a/0x10a [ 0.000000] [<ffffffff8109eb61>] ? lockdep_init_map+0xb9/0x564 [ 0.000000] [<ffffffff813ec548>] alloc_cpumask_var_node+0x4a/0x10a [ 0.000000] [<ffffffff813ec62c>] alloc_cpumask_var+0x24/0x3a [ 0.000000] [<ffffffff819e6306>] cpupri_init+0x7f/0x112 [ 0.000000] [<ffffffff819e5a30>] init_rootdomain+0x72/0xb7 [ 0.000000] [<ffffffff821facce>] sched_init+0x109/0x660 [ 0.000000] [<ffffffff82203082>] ? kmem_cache_init+0x193/0x1b2 [ 0.000000] [<ffffffff821dfd7a>] start_kernel+0x218/0x3f3 [ 0.000000] [<ffffffff821df2a9>] x86_64_start_reservations+0xb9/0xd4 [ 0.000000] [<ffffffff821df3b2>] x86_64_start_kernel+0xee/0x109 [ 0.000000] ---[ end trace a7919e7f17c0a725 ]---
works with 8 sockets numa amd64 box.
YH
--- init/main.c | 28 ++++++++++++++++------------ kernel/irq/handle.c | 23 ++++++++--------------- kernel/sched.c | 34 +++++++++++++--------------------- kernel/sched_cpupri.c | 9 ++++++--- mm/slub.c | 17 ++++++++++------- 5 files changed, 53 insertions(+), 58 deletions(-)
init/main.c +16 -12
@@ -576,6 +576,22 @@ asmlinkage void __init start_kernel(voidsetup_nr_cpu_ids ();smp_prepare_boot_cpu ();/* arch-specific boot-cpu hooks */ +build_all_zonelists (); +page_alloc_init (); + +printk (KERN_NOTICE "Kernel command line: %s\n" , boot_command_line); +parse_early_param (); +parse_args ("Booting kernel" , static_command_line, __start___param, + __stop___param - __start___param, + &unknown_bootoption); +/* +* Setup kernel memory allocators +*/ +pidhash_init (); +vmalloc_init (); +vfs_caches_init_early (); +mem_init (); +kmem_cache_init ();/* * Set up the scheduler prior starting any interrupts (such as the * timer interrupt). Full topology setup happens at smp_init()
@@ -587,13 +603,6 @@ asmlinkage void __init start_kernel(void* fragile until we cpu_idle() for the first time. */ preempt_disable (); -build_all_zonelists (); -page_alloc_init (); -printk (KERN_NOTICE "Kernel command line: %s\n" , boot_command_line); -parse_early_param (); -parse_args ("Booting kernel" , static_command_line, __start___param, - __stop___param - __start___param, - &unknown_bootoption);if (!irqs_disabled ()) {printk (KERN_WARNING "start_kernel(): bug: interrupts were " "enabled *very* early, fixing it\n" );
@@ -605,7 +614,6 @@ asmlinkage void __init start_kernel(void/* init some links before init_ISA_irqs() */ early_irq_init ();init_IRQ (); -pidhash_init ();init_timers ();hrtimers_init ();softirq_init ();
@@ -647,14 +655,10 @@ asmlinkage void __init start_kernel(void initrd_start = 0; }#endif -vmalloc_init (); -vfs_caches_init_early ();cpuset_init_early ();page_cgroup_init (); -mem_init ();enable_debug_pagealloc ();cpu_hotplug_init (); -kmem_cache_init ();kmemtrace_init ();debug_objects_mem_init ();idr_init_cache ();
kernel/irq/handle.c +8 -15
@@ -18,7 +18,7 @@#include <linux/kernel_stat.h> #include <linux/rculist.h> #include <linux/hash.h> -#include <linux/bootmem.h> +#include <linux/slab.h> #include <trace/events/irq.h> #include "internals.h"
@@ -45,7 +45,7 @@ void handle_bad_irq(unsigned int irq, st#if defined(CONFIG_SMP ) && defined(CONFIG_GENERIC_HARDIRQS )static void __init init_irq_default_affinity (void ) { -alloc_bootmem_cpumask_var (&irq_default_affinity); +alloc_cpumask_var (&irq_default_affinity,GFP_NOWAIT );cpumask_setall (irq_default_affinity); }#else
@@ -86,12 +86,8 @@ void __ref init_kstat_irqs(struct irq_de {void *ptr; -if (slab_is_available ()) - ptr =kzalloc_node (nr *sizeof (*desc->kstat_irqs), -GFP_ATOMIC , node); -else - ptr =alloc_bootmem_node (NODE_DATA (node), - nr *sizeof (*desc->kstat_irqs)); + ptr =kzalloc_node (nr *sizeof (*desc->kstat_irqs), +GFP_ATOMIC , node);/* * don't overwite if can not get new one
@@ -162,12 +158,12 @@ int __init early_irq_init(void) legacy_count =ARRAY_SIZE (irq_desc_legacy);/* allocate irq_desc_ptrs array based on nr_irqs */ - irq_desc_ptrs =alloc_bootmem (nr_irqs *sizeof (void *)); + irq_desc_ptrs =kcalloc (nr_irqs,sizeof (void *),GFP_NOWAIT );/* allocate based on nr_cpu_ids */ /* FIXME: invert kstat_irgs, and it'd be a per_cpu_alloc'd thing */ - kstat_irqs_legacy =alloc_bootmem (NR_IRQS_LEGACY * nr_cpu_ids * -sizeof (int )); + kstat_irqs_legacy =kzalloc (NR_IRQS_LEGACY * nr_cpu_ids * +sizeof (int ),GFP_NOWAIT );for (i =0 ; i < legacy_count; i++) { desc[i].irq = i;
@@ -214,10 +210,7 @@ struct irq_desc * __ref irq_to_desc_alloif (desc) goto out_unlock; -if (slab_is_available ()) - desc =kzalloc_node (sizeof (*desc),GFP_ATOMIC , node); -else - desc =alloc_bootmem_node (NODE_DATA (node),sizeof (*desc)); + desc =kzalloc_node (sizeof (*desc),GFP_ATOMIC , node);printk (KERN_DEBUG " alloc irq_desc for %d on node %d\n" , irq, node);if (!desc) {
kernel/sched.c +13 -21
@@ -69,7 +69,6 @@#include <linux/pagemap.h> #include <linux/hrtimer.h> #include <linux/tick.h> -#include <linux/bootmem.h> #include <linux/debugfs.h> #include <linux/ctype.h> #include <linux/ftrace.h>
@@ -7821,24 +7820,21 @@ static void rq_attach_root(struct rq *rq static int __init_refok init_rootdomain(struct root_domain *rd, bool bootmem) { +gfp_t gfp =GFP_KERNEL ; +memset (rd,0 ,sizeof (*rd)); -if (bootmem) { -alloc_bootmem_cpumask_var (&def_root_domain.span); -alloc_bootmem_cpumask_var (&def_root_domain.online); -alloc_bootmem_cpumask_var (&def_root_domain.rto_mask); -cpupri_init (&rd->cpupri, true); -return 0 ; - } +if (bootmem) + gfp =GFP_NOWAIT ; -if (!alloc_cpumask_var (&rd->span,GFP_KERNEL )) +if (!alloc_cpumask_var (&rd->span, gfp)) goto out; -if (!alloc_cpumask_var (&rd->online,GFP_KERNEL )) +if (!alloc_cpumask_var (&rd->online, gfp)) goto free_span; -if (!alloc_cpumask_var (&rd->rto_mask,GFP_KERNEL )) +if (!alloc_cpumask_var (&rd->rto_mask, gfp)) goto free_online; -if (cpupri_init (&rd->cpupri, false) !=0 ) +if (cpupri_init (&rd->cpupri, bootmem) !=0 ) goto free_rto_mask;return 0 ;
@@ -9157,12 +9153,8 @@ void __init sched_init(void)#ifdef CONFIG_CPUMASK_OFFSTACK alloc_size +=num_possible_cpus () *cpumask_size ();#endif -/* -* As sched_init() is called before page_alloc is setup, -* we use alloc_bootmem(). -*/ if (alloc_size) { - ptr = (unsigned long )alloc_bootmem (alloc_size); + ptr = (unsigned long )kzalloc (alloc_size,GFP_NOWAIT );#ifdef CONFIG_FAIR_GROUP_SCHED init_task_group.se = (struct sched_entity **)ptr;
@@ -9353,13 +9345,13 @@ void __init sched_init(void) current->sched_class = &fair_sched_class;/* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */ -alloc_bootmem_cpumask_var (&nohz_cpu_mask); +alloc_cpumask_var (&nohz_cpu_mask,GFP_NOWAIT );#ifdef CONFIG_SMP #ifdef CONFIG_NO_HZ -alloc_bootmem_cpumask_var (&nohz.cpu_mask); -alloc_bootmem_cpumask_var (&nohz.ilb_grp_nohz_mask); +alloc_cpumask_var (&nohz.cpu_mask,GFP_NOWAIT ); +alloc_cpumask_var (&nohz.ilb_grp_nohz_mask,GFP_NOWAIT );#endif -alloc_bootmem_cpumask_var (&cpu_isolated_map); +alloc_cpumask_var (&cpu_isolated_map,GFP_NOWAIT );#endif /* SMP */ perf_counter_init ();
mm/slub.c +10 -7
@@ -2582,13 +2582,16 @@ static struct kmem_cache *create_kmallocif (gfp_flags &SLUB_DMA ) flags =SLAB_CACHE_DMA ; -down_write (&slub_lock); +/* +* This function is called with IRQs disabled during early-boot on +* single CPU so there's no need to take slub_lock here. +*/ if (!kmem_cache_open (s, gfp_flags, name, size,ARCH_KMALLOC_MINALIGN , flags,NULL )) goto panic;list_add (&s->list, &slab_caches); -up_write (&slub_lock); +if (sysfs_slab_add (s)) goto panic;return s;
@@ -3048,7 +3051,7 @@ void __init kmem_cache_init(void)* kmem_cache_open for slab_state == DOWN. */ create_kmalloc_cache (&kmalloc_caches[0 ],"kmem_cache_node" , -sizeof (struct kmem_cache_node ),GFP_KERNEL ); +sizeof (struct kmem_cache_node ),GFP_NOWAIT ); kmalloc_caches[0 ].refcount =-1 ; caches++;
@@ -3061,16 +3064,16 @@ void __init kmem_cache_init(void)/* Caches that are not of the two-to-the-power-of size */ if (KMALLOC_MIN_SIZE <=64 ) {create_kmalloc_cache (&kmalloc_caches[1 ], -"kmalloc-96" ,96 ,GFP_KERNEL ); +"kmalloc-96" ,96 ,GFP_NOWAIT ); caches++;create_kmalloc_cache (&kmalloc_caches[2 ], -"kmalloc-192" ,192 ,GFP_KERNEL ); +"kmalloc-192" ,192 ,GFP_NOWAIT ); caches++; }for (i =KMALLOC_SHIFT_LOW ; i <SLUB_PAGE_SHIFT ; i++) {create_kmalloc_cache (&kmalloc_caches[i], -"kmalloc" ,1 << i,GFP_KERNEL ); +"kmalloc" ,1 << i,GFP_NOWAIT ); caches++; }
@@ -3107,7 +3110,7 @@ void __init kmem_cache_init(void)/* Provide the correct kmalloc names now that the caches are up */ for (i =KMALLOC_SHIFT_LOW ; i <SLUB_PAGE_SHIFT ; i++) kmalloc_caches[i]. name = -kasprintf (GFP_KERNEL ,"kmalloc-%d" ,1 << i); +kasprintf (GFP_NOWAIT ,"kmalloc-%d" ,1 << i);#ifdef CONFIG_SMP register_cpu_notifier (&slab_notifier);
kernel/sched_cpupri.c +6 -3
@@ -156,16 +156,19 @@ int __init_refok cpupri_init(struct cpup {int i; +gfp_t gfp =GFP_KERNEL ; +memset (cp,0 ,sizeof (*cp)); +if (bootmem) + gfp =GFP_NOWAIT ; +for (i =0 ; i <CPUPRI_NR_PRIORITIES ; i++) {struct cpupri_vec *vec = &cp->pri_to_cpu[i];spin_lock_init (&vec->lock); vec->count =0 ; -if (bootmem) -alloc_bootmem_cpumask_var (&vec->mask); -else if (!alloc_cpumask_var (&vec->mask,GFP_KERNEL )) +if (!alloc_cpumask_var (&vec->mask, gfp)) goto cleanup; }
Quoted 33 lines · Yinghai Lu
Quoted 4 lines
Ingo, here's a patch that boots UMA+SMP+SLUB x86-64 kernel on qemu all the way to userspace. It probably breaks bunch of things for now but something for you to play with if you want.
updated with tip/master. also add change to
cpupri_init
otherwise will get
[ 0.000000] Memory: 523096612k/537526272k available (10461k kernel code, 656156k absent, 13773504k reserved, 7186k data, 2548k init) [ 0.000000] SLUB: Genslabs=14, HWalign=64, Order=0-3, MinObjects=0, CPUs=32, Nodes=8
[ 0.000000] ------------[ cut here ]------------
[ 0.000000] WARNING: atkernel/lockdep.c:2282lockdep_trace_alloc+0xaf/0xee()
[ 0.000000] Hardware name: Sun Fire X4600 M2
[ 0.000000] Modules linked in:
[ 0.000000] Pid: 0, comm: swapper Not tainted 2.6.30-rc6-tip-01778-g0afdd0f-dirty #259
[ 0.000000] Call Trace:
[ 0.000000] [<ffffffff810a0274>] ?lockdep_trace_alloc+0xaf/0xee
[ 0.000000] [<ffffffff81075ab0>]warn_slowpath_common+0x88/0xcb
[ 0.000000] [<ffffffff81075b15>]warn_slowpath_null+0x22/0x38
[ 0.000000] [<ffffffff810a0274>]lockdep_trace_alloc+0xaf/0xee
[ 0.000000] [<ffffffff8110301b>]kmem_cache_alloc_node+0x38/0x14d
[ 0.000000] [<ffffffff813ec548>] ?alloc_cpumask_var_node+0x4a/0x10a
[ 0.000000] [<ffffffff8109eb61>] ?lockdep_init_map+0xb9/0x564
[ 0.000000] [<ffffffff813ec548>]alloc_cpumask_var_node+0x4a/0x10a
[ 0.000000] [<ffffffff813ec62c>]alloc_cpumask_var+0x24/0x3a
[ 0.000000] [<ffffffff819e6306>]cpupri_init+0x7f/0x112
[ 0.000000] [<ffffffff819e5a30>]init_rootdomain+0x72/0xb7
[ 0.000000] [<ffffffff821facce>]sched_init+0x109/0x660
[ 0.000000] [<ffffffff82203082>] ?kmem_cache_init+0x193/0x1b2
[ 0.000000] [<ffffffff821dfd7a>]start_kernel+0x218/0x3f3
[ 0.000000] [<ffffffff821df2a9>]x86_64_start_reservations+0xb9/0xd4
[ 0.000000] [<ffffffff821df3b2>]x86_64_start_kernel+0xee/0x109
[ 0.000000] ---[ end tracea7919e7f17c0a725]---
Quoted 1 line
works with 8 sockets numa amd64 box.
Oh, cool. Thanks for testing! I'll rebase on top of tip and merge your changes to my patch.
Quoted 52 lines · Yinghai Lu
Quoted 9 lines · Pekka J Enberg
Quoted 8 lines · Linus Torvalds
Quoted 4 lines
Quoted 3 lines
I hate that stupid bootmem allocator. I suspect we seriously over-use it, and that we _should_ be able to do the SL*B init earlier.
Hm, tempting thought - not sure how to pull it off though.
As far as I can recall, one of the things that historically made us want to use the bootmem allocator even relatively late was that the real SLAB allocator had to wait until all the node information etc was initialized.
Quoted 9 lines
Quoted 4 lines
That's pretty damn late. And I wonder if SLUB (and SLOB) might not need a lot less initialization, and work much earlier. Something like that might be the final nail in the coffin for SLAB, and convince me to just say 'we don't support it any more".
Ingo, here's a patch that boots UMA+SMP+SLUB x86-64 kernel on qemu all the way to userspace. It probably breaks bunch of things for now but something for you to play with if you want.
updated with tip/master. also add change to
cpupri_initotherwise will get[ 0.000000] Memory: 523096612k/537526272k available (10461k kernel code, 656156k absent, 13773504k reserved, 7186k data, 2548k init) [ 0.000000] SLUB: Genslabs=14, HWalign=64, Order=0-3, MinObjects=0, CPUs=32, Nodes=8 [ 0.000000] ------------[ cut here ]------------ [ 0.000000] WARNING: at kernel/lockdep.c:2282 lockdep_trace_alloc+0xaf/0xee() [ 0.000000] Hardware name: Sun Fire X4600 M2 [ 0.000000] Modules linked in: [ 0.000000] Pid: 0, comm: swapper Not tainted 2.6.30-rc6-tip-01778-g0afdd0f-dirty #259 [ 0.000000] Call Trace: [ 0.000000] [<ffffffff810a0274>] ? lockdep_trace_alloc+0xaf/0xee [ 0.000000] [<ffffffff81075ab0>] warn_slowpath_common+0x88/0xcb [ 0.000000] [<ffffffff81075b15>] warn_slowpath_null+0x22/0x38 [ 0.000000] [<ffffffff810a0274>] lockdep_trace_alloc+0xaf/0xee [ 0.000000] [<ffffffff8110301b>] kmem_cache_alloc_node+0x38/0x14d [ 0.000000] [<ffffffff813ec548>] ? alloc_cpumask_var_node+0x4a/0x10a [ 0.000000] [<ffffffff8109eb61>] ? lockdep_init_map+0xb9/0x564 [ 0.000000] [<ffffffff813ec548>] alloc_cpumask_var_node+0x4a/0x10a [ 0.000000] [<ffffffff813ec62c>] alloc_cpumask_var+0x24/0x3a [ 0.000000] [<ffffffff819e6306>] cpupri_init+0x7f/0x112 [ 0.000000] [<ffffffff819e5a30>] init_rootdomain+0x72/0xb7 [ 0.000000] [<ffffffff821facce>] sched_init+0x109/0x660 [ 0.000000] [<ffffffff82203082>] ? kmem_cache_init+0x193/0x1b2 [ 0.000000] [<ffffffff821dfd7a>] start_kernel+0x218/0x3f3 [ 0.000000] [<ffffffff821df2a9>] x86_64_start_reservations+0xb9/0xd4 [ 0.000000] [<ffffffff821df3b2>] x86_64_start_kernel+0xee/0x109 [ 0.000000] ---[ end trace a7919e7f17c0a725 ]---works with 8 sockets numa amd64 box.
YH
Quoted 7 lines
--- init/main.c | 28 ++++++++++++++++------------ kernel/irq/handle.c | 23 ++++++++--------------- kernel/sched.c | 34 +++++++++++++--------------------- kernel/sched_cpupri.c | 9 ++++++--- mm/slub.c | 17 ++++++++++------- 5 files changed, 53 insertions(+), 58 deletions(-)
Very nice!
Would it be possible to restructure things to move kmalloc init to before IRQ init as well? We have a couple of uglinesses there too.
Conceptually, memory should be the first thing set up in general, in a kernel. It does not need IRQs, timers, the scheduler or any of the IO facilities and abstractions. All of them need memory though - and as Linux scales to more and more hardware via the same single image, so will we get more and more dynamic concepts like cpumask_var_t and sparse-irqs, which want to allocate very early.
setup_arch() is one huge function that sets up all architecture details at once - but if we split a separate setup_arch_mem() out of it, and left the rest in setup_arch (and moved it further down), we could remove much of bootmem (especially the ugly uses).
This might even be doable realistically, and we could thus librarize bootmem and eliminate it from x86 at least. Perhaps.
Ingo
Quoted 65 lines · Ingo Molnar
Quoted 50 lines · Yinghai Lu
Quoted 9 lines · Pekka J Enberg
Quoted 8 lines · Linus Torvalds
Quoted 4 lines
Quoted 3 lines
I hate that stupid bootmem allocator. I suspect we seriously over-use it, and that we _should_ be able to do the SL*B init earlier.
Hm, tempting thought - not sure how to pull it off though.
As far as I can recall, one of the things that historically made us want to use the bootmem allocator even relatively late was that the real SLAB allocator had to wait until all the node information etc was initialized.
Quoted 8 lines
Quoted 4 lines
That's pretty damn late. And I wonder if SLUB (and SLOB) might not need a lot less initialization, and work much earlier. Something like that might be the final nail in the coffin for SLAB, and convince me to just say 'we don't support it any more".
Ingo, here's a patch that boots UMA+SMP+SLUB x86-64 kernel on qemu all the way to userspace. It probably breaks bunch of things for now but something for you to play with if you want.
updated with tip/master. also add change to
cpupri_initotherwise will get[ 0.000000] Memory: 523096612k/537526272k available (10461k kernel code, 656156k absent, 13773504k reserved, 7186k data, 2548k init) [ 0.000000] SLUB: Genslabs=14, HWalign=64, Order=0-3, MinObjects=0, CPUs=32, Nodes=8 [ 0.000000] ------------[ cut here ]------------ [ 0.000000] WARNING: at kernel/lockdep.c:2282 lockdep_trace_alloc+0xaf/0xee() [ 0.000000] Hardware name: Sun Fire X4600 M2 [ 0.000000] Modules linked in: [ 0.000000] Pid: 0, comm: swapper Not tainted 2.6.30-rc6-tip-01778-g0afdd0f-dirty #259 [ 0.000000] Call Trace: [ 0.000000] [<ffffffff810a0274>] ? lockdep_trace_alloc+0xaf/0xee [ 0.000000] [<ffffffff81075ab0>] warn_slowpath_common+0x88/0xcb [ 0.000000] [<ffffffff81075b15>] warn_slowpath_null+0x22/0x38 [ 0.000000] [<ffffffff810a0274>] lockdep_trace_alloc+0xaf/0xee [ 0.000000] [<ffffffff8110301b>] kmem_cache_alloc_node+0x38/0x14d [ 0.000000] [<ffffffff813ec548>] ? alloc_cpumask_var_node+0x4a/0x10a [ 0.000000] [<ffffffff8109eb61>] ? lockdep_init_map+0xb9/0x564 [ 0.000000] [<ffffffff813ec548>] alloc_cpumask_var_node+0x4a/0x10a [ 0.000000] [<ffffffff813ec62c>] alloc_cpumask_var+0x24/0x3a [ 0.000000] [<ffffffff819e6306>] cpupri_init+0x7f/0x112 [ 0.000000] [<ffffffff819e5a30>] init_rootdomain+0x72/0xb7 [ 0.000000] [<ffffffff821facce>] sched_init+0x109/0x660 [ 0.000000] [<ffffffff82203082>] ? kmem_cache_init+0x193/0x1b2 [ 0.000000] [<ffffffff821dfd7a>] start_kernel+0x218/0x3f3 [ 0.000000] [<ffffffff821df2a9>] x86_64_start_reservations+0xb9/0xd4 [ 0.000000] [<ffffffff821df3b2>] x86_64_start_kernel+0xee/0x109 [ 0.000000] ---[ end trace a7919e7f17c0a725 ]---works with 8 sockets numa amd64 box.
YH
Quoted 7 lines
--- init/main.c | 28 ++++++++++++++++------------ kernel/irq/handle.c | 23 ++++++++--------------- kernel/sched.c | 34 +++++++++++++--------------------- kernel/sched_cpupri.c | 9 ++++++--- mm/slub.c | 17 ++++++++++------- 5 files changed, 53 insertions(+), 58 deletions(-)Very nice!
Would it be possible to restructure things to move kmalloc init to before IRQ init as well? We have a couple of uglinesses there too.
Quoted 6 lines
Conceptually, memory should be the first thing set up in general, in a kernel. It does not need IRQs, timers, the scheduler or any of the IO facilities and abstractions. All of them need memory though - and as Linux scales to more and more hardware via the same single image, so will we get more and more dynamic concepts like
cpumask_var_tand sparse-irqs, which want to allocate very early.
Pekka's patch already made kmalloc before early_irq_init()/init_IRQ...
we can clean up alloc_desc_masks and alloc_cpumask_var_node could be much simplified too.
[PATCH] x86: remove some alloc_bootmem_cpumask_var calling
except some is called from setup_percpu_area...
--- arch/x86/kernel/apic/io_apic.c | 4 ++-- include/linux/irq.h | 18 +++++++----------- kernel/cpuset.c | 2 +- kernel/profile.c | 6 ------ lib/cpumask.c | 11 ++--------- 5 files changed, 12 insertions(+), 29 deletions(-)
include/linux/irq.h +7 -11
@@ -430,23 +430,19 @@ extern int set_irq_msi(unsigned int irq,* Returns true if successful (or not required). */ static inline bool alloc_desc_masks (struct irq_desc *desc,int node, -bool boot) +bool boot) { -#ifdef CONFIG_CPUMASK_OFFSTACK -if (boot) { -alloc_bootmem_cpumask_var (&desc->affinity); +gfp_t gfp =GFP_ATOMIC ; -#ifdef CONFIG_GENERIC_PENDING_IRQ -alloc_bootmem_cpumask_var (&desc->pending_mask); -#endif -return true; - } +if (boot) + gfp =GFP_NOWAIT ; -if (!alloc_cpumask_var_node (&desc->affinity,GFP_ATOMIC , node)) +#ifdef CONFIG_CPUMASK_OFFSTACK +if (!alloc_cpumask_var_node (&desc->affinity, gfp, node))return false;#ifdef CONFIG_GENERIC_PENDING_IRQ -if (!alloc_cpumask_var_node (&desc->pending_mask,GFP_ATOMIC , node)) { +if (!alloc_cpumask_var_node (&desc->pending_mask, gfp, node)) {free_cpumask_var (desc->affinity);return false; }
lib/cpumask.c +2 -9
@@ -92,15 +92,8 @@ int cpumask_any_but(const struct cpumask*/ bool alloc_cpumask_var_node (cpumask_var_t *mask,gfp_t flags,int node) { -if (likely (slab_is_available ())) - *mask =kmalloc_node (cpumask_size (), flags, node); -else { -#ifdef CONFIG_DEBUG_PER_CPU_MAPS -printk (KERN_ERR -"=> alloc_cpumask_var: kmalloc not available!\n" ); -#endif - *mask =NULL ; - } + *mask =kmalloc_node (cpumask_size (), flags, node); +#ifdef CONFIG_DEBUG_PER_CPU_MAPS if (!*mask) {printk (KERN_ERR "=> alloc_cpumask_var: failed!\n" );
arch/x86/kernel/apic/io_apic.c +2 -2
@@ -185,8 +185,8 @@ int __init arch_early_irq_init(void)for (i =0 ; i < count; i++) { desc =irq_to_desc (i); desc->chip_data = &cfg[i]; -alloc_bootmem_cpumask_var (&cfg[i].domain); -alloc_bootmem_cpumask_var (&cfg[i].old_domain); +alloc_cpumask_var (&cfg[i].domain,GFP_NOWAIT ); +alloc_cpumask_var (&cfg[i].old_domain,GFP_NOWAIT );if (i <NR_IRQS_LEGACY )cpumask_setall (cfg[i].domain); }
kernel/cpuset.c +1 -1
@@ -1857,7 +1857,7 @@ struct cgroup_subsys cpuset_subsys = {int __init cpuset_init_early (void ) { -alloc_bootmem_cpumask_var (&top_cpuset.cpus_allowed); +alloc_cpumask_var (&top_cpuset.cpus_allowed,GFP_NOWAIT ); top_cpuset.mems_generation = cpuset_mems_generation++;return 0 ;
kernel/profile.c +0 -6
@@ -111,12 +111,6 @@ int __ref profile_init(void)/* only text is profiled */ prof_len = (_etext - _stext) >> prof_shift; buffer_bytes = prof_len*sizeof (atomic_t); -if (!slab_is_available ()) { - prof_buffer =alloc_bootmem (buffer_bytes); -alloc_bootmem_cpumask_var (&prof_cpu_mask); -cpumask_copy (prof_cpu_mask, cpu_possible_mask); -return 0 ; - }if (!alloc_cpumask_var (&prof_cpu_mask,GFP_KERNEL ))return -ENOMEM ;
Quoted 73 lines · Yinghai Lu
Quoted 65 lines · Ingo Molnar
Quoted 50 lines · Yinghai Lu
Quoted 9 lines · Pekka J Enberg
Quoted 8 lines · Linus Torvalds
Quoted 4 lines
Quoted 3 lines
I hate that stupid bootmem allocator. I suspect we seriously over-use it, and that we _should_ be able to do the SL*B init earlier.
Hm, tempting thought - not sure how to pull it off though.
As far as I can recall, one of the things that historically made us want to use the bootmem allocator even relatively late was that the real SLAB allocator had to wait until all the node information etc was initialized.
Quoted 8 lines
Quoted 4 lines
That's pretty damn late. And I wonder if SLUB (and SLOB) might not need a lot less initialization, and work much earlier. Something like that might be the final nail in the coffin for SLAB, and convince me to just say 'we don't support it any more".
Ingo, here's a patch that boots UMA+SMP+SLUB x86-64 kernel on qemu all the way to userspace. It probably breaks bunch of things for now but something for you to play with if you want.
updated with tip/master. also add change to
cpupri_initotherwise will get[ 0.000000] Memory: 523096612k/537526272k available (10461k kernel code, 656156k absent, 13773504k reserved, 7186k data, 2548k init) [ 0.000000] SLUB: Genslabs=14, HWalign=64, Order=0-3, MinObjects=0, CPUs=32, Nodes=8 [ 0.000000] ------------[ cut here ]------------ [ 0.000000] WARNING: at kernel/lockdep.c:2282 lockdep_trace_alloc+0xaf/0xee() [ 0.000000] Hardware name: Sun Fire X4600 M2 [ 0.000000] Modules linked in: [ 0.000000] Pid: 0, comm: swapper Not tainted 2.6.30-rc6-tip-01778-g0afdd0f-dirty #259 [ 0.000000] Call Trace: [ 0.000000] [<ffffffff810a0274>] ? lockdep_trace_alloc+0xaf/0xee [ 0.000000] [<ffffffff81075ab0>] warn_slowpath_common+0x88/0xcb [ 0.000000] [<ffffffff81075b15>] warn_slowpath_null+0x22/0x38 [ 0.000000] [<ffffffff810a0274>] lockdep_trace_alloc+0xaf/0xee [ 0.000000] [<ffffffff8110301b>] kmem_cache_alloc_node+0x38/0x14d [ 0.000000] [<ffffffff813ec548>] ? alloc_cpumask_var_node+0x4a/0x10a [ 0.000000] [<ffffffff8109eb61>] ? lockdep_init_map+0xb9/0x564 [ 0.000000] [<ffffffff813ec548>] alloc_cpumask_var_node+0x4a/0x10a [ 0.000000] [<ffffffff813ec62c>] alloc_cpumask_var+0x24/0x3a [ 0.000000] [<ffffffff819e6306>] cpupri_init+0x7f/0x112 [ 0.000000] [<ffffffff819e5a30>] init_rootdomain+0x72/0xb7 [ 0.000000] [<ffffffff821facce>] sched_init+0x109/0x660 [ 0.000000] [<ffffffff82203082>] ? kmem_cache_init+0x193/0x1b2 [ 0.000000] [<ffffffff821dfd7a>] start_kernel+0x218/0x3f3 [ 0.000000] [<ffffffff821df2a9>] x86_64_start_reservations+0xb9/0xd4 [ 0.000000] [<ffffffff821df3b2>] x86_64_start_kernel+0xee/0x109 [ 0.000000] ---[ end trace a7919e7f17c0a725 ]---works with 8 sockets numa amd64 box.
YH
Quoted 7 lines
--- init/main.c | 28 ++++++++++++++++------------ kernel/irq/handle.c | 23 ++++++++--------------- kernel/sched.c | 34 +++++++++++++--------------------- kernel/sched_cpupri.c | 9 ++++++--- mm/slub.c | 17 ++++++++++------- 5 files changed, 53 insertions(+), 58 deletions(-)Very nice!
Would it be possible to restructure things to move kmalloc init to before IRQ init as well? We have a couple of uglinesses there too.
Quoted 6 lines
Conceptually, memory should be the first thing set up in general, in a kernel. It does not need IRQs, timers, the scheduler or any of the IO facilities and abstractions. All of them need memory though - and as Linux scales to more and more hardware via the same single image, so will we get more and more dynamic concepts like
cpumask_var_tand sparse-irqs, which want to allocate very early.
Quoted 4 lines
Pekka's patch already made kmalloc before early_irq_init()/init_IRQ...
we can clean up
alloc_desc_masksandalloc_cpumask_var_nodecould be much simplified too.
That's nice!
Ok, i think this all looks pretty realistic - but there's quite a bit of layering on top of pending changes in the x86 and irq trees. We could do this on top of those topic branches in -tip, and rebase in the merge window. Or delay it to .32.
... plus i think we are _very_ close to being able to remove all of bootmem on x86 (with some compatibility/migration mechanism in place). Which bootmem calls do we have before kmalloc init with Pekka's patch applied? I think it's mostly the page table init code.
( beyond the page allocator internal use - where we could use
straight e820 based APIs that clip memory off from the beginning of existing e820 RAM ranges - enriched with NUMA/SRAT locality info. )
Ingo
Quoted 5 lines · Ingo Molnar
Ok, i think this all looks pretty realistic - but there's quite a bit of layering on top of pending changes in the x86 and irq trees. We could do this on top of those topic branches in -tip, and rebase in the merge window. Or delay it to .32.
would have move setup_per_cpu_areas after mem_init(). some kind of limiting bootmem related in setup_arch()
Quoted 5 lines
... plus i think we are _very_ close to being able to remove all of bootmem on x86 (with some compatibility/migration mechanism in place). Which bootmem calls do we have before kmalloc init with Pekka's patch applied? I think it's mostly the page table init code.
need to decide what should be in setup_arch_mem, or setup_arch_rest(). before initmem_init() ==> setup_arch_mem
after initmem_init()
reserve_bootmem related should stay in setup_arch_mem
try to move other call in setup_arch to _reset after ane setup_arch_rest will be called after mem_init()
Quoted 5 lines
( beyond the page allocator internal use - where we could use
straight e820 based APIs that clip memory off from the beginning of existing e820 RAM ranges - enriched with NUMA/SRAT locality info. )
yes. it is there. need to dynamic early_res array.
YH
Quoted 10 lines · Ingo Molnar
Ok, i think this all looks pretty realistic - but there's quite a bit of layering on top of pending changes in the x86 and irq trees. We could do this on top of those topic branches in -tip, and rebase in the merge window. Or delay it to .32.
... plus i think we are _very_ close to being able to remove all of bootmem on x86 (with some compatibility/migration mechanism in place). Which bootmem calls do we have before kmalloc init with Pekka's patch applied? I think it's mostly the page table init code.
Quoted 4 lines
( beyond the page allocator internal use - where we could use
straight e820 based APIs that clip memory off from the beginning of existing e820 RAM ranges - enriched with NUMA/SRAT locality info. )
OK, here's a version of the patch with Yinghai's rebasing on top of tip/master. All in-kernel memory allocators boot cleanly now on my configuration (I did not try SLQB but it probably needs fixing). I would appreciate if someone gave SLAB+NUMA configuration a try.
Ingo, I don't really want to carry this in slab.git so perhaps you could put this and Yinghai's irq init cleanup in a "earlyslab" branch in tip for testing?
And oh, Christoph/Matt, can I have your NAK/ACK for this patch?
Pekka
Quoted 1 line
From
f5338db5f1b959cb82ed811037a545e19c7b2b7bMon Sep 17 00:00:00 2001
From: Pekka Enberg <[email protected]>
Date: Mon, 25 May 2009 09:44:39 +0300
Subject: [PATCH] slab: setup allocators earlier in the boot sequence
This patch makes kmalloc() available earlier in the boot sequence so we can get rid of some bootmem allocations. The bulk of the changes are due to kmem_cache_init() being called with interrupts disabled which requires some changes to allocator boostrap code.
--- init/main.c | 28 +++++++++------- kernel/irq/handle.c | 23 +++++--------- kernel/sched.c | 34 +++++++------------ kernel/sched_cpupri.c | 9 +++-- mm/slab.c | 85 ++++++++++++++++++++++++++----------------------- mm/slub.c | 17 ++++++---- 6 files changed, 98 insertions(+), 98 deletions(-)
init/main.c +16 -12
@@ -576,6 +576,22 @@ asmlinkage void __init start_kernel(void)setup_nr_cpu_ids ();smp_prepare_boot_cpu ();/* arch-specific boot-cpu hooks */ +build_all_zonelists (); +page_alloc_init (); + +printk (KERN_NOTICE "Kernel command line: %s\n" , boot_command_line); +parse_early_param (); +parse_args ("Booting kernel" , static_command_line, __start___param, + __stop___param - __start___param, + &unknown_bootoption); +/* +* Setup kernel memory allocators +*/ +pidhash_init (); +vmalloc_init (); +vfs_caches_init_early (); +mem_init (); +kmem_cache_init ();/* * Set up the scheduler prior starting any interrupts (such as the * timer interrupt). Full topology setup happens at smp_init()
@@ -587,13 +603,6 @@ asmlinkage void __init start_kernel(void)* fragile until we cpu_idle() for the first time. */ preempt_disable (); -build_all_zonelists (); -page_alloc_init (); -printk (KERN_NOTICE "Kernel command line: %s\n" , boot_command_line); -parse_early_param (); -parse_args ("Booting kernel" , static_command_line, __start___param, - __stop___param - __start___param, - &unknown_bootoption);if (!irqs_disabled ()) {printk (KERN_WARNING "start_kernel(): bug: interrupts were " "enabled *very* early, fixing it\n" );
@@ -605,7 +614,6 @@ asmlinkage void __init start_kernel(void)/* init some links before init_ISA_irqs() */ early_irq_init ();init_IRQ (); -pidhash_init ();init_timers ();hrtimers_init ();softirq_init ();
@@ -647,14 +655,10 @@ asmlinkage void __init start_kernel(void) initrd_start = 0; }#endif -vmalloc_init (); -vfs_caches_init_early ();cpuset_init_early ();page_cgroup_init (); -mem_init ();enable_debug_pagealloc ();cpu_hotplug_init (); -kmem_cache_init ();kmemtrace_init ();debug_objects_mem_init ();idr_init_cache ();
kernel/irq/handle.c +8 -15
@@ -18,7 +18,7 @@#include <linux/kernel_stat.h> #include <linux/rculist.h> #include <linux/hash.h> -#include <linux/bootmem.h> +#include <linux/slab.h> #include <trace/events/irq.h> #include "internals.h"
@@ -45,7 +45,7 @@ void handle_bad_irq(unsigned int irq, struct irq_desc *desc)#if defined(CONFIG_SMP ) && defined(CONFIG_GENERIC_HARDIRQS )static void __init init_irq_default_affinity (void ) { -alloc_bootmem_cpumask_var (&irq_default_affinity); +alloc_cpumask_var (&irq_default_affinity,GFP_NOWAIT );cpumask_setall (irq_default_affinity); }#else
@@ -86,12 +86,8 @@ void __ref init_kstat_irqs(struct irq_desc *desc, int node, int nr) {void *ptr; -if (slab_is_available ()) - ptr =kzalloc_node (nr *sizeof (*desc->kstat_irqs), -GFP_ATOMIC , node); -else - ptr =alloc_bootmem_node (NODE_DATA (node), - nr *sizeof (*desc->kstat_irqs)); + ptr =kzalloc_node (nr *sizeof (*desc->kstat_irqs), +GFP_ATOMIC , node);/* * don't overwite if can not get new one
@@ -162,12 +158,12 @@ int __init early_irq_init(void) legacy_count =ARRAY_SIZE (irq_desc_legacy);/* allocate irq_desc_ptrs array based on nr_irqs */ - irq_desc_ptrs =alloc_bootmem (nr_irqs *sizeof (void *)); + irq_desc_ptrs =kcalloc (nr_irqs,sizeof (void *),GFP_NOWAIT );/* allocate based on nr_cpu_ids */ /* FIXME: invert kstat_irgs, and it'd be a per_cpu_alloc'd thing */ - kstat_irqs_legacy =alloc_bootmem (NR_IRQS_LEGACY * nr_cpu_ids * -sizeof (int )); + kstat_irqs_legacy =kzalloc (NR_IRQS_LEGACY * nr_cpu_ids * +sizeof (int ),GFP_NOWAIT );for (i =0 ; i < legacy_count; i++) { desc[i].irq = i;
@@ -214,10 +210,7 @@ struct irq_desc * __ref irq_to_desc_alloc_node(unsigned int irq, int node)if (desc) goto out_unlock; -if (slab_is_available ()) - desc =kzalloc_node (sizeof (*desc),GFP_ATOMIC , node); -else - desc =alloc_bootmem_node (NODE_DATA (node),sizeof (*desc)); + desc =kzalloc_node (sizeof (*desc),GFP_ATOMIC , node);printk (KERN_DEBUG " alloc irq_desc for %d on node %d\n" , irq, node);if (!desc) {
kernel/sched.c +13 -21
@@ -69,7 +69,6 @@#include <linux/pagemap.h> #include <linux/hrtimer.h> #include <linux/tick.h> -#include <linux/bootmem.h> #include <linux/debugfs.h> #include <linux/ctype.h> #include <linux/ftrace.h>
@@ -7821,24 +7820,21 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd) static int __init_refok init_rootdomain(struct root_domain *rd, bool bootmem) { +gfp_t gfp =GFP_KERNEL ; +memset (rd,0 ,sizeof (*rd)); -if (bootmem) { -alloc_bootmem_cpumask_var (&def_root_domain.span); -alloc_bootmem_cpumask_var (&def_root_domain.online); -alloc_bootmem_cpumask_var (&def_root_domain.rto_mask); -cpupri_init (&rd->cpupri, true); -return 0 ; - } +if (bootmem) + gfp =GFP_NOWAIT ; -if (!alloc_cpumask_var (&rd->span,GFP_KERNEL )) +if (!alloc_cpumask_var (&rd->span, gfp)) goto out; -if (!alloc_cpumask_var (&rd->online,GFP_KERNEL )) +if (!alloc_cpumask_var (&rd->online, gfp)) goto free_span; -if (!alloc_cpumask_var (&rd->rto_mask,GFP_KERNEL )) +if (!alloc_cpumask_var (&rd->rto_mask, gfp)) goto free_online; -if (cpupri_init (&rd->cpupri, false) !=0 ) +if (cpupri_init (&rd->cpupri, bootmem) !=0 ) goto free_rto_mask;return 0 ;
@@ -9157,12 +9153,8 @@ void __init sched_init(void)#ifdef CONFIG_CPUMASK_OFFSTACK alloc_size +=num_possible_cpus () *cpumask_size ();#endif -/* -* As sched_init() is called before page_alloc is setup, -* we use alloc_bootmem(). -*/ if (alloc_size) { - ptr = (unsigned long )alloc_bootmem (alloc_size); + ptr = (unsigned long )kzalloc (alloc_size,GFP_NOWAIT );#ifdef CONFIG_FAIR_GROUP_SCHED init_task_group.se = (struct sched_entity **)ptr;
@@ -9353,13 +9345,13 @@ void __init sched_init(void) current->sched_class = &fair_sched_class;/* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */ -alloc_bootmem_cpumask_var (&nohz_cpu_mask); +alloc_cpumask_var (&nohz_cpu_mask,GFP_NOWAIT );#ifdef CONFIG_SMP #ifdef CONFIG_NO_HZ -alloc_bootmem_cpumask_var (&nohz.cpu_mask); -alloc_bootmem_cpumask_var (&nohz.ilb_grp_nohz_mask); +alloc_cpumask_var (&nohz.cpu_mask,GFP_NOWAIT ); +alloc_cpumask_var (&nohz.ilb_grp_nohz_mask,GFP_NOWAIT );#endif -alloc_bootmem_cpumask_var (&cpu_isolated_map); +alloc_cpumask_var (&cpu_isolated_map,GFP_NOWAIT );#endif /* SMP */ perf_counter_init ();
kernel/sched_cpupri.c +6 -3
@@ -156,16 +156,19 @@ int __init_refok cpupri_init(struct cpupri *cp, bool bootmem) {int i; +gfp_t gfp =GFP_KERNEL ; +memset (cp,0 ,sizeof (*cp)); +if (bootmem) + gfp =GFP_NOWAIT ; +for (i =0 ; i <CPUPRI_NR_PRIORITIES ; i++) {struct cpupri_vec *vec = &cp->pri_to_cpu[i];spin_lock_init (&vec->lock); vec->count =0 ; -if (bootmem) -alloc_bootmem_cpumask_var (&vec->mask); -else if (!alloc_cpumask_var (&vec->mask,GFP_KERNEL )) +if (!alloc_cpumask_var (&vec->mask, gfp)) goto cleanup; }
mm/slab.c +45 -40
288 lines not on this page · raw message
mm/slub.c +10 -7
@@ -2582,13 +2582,16 @@ static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s,if (gfp_flags &SLUB_DMA ) flags =SLAB_CACHE_DMA ; -down_write (&slub_lock); +/* +* This function is called with IRQs disabled during early-boot on +* single CPU so there's no need to take slub_lock here. +*/ if (!kmem_cache_open (s, gfp_flags, name, size,ARCH_KMALLOC_MINALIGN , flags,NULL )) goto panic;list_add (&s->list, &slab_caches); -up_write (&slub_lock); +if (sysfs_slab_add (s)) goto panic;return s;
@@ -3048,7 +3051,7 @@ void __init kmem_cache_init(void)* kmem_cache_open for slab_state == DOWN. */ create_kmalloc_cache (&kmalloc_caches[0 ],"kmem_cache_node" , -sizeof (struct kmem_cache_node ),GFP_KERNEL ); +sizeof (struct kmem_cache_node ),GFP_NOWAIT ); kmalloc_caches[0 ].refcount =-1 ; caches++;
@@ -3061,16 +3064,16 @@ void __init kmem_cache_init(void)/* Caches that are not of the two-to-the-power-of size */ if (KMALLOC_MIN_SIZE <=64 ) {create_kmalloc_cache (&kmalloc_caches[1 ], -"kmalloc-96" ,96 ,GFP_KERNEL ); +"kmalloc-96" ,96 ,GFP_NOWAIT ); caches++;create_kmalloc_cache (&kmalloc_caches[2 ], -"kmalloc-192" ,192 ,GFP_KERNEL ); +"kmalloc-192" ,192 ,GFP_NOWAIT ); caches++; }for (i =KMALLOC_SHIFT_LOW ; i <SLUB_PAGE_SHIFT ; i++) {create_kmalloc_cache (&kmalloc_caches[i], -"kmalloc" ,1 << i,GFP_KERNEL ); +"kmalloc" ,1 << i,GFP_NOWAIT ); caches++; }
@@ -3107,7 +3110,7 @@ void __init kmem_cache_init(void)/* Provide the correct kmalloc names now that the caches are up */ for (i =KMALLOC_SHIFT_LOW ; i <SLUB_PAGE_SHIFT ; i++) kmalloc_caches[i]. name = -kasprintf (GFP_KERNEL ,"kmalloc-%d" ,1 << i); +kasprintf (GFP_NOWAIT ,"kmalloc-%d" ,1 << i);#ifdef CONFIG_SMP register_cpu_notifier (&slab_notifier);
Signature · 1 line
1.6.0.4
Quoted 68 lines · Pekka Enberg
Quoted 10 lines · Ingo Molnar
Ok, i think this all looks pretty realistic - but there's quite a bit of layering on top of pending changes in the x86 and irq trees. We could do this on top of those topic branches in -tip, and rebase in the merge window. Or delay it to .32.
... plus i think we are _very_ close to being able to remove all of bootmem on x86 (with some compatibility/migration mechanism in place). Which bootmem calls do we have before kmalloc init with Pekka's patch applied? I think it's mostly the page table init code.
Quoted 4 lines
( beyond the page allocator internal use - where we could use
straight e820 based APIs that clip memory off from the beginning of existing e820 RAM ranges - enriched with NUMA/SRAT locality info. )OK, here's a version of the patch with Yinghai's rebasing on top of tip/master. All in-kernel memory allocators boot cleanly now on my configuration (I did not try SLQB but it probably needs fixing). I would appreciate if someone gave SLAB+NUMA configuration a try.
Ingo, I don't really want to carry this in slab.git so perhaps you could put this and Yinghai's irq init cleanup in a "earlyslab" branch in tip for testing?
And oh, Christoph/Matt, can I have your NAK/ACK for this patch?
Pekka
Quoted 1 line
From
f5338db5f1b959cb82ed811037a545e19c7b2b7bMon Sep 17 00:00:00 2001From: Pekka Enberg <[email protected]>
Date: Mon, 25 May 2009 09:44:39 +0300
Subject: [PATCH] slab: setup allocators earlier in the boot sequenceThis patch makes
kmalloc()available earlier in the boot sequence so we can get rid of some bootmem allocations. The bulk of the changes are due tokmem_cache_init()being called with interrupts disabled which requires some changes to allocator boostrap code.[ [email protected]: rebase to tip ]Cc Nick Piggin <[email protected]>Cc Matt Mackall <[email protected]>Cc Christoph Lameter <[email protected]>Cc Linus Torvalds <[email protected]>Signed-off-by Pekka Enberg <[email protected]>--- init/main.c | 28 +++++++++------- kernel/irq/handle.c | 23 +++++--------- kernel/sched.c | 34 +++++++------------ kernel/sched_cpupri.c | 9 +++-- mm/slab.c | 85 ++++++++++++++++++++++++++----------------------- mm/slub.c | 17 ++++++---- 6 files changed, 98 insertions(+), 98 deletions(-)init/main.c +6
-0@@ -576,6 +576,22 @@ asmlinkage void __init start_kernel(void)setup_nr_cpu_ids ();smp_prepare_boot_cpu ();/* arch-specific boot-cpu hooks */ +build_all_zonelists (); +page_alloc_init (); + +printk (KERN_NOTICE "Kernel command line: %s\n" , boot_command_line); +parse_early_param (); + parse_args("Booting kernel", static_command_line, __start___param,
init/main.c asmlinkage void __init start_kernel(void) · line 594
Quoted 10 lines
init/main.c +10
-0+ __stop___param - __start___param, + &unknown_bootoption); +/* +* Setup kernel memory allocators +*/ +pidhash_init (); +vmalloc_init (); +vfs_caches_init_early (); +mem_init (); +kmem_cache_init ();
Looks quite OK to me. The comment above maybe misleading?
The bootmem allocations of course are required because some hashes may need to be larger than MAX_ORDER without using vmalloc.
kmem_cache_init (and mem_init, partially) sets up the kernel memory allocators...
I can take a look at the SLQB side.
Quoted 20 lines · Nick Piggin
Quoted 20 lines
init/main.c +0
-0@@ -576,6 +576,22 @@ asmlinkage void __init start_kernel(void) setup_nr_cpu_ids();
smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */+
build_all_zonelists();
+page_alloc_init();
+
+printk(KERN_NOTICE "Kernel command line: %s\n", boot_command_line);
+parse_early_param();
+ parse_args("Booting kernel",static_command_line,__start___param,
+ __stop___param -__start___param,
+ &unknown_bootoption);
+ /*
+ * Setup kernel memory allocators
+ */
+pidhash_init();
+vmalloc_init();
+vfs_caches_init_early();
+mem_init();
+kmem_cache_init();
Quoted 7 lines · Nick Piggin
Looks quite OK to me. The comment above maybe misleading?
The bootmem allocations of course are required because some hashes may need to be larger than
MAX_ORDERwithout using vmalloc.
kmem_cache_init(andmem_init, partially) sets up the kernel memory allocators...
Which comment are you talking about? The "setup kernel memory allocators" one? Yeah, I should probably move it couple of lines down. It's just that we absolutely need to do the other ones before calling mem_init() and then kmem_cache_init().
Quoted 1 line · Nick Piggin
I can take a look at the SLQB side.
Great!
Pekka
Quoted 9 lines · Pekka Enberg
Quoted 5 lines · Nick Piggin
The bootmem allocations of course are required because some hashes may need to be larger than
MAX_ORDERwithout using vmalloc.
kmem_cache_init(andmem_init, partially) sets up the kernel memory allocators...Which comment are you talking about? The "setup kernel memory allocators" one? Yeah, I should probably move it couple of lines down.
Yes, just that one.
Quoted 6 lines · Nick Piggin
Quoted 6 lines
Quoted 5 lines
The bootmem allocations of course are required because some hashes may need to be larger than
MAX_ORDERwithout using vmalloc.
kmem_cache_init(andmem_init, partially) sets up the kernel memory allocators...
Quoted 2 lines · Pekka Enberg
Quoted 2 lines
Which comment are you talking about? The "setup kernel memory allocators" one? Yeah, I should probably move it couple of lines down.
Quoted 1 line · Nick Piggin
Yes, just that one.
Here's an updated patch that does that. Thanks!
Pekka
Quoted 1 line
From
fbd1061276b607d6eca65d4bc50d7d5c0d2e27ceMon Sep 17 00:00:00 2001
From: Pekka Enberg <[email protected]>
Date: Mon, 25 May 2009 09:44:39 +0300
Subject: [PATCH] slab: setup allocators earlier in the boot sequence
This patch makes kmalloc() available earlier in the boot sequence so we can get rid of some bootmem allocations. The bulk of the changes are due to kmem_cache_init() being called with interrupts disabled which requires some changes to allocator boostrap code.
--- init/main.c | 28 +++++++++------- kernel/irq/handle.c | 23 +++++--------- kernel/sched.c | 34 +++++++------------ kernel/sched_cpupri.c | 9 +++-- mm/slab.c | 85 ++++++++++++++++++++++++++----------------------- mm/slub.c | 17 ++++++---- 6 files changed, 98 insertions(+), 98 deletions(-)
init/main.c +16 -12
@@ -576,6 +576,22 @@ asmlinkage void __init start_kernel(void)setup_nr_cpu_ids ();smp_prepare_boot_cpu ();/* arch-specific boot-cpu hooks */ +build_all_zonelists (); +page_alloc_init (); + +printk (KERN_NOTICE "Kernel command line: %s\n" , boot_command_line); +parse_early_param (); +parse_args ("Booting kernel" , static_command_line, __start___param, + __stop___param - __start___param, + &unknown_bootoption); +pidhash_init (); +vmalloc_init (); +vfs_caches_init_early (); +/* +* Set up kernel memory allocators +*/ +mem_init (); +kmem_cache_init ();/* * Set up the scheduler prior starting any interrupts (such as the * timer interrupt). Full topology setup happens at smp_init()
@@ -587,13 +603,6 @@ asmlinkage void __init start_kernel(void)* fragile until we cpu_idle() for the first time. */ preempt_disable (); -build_all_zonelists (); -page_alloc_init (); -printk (KERN_NOTICE "Kernel command line: %s\n" , boot_command_line); -parse_early_param (); -parse_args ("Booting kernel" , static_command_line, __start___param, - __stop___param - __start___param, - &unknown_bootoption);if (!irqs_disabled ()) {printk (KERN_WARNING "start_kernel(): bug: interrupts were " "enabled *very* early, fixing it\n" );
@@ -605,7 +614,6 @@ asmlinkage void __init start_kernel(void)/* init some links before init_ISA_irqs() */ early_irq_init ();init_IRQ (); -pidhash_init ();init_timers ();hrtimers_init ();softirq_init ();
@@ -647,14 +655,10 @@ asmlinkage void __init start_kernel(void) initrd_start = 0; }#endif -vmalloc_init (); -vfs_caches_init_early ();cpuset_init_early ();page_cgroup_init (); -mem_init ();enable_debug_pagealloc ();cpu_hotplug_init (); -kmem_cache_init ();kmemtrace_init ();debug_objects_mem_init ();idr_init_cache ();
kernel/irq/handle.c +8 -15
@@ -18,7 +18,7 @@#include <linux/kernel_stat.h> #include <linux/rculist.h> #include <linux/hash.h> -#include <linux/bootmem.h> +#include <linux/slab.h> #include <trace/events/irq.h> #include "internals.h"
@@ -45,7 +45,7 @@ void handle_bad_irq(unsigned int irq, struct irq_desc *desc)#if defined(CONFIG_SMP ) && defined(CONFIG_GENERIC_HARDIRQS )static void __init init_irq_default_affinity (void ) { -alloc_bootmem_cpumask_var (&irq_default_affinity); +alloc_cpumask_var (&irq_default_affinity,GFP_NOWAIT );cpumask_setall (irq_default_affinity); }#else
@@ -86,12 +86,8 @@ void __ref init_kstat_irqs(struct irq_desc *desc, int node, int nr) {void *ptr; -if (slab_is_available ()) - ptr =kzalloc_node (nr *sizeof (*desc->kstat_irqs), -GFP_ATOMIC , node); -else - ptr =alloc_bootmem_node (NODE_DATA (node), - nr *sizeof (*desc->kstat_irqs)); + ptr =kzalloc_node (nr *sizeof (*desc->kstat_irqs), +GFP_ATOMIC , node);/* * don't overwite if can not get new one
@@ -162,12 +158,12 @@ int __init early_irq_init(void) legacy_count =ARRAY_SIZE (irq_desc_legacy);/* allocate irq_desc_ptrs array based on nr_irqs */ - irq_desc_ptrs =alloc_bootmem (nr_irqs *sizeof (void *)); + irq_desc_ptrs =kcalloc (nr_irqs,sizeof (void *),GFP_NOWAIT );/* allocate based on nr_cpu_ids */ /* FIXME: invert kstat_irgs, and it'd be a per_cpu_alloc'd thing */ - kstat_irqs_legacy =alloc_bootmem (NR_IRQS_LEGACY * nr_cpu_ids * -sizeof (int )); + kstat_irqs_legacy =kzalloc (NR_IRQS_LEGACY * nr_cpu_ids * +sizeof (int ),GFP_NOWAIT );for (i =0 ; i < legacy_count; i++) { desc[i].irq = i;
@@ -214,10 +210,7 @@ struct irq_desc * __ref irq_to_desc_alloc_node(unsigned int irq, int node)if (desc) goto out_unlock; -if (slab_is_available ()) - desc =kzalloc_node (sizeof (*desc),GFP_ATOMIC , node); -else - desc =alloc_bootmem_node (NODE_DATA (node),sizeof (*desc)); + desc =kzalloc_node (sizeof (*desc),GFP_ATOMIC , node);printk (KERN_DEBUG " alloc irq_desc for %d on node %d\n" , irq, node);if (!desc) {
kernel/sched.c +13 -21
@@ -69,7 +69,6 @@#include <linux/pagemap.h> #include <linux/hrtimer.h> #include <linux/tick.h> -#include <linux/bootmem.h> #include <linux/debugfs.h> #include <linux/ctype.h> #include <linux/ftrace.h>
@@ -7821,24 +7820,21 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd) static int __init_refok init_rootdomain(struct root_domain *rd, bool bootmem) { +gfp_t gfp =GFP_KERNEL ; +memset (rd,0 ,sizeof (*rd)); -if (bootmem) { -alloc_bootmem_cpumask_var (&def_root_domain.span); -alloc_bootmem_cpumask_var (&def_root_domain.online); -alloc_bootmem_cpumask_var (&def_root_domain.rto_mask); -cpupri_init (&rd->cpupri, true); -return 0 ; - } +if (bootmem) + gfp =GFP_NOWAIT ; -if (!alloc_cpumask_var (&rd->span,GFP_KERNEL )) +if (!alloc_cpumask_var (&rd->span, gfp)) goto out; -if (!alloc_cpumask_var (&rd->online,GFP_KERNEL )) +if (!alloc_cpumask_var (&rd->online, gfp)) goto free_span; -if (!alloc_cpumask_var (&rd->rto_mask,GFP_KERNEL )) +if (!alloc_cpumask_var (&rd->rto_mask, gfp)) goto free_online; -if (cpupri_init (&rd->cpupri, false) !=0 ) +if (cpupri_init (&rd->cpupri, bootmem) !=0 ) goto free_rto_mask;return 0 ;
@@ -9157,12 +9153,8 @@ void __init sched_init(void)#ifdef CONFIG_CPUMASK_OFFSTACK alloc_size +=num_possible_cpus () *cpumask_size ();#endif -/* -* As sched_init() is called before page_alloc is setup, -* we use alloc_bootmem(). -*/ if (alloc_size) { - ptr = (unsigned long )alloc_bootmem (alloc_size); + ptr = (unsigned long )kzalloc (alloc_size,GFP_NOWAIT );#ifdef CONFIG_FAIR_GROUP_SCHED init_task_group.se = (struct sched_entity **)ptr;
@@ -9353,13 +9345,13 @@ void __init sched_init(void) current->sched_class = &fair_sched_class;/* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */ -alloc_bootmem_cpumask_var (&nohz_cpu_mask); +alloc_cpumask_var (&nohz_cpu_mask,GFP_NOWAIT );#ifdef CONFIG_SMP #ifdef CONFIG_NO_HZ -alloc_bootmem_cpumask_var (&nohz.cpu_mask); -alloc_bootmem_cpumask_var (&nohz.ilb_grp_nohz_mask); +alloc_cpumask_var (&nohz.cpu_mask,GFP_NOWAIT ); +alloc_cpumask_var (&nohz.ilb_grp_nohz_mask,GFP_NOWAIT );#endif -alloc_bootmem_cpumask_var (&cpu_isolated_map); +alloc_cpumask_var (&cpu_isolated_map,GFP_NOWAIT );#endif /* SMP */ perf_counter_init ();
kernel/sched_cpupri.c +6 -3
@@ -156,16 +156,19 @@ int __init_refok cpupri_init(struct cpupri *cp, bool bootmem) {int i; +gfp_t gfp =GFP_KERNEL ; +memset (cp,0 ,sizeof (*cp)); +if (bootmem) + gfp =GFP_NOWAIT ; +for (i =0 ; i <CPUPRI_NR_PRIORITIES ; i++) {struct cpupri_vec *vec = &cp->pri_to_cpu[i];spin_lock_init (&vec->lock); vec->count =0 ; -if (bootmem) -alloc_bootmem_cpumask_var (&vec->mask); -else if (!alloc_cpumask_var (&vec->mask,GFP_KERNEL )) +if (!alloc_cpumask_var (&vec->mask, gfp)) goto cleanup; }
mm/slab.c +45 -40
288 lines not on this page · raw message
mm/slub.c +10 -7
@@ -2582,13 +2582,16 @@ static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s,if (gfp_flags &SLUB_DMA ) flags =SLAB_CACHE_DMA ; -down_write (&slub_lock); +/* +* This function is called with IRQs disabled during early-boot on +* single CPU so there's no need to take slub_lock here. +*/ if (!kmem_cache_open (s, gfp_flags, name, size,ARCH_KMALLOC_MINALIGN , flags,NULL )) goto panic;list_add (&s->list, &slab_caches); -up_write (&slub_lock); +if (sysfs_slab_add (s)) goto panic;return s;
@@ -3048,7 +3051,7 @@ void __init kmem_cache_init(void)* kmem_cache_open for slab_state == DOWN. */ create_kmalloc_cache (&kmalloc_caches[0 ],"kmem_cache_node" , -sizeof (struct kmem_cache_node ),GFP_KERNEL ); +sizeof (struct kmem_cache_node ),GFP_NOWAIT ); kmalloc_caches[0 ].refcount =-1 ; caches++;
@@ -3061,16 +3064,16 @@ void __init kmem_cache_init(void)/* Caches that are not of the two-to-the-power-of size */ if (KMALLOC_MIN_SIZE <=64 ) {create_kmalloc_cache (&kmalloc_caches[1 ], -"kmalloc-96" ,96 ,GFP_KERNEL ); +"kmalloc-96" ,96 ,GFP_NOWAIT ); caches++;create_kmalloc_cache (&kmalloc_caches[2 ], -"kmalloc-192" ,192 ,GFP_KERNEL ); +"kmalloc-192" ,192 ,GFP_NOWAIT ); caches++; }for (i =KMALLOC_SHIFT_LOW ; i <SLUB_PAGE_SHIFT ; i++) {create_kmalloc_cache (&kmalloc_caches[i], -"kmalloc" ,1 << i,GFP_KERNEL ); +"kmalloc" ,1 << i,GFP_NOWAIT ); caches++; }
@@ -3107,7 +3110,7 @@ void __init kmem_cache_init(void)/* Provide the correct kmalloc names now that the caches are up */ for (i =KMALLOC_SHIFT_LOW ; i <SLUB_PAGE_SHIFT ; i++) kmalloc_caches[i]. name = -kasprintf (GFP_KERNEL ,"kmalloc-%d" ,1 << i); +kasprintf (GFP_NOWAIT ,"kmalloc-%d" ,1 << i);#ifdef CONFIG_SMP register_cpu_notifier (&slab_notifier);
Signature · 1 line
1.6.0.4
Quoted 8 lines · Pekka J Enberg
Quoted 6 lines · Nick Piggin
Quoted 6 lines
Quoted 5 lines
The bootmem allocations of course are required because some hashes may need to be larger than
MAX_ORDERwithout using vmalloc.
kmem_cache_init(andmem_init, partially) sets up the kernel memory allocators...
Quoted 8 lines
Quoted 2 lines · Pekka Enberg
Quoted 2 lines
Which comment are you talking about? The "setup kernel memory allocators" one? Yeah, I should probably move it couple of lines down.
Quoted 1 line · Nick Piggin
Yes, just that one.
Here's an updated patch that does that. Thanks!
If we're gonna do that, we probably ought to have a comment.
/* These use large bootmem allocations and must precede kmem_cache_init */
Signature · 1 line
http://selenic.com : development and support for Mercurial and Linux
Quoted 14 lines · Pekka J Enberg
init/main.c +6
-0@@ -576,6 +576,22 @@ asmlinkage void __init start_kernel(void)setup_nr_cpu_ids ();smp_prepare_boot_cpu ();/* arch-specific boot-cpu hooks */ +build_all_zonelists (); +page_alloc_init (); + +printk (KERN_NOTICE "Kernel command line: %s\n" , boot_command_line); +parse_early_param (); + parse_args("Booting kernel", static_command_line, __start___param,
init/main.c asmlinkage void __init start_kernel(void) · line 594
Quoted 10 lines
init/main.c +10
-0+ __stop___param - __start___param, + &unknown_bootoption); +pidhash_init (); +vmalloc_init (); +vfs_caches_init_early (); +/* +* Set up kernel memory allocators +*/ +mem_init (); +kmem_cache_init ();
So what strikes me is a question:
- why do we want to do pidhash_init and vfs_caches_init_early() so early?
Yes, pidhash_init() now uses alloc_bootmem. It's an allocation that is not trivially small, but it's not humongous either (max 4096 hash list heads, one pointer each).
And vfs_caches_init_early() is actually doing some rather strange things, like doing a "alloc_large_system_hash()" but not unconditionally: it does it in the "late" initialization too, if not done early. inode_init_early does soemthing very similar (ie a _conditional_ early init).
So none of this seems to really get a huge advantage from the early init. There seems to be some subtle NUMA issues, but do we really want that? I get the feeling that nobody ever wanted to do it early, and then the NUMA people said "I don't wnt to do this early, but I don't want to touch the non-NUMA case, so I'll do it early for non-numa, and late for numa".
I'm also not entirely sure we really need to do vmalloc_init() that early, but I dunno. It also uses alloc_bootmem().
Linus
Hi Linus,
Quoted 28 lines · Linus Torvalds
Quoted 14 lines · Pekka J Enberg
init/main.c +6
-0@@ -576,6 +576,22 @@ asmlinkage void __init start_kernel(void)setup_nr_cpu_ids ();smp_prepare_boot_cpu ();/* arch-specific boot-cpu hooks */ +build_all_zonelists (); +page_alloc_init (); + +printk (KERN_NOTICE "Kernel command line: %s\n" , boot_command_line); +parse_early_param (); + parse_args("Booting kernel", static_command_line, __start___param,init/main.c asmlinkage void __init start_kernel(void) · line 594
Quoted 10 lines
init/main.c +10
-0+ __stop___param - __start___param, + &unknown_bootoption); +pidhash_init (); +vmalloc_init (); +vfs_caches_init_early (); +/* +* Set up kernel memory allocators +*/ +mem_init (); +kmem_cache_init ();So what strikes me is a question:
Quoted 3 lines
- why do we want to do
pidhash_initandvfs_caches_init_early()so early?Yes,
pidhash_init()now usesalloc_bootmem. It's an allocation that is not trivially small, but it's not humongous either (max 4096 hash list heads, one pointer each).
I can certainly fix that up to use kmalloc() or vmalloc(). I moved it because I wasn't sure how much it's actually allocating and wanted to do the conservative thing here.
Quoted 3 lines · Linus Torvalds
And
vfs_caches_init_early()is actually doing some rather strange things, like doing a "alloc_large_system_hash()" but not unconditionally: it does it in the "late" initialization too, if not done early.inode_init_earlydoes soemthing very similar (ie a _conditional_ early init).So none of this seems to really get a huge advantage from the early init. There seems to be some subtle NUMA issues, but do we really want that? I get the feeling that nobody ever wanted to do it early, and then the NUMA people said "I don't wnt to do this early, but I don't want to touch the non-NUMA case, so I'll do it early for non-numa, and late for numa".
SLUB does sysfs setup in kmem_cache_init() and if I saw some oopses if I don't call vfs_caches_init_early() first. I didn't look too closely, though.
Quoted 1 line · Linus Torvalds
I'm also not entirely sure we really need to do
vmalloc_init()that early, but I dunno. It also usesalloc_bootmem().
We can do that later but then we need to fix up vmalloc_init(). There's actually a patch floating around to do that.
Pekka
Quoted 3 lines · Pekka Enberg
We can do that later but then we need to fix up
vmalloc_init(). There's actually a patch floating around to do that.
Hey, all my suggestions were definitely for the "do it later". I'd hate to have this blow up to one large patch. I'm perfectly ok with the patch I replied to, I was more thinking that maybe there is then room for further (independent) fixups too.
So [vmalloc|pidhash|vfs_caches_eary]_init all fall under the "let's look at that too. If we start off with just the scheduler init ordering, that's good.
Linus
Hi Linus,
Quoted 3 lines · Linus Torvalds
Hey, all my suggestions were definitely for the "do it later". I'd hate to have this blow up to one large patch. I'm perfectly ok with the patch I replied to, I was more thinking that maybe there is then room for further (independent) fixups too.
So [vmalloc|pidhash|vfs_caches_eary]_init all fall under the "let's look at that too. If we start off with just the scheduler init ordering, that's good.
OK, makes sense. I'll try to sneak the current patch past Ingo first and then do the ones you pointed out as incremental patches.
Pekka
trying to put setup_percpu_areas() after mem_init/kmem_cache_init...
http://git.kernel.org/?p=linux/kernel/git/yinghai/linux-2.6-yinghai.git;a=summarygit://git.kernel.org/pub/scm/linux/kernel/git/yinghai/linux-2.6-yinghai.git
based on tip/master, on AMD 8 sockets, NUMA + SLUB.
got
[ 49.962048] Freeing unused kernel memory: 2564k freed INIT: [ 49.984095] BUG: unable to handle kernel paging request at ffff888687090860 [ 49.988020] IP: [<ffffffff81135249>] __slab_free+0x61/0x2ee [ 49.988020] PGD a067 PUD 0 [ 49.988020] Oops: 0002 [#1] SMP [ 49.988020] last sysfs file: [ 49.988020] CPU 0 [ 49.988020] Modules linked in: [ 49.988020] Pid: 1, comm: init Not tainted 2.6.30-rc7-tip-01641-g4c0d14e-dirty #285 Sun Fire X4600 M2 [ 49.988020] RIP: 0010:[<ffffffff81135249>] [<ffffffff81135249>] __slab_free+0x61/0x2ee [ 49.988020] RSP: 0018:ffff888021be5a08 EFLAGS: 00010046 [ 49.988020] RAX: 00000000cccccccc RBX: ffffffff8251ce60 RCX: ffffffff81185e56 [ 49.988020] RDX: 0000000000000000 RSI: ffffea0340d42110 RDI: ffffffff82247000 [ 49.988020] RBP: ffff888021be5a58 R08: 00000000cccccccc R09: 0000000000000000 [ 49.988020] R10: ffff885021be6200 R11: 000000002cce478a R12: ffffea0340d42110 [ 49.988020] R13: ffff888020a2a200 R14: ffffffff82247000 R15: ffffffff82247000 [ 49.988020] FS: 00000000006d1880(0063) GS:ffffc90000000000(0000) knlGS:0000000000000000 [ 49.988020] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 49.988020] CR2: ffff888687090860 CR3: 0000001020576000 CR4: 00000000000006f0 [ 49.988020] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 49.988020] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 [ 49.988020] Process init (pid: 1, threadinfo ffff888021be4000, task ffff887021be8000) [ 49.988020] Stack: [ 49.988020] ffff888021be5a58 cccccccc810ae203 ffffffff81185e56 000000002cce478a [ 49.988020] 000000002cce478a ffffffff8251ce60 ffffea0340d42110 ffff888020a2a200 [ 49.988020] 0000000000000286 ffffffff82247000 ffff888021be5aa8 ffffffff8113603d [ 49.988020] Call Trace: [ 49.988020] [<ffffffff81185e56>] ? free_proc_entry+0x79/0x94 [ 49.988020] [<ffffffff8113603d>] kfree+0x10c/0x14a [ 49.988020] [<ffffffff81185e56>] ? free_proc_entry+0x79/0x94 [ 49.988020] [<ffffffff81185e56>] free_proc_entry+0x79/0x94 [ 49.988020] [<ffffffff81186094>] remove_proc_entry+0x223/0x249 [ 49.988020] [<ffffffff810524d5>] ? mask_IO_APIC_irq_desc+0x36/0x112 [ 49.988020] [<ffffffff81a8a887>] ? _spin_unlock_irqrestore+0x50/0x76 [ 49.988020] [<ffffffff810acd60>] ? trace_hardirqs_off+0x20/0x36 [ 49.988020] [<ffffffff81a8a887>] ? _spin_unlock_irqrestore+0x50/0x76 [ 49.988020] [<ffffffff810add3b>] ? mark_held_locks+0x60/0x96 [ 49.988020] [<ffffffff81a8a88e>] ? _spin_unlock_irqrestore+0x57/0x76 [ 49.988020] [<ffffffff810ae065>] ? trace_hardirqs_on_caller+0x13b/0x17d [ 49.988020] [<ffffffff810cef38>] unregister_handler_proc+0x46/0x61 [ 49.988020] [<ffffffff810cd36e>] __free_irq+0x135/0x191 [ 49.988020] [<ffffffff810cd3eb>] free_irq+0x21/0x3f [ 49.988020] [<ffffffff814f3bee>] serial8250_shutdown+0x130/0x170 [ 49.988020] [<ffffffff814ef6c2>] uart_shutdown+0xd0/0x124 [ 49.988020] [<ffffffff814f0c41>] uart_close+0x14c/0x1f7 [ 49.988020] [<ffffffff814bbffa>] ? tty_fasync+0x11b/0x13e [ 49.988020] [<ffffffff814befe7>] tty_release_dev+0x1d3/0x51c [ 49.988020] [<ffffffff8113c5f2>] ? sys_close+0x45/0x112 [ 49.988020] [<ffffffff814bf35d>] tty_release+0x2d/0x4f [ 49.988020] [<ffffffff8113fd24>] __fput+0xff/0x1d1 [ 49.988020] [<ffffffff8113fe26>] fput+0x30/0x46 [ 49.988020] [<ffffffff8113c58c>] filp_close+0x78/0x99 [ 49.988020] [<ffffffff81a8a8e7>] ? _spin_unlock+0x3a/0x55 [ 49.988020] [<ffffffff8113c66a>] sys_close+0xbd/0x112 [ 49.988020] [<ffffffff81034edb>] system_call_fastpath+0x16/0x1b [ 49.988020] Code: eb 09 f3 90 41 f6 04 24 01 75 f7 f0 41 0f ba 2c 24 00 19 c0 85 c0 75 ec 41 f6 04 24 02 0f 85 ae 00 00 00 8b 45 bc 49 8b 54 24 50 <49> 89 54 c5 00 41 8b 44 24 0c 4d 89 6c 24 50 8d 48 ff 49 8b 04 [ 49.988020] RIP [<ffffffff81135249>] __slab_free+0x61/0x2ee [ 49.988020] RSP <ffff888021be5a08> [ 49.988020] CR2: ffff888687090860 [ 49.988020] ---[ end trace 4dde5b4c04e715b6 ]---
Quoted 6 lines · Pekka Enberg
Quoted 5 lines · Linus Torvalds
And
vfs_caches_init_early()is actually doing some rather strange things, like doing a "alloc_large_system_hash()" but not unconditionally: it does it in the "late" initialization too, if not done early.inode_init_earlydoes soemthing very similar (ie a _conditional_ early init).
Quoted 8 lines
Quoted 5 lines
So none of this seems to really get a huge advantage from the early init. There seems to be some subtle NUMA issues, but do we really want that? I get the feeling that nobody ever wanted to do it early, and then the NUMA people said "I don't wnt to do this early, but I don't want to touch the non-NUMA case, so I'll do it early for non-numa, and late for numa".
SLUB does sysfs setup in
kmem_cache_init()and if I saw some oopses if I don't callvfs_caches_init_early()first. I didn't look too closely, though.
Did you also test the NUMA/hashdist case? vfs_caches_init_early doesn't do much in that case.
I would say it is much more robust to do sysfs setup later if we move the slab setup so early. Probably it is just quite lucky not to explode in the !numa case because the vfs needs quite a bit of setting up...
Quoted 13 lines · Nick Piggin
Quoted 12 lines · Pekka Enberg
Quoted 10 lines · Linus Torvalds
And
vfs_caches_init_early()is actually doing some rather strange things, like doing a "alloc_large_system_hash()" but not unconditionally: it does it in the "late" initialization too, if not done early.inode_init_earlydoes soemthing very similar (ie a _conditional_ early init).So none of this seems to really get a huge advantage from the early init. There seems to be some subtle NUMA issues, but do we really want that? I get the feeling that nobody ever wanted to do it early, and then the NUMA people said "I don't wnt to do this early, but I don't want to touch the non-NUMA case, so I'll do it early for non-numa, and late for numa".
Quoted 5 lines
Quoted 2 lines
SLUB does sysfs setup in
kmem_cache_init()and if I saw some oopses if I don't callvfs_caches_init_early()first. I didn't look too closely, though.Did you also test the NUMA/hashdist case?
vfs_caches_init_earlydoesn't do much in that case.
No, I tested UMA only.
Quoted 3 lines · Nick Piggin
I would say it is much more robust to do sysfs setup later if we move the slab setup so early. Probably it is just quite lucky not to explode in the !numa case because the vfs needs quite a bit of setting up...
That should not be an issue. SLUB already defers sysfs registration until slab_sysfs_init() initcall has been run. So my patches have zero change in how SLUB interracts with sysfs, actually.
Pekka
Quoted 22 lines · Pekka Enberg
Quoted 13 lines · Nick Piggin
Quoted 12 lines · Pekka Enberg
Quoted 10 lines · Linus Torvalds
And
vfs_caches_init_early()is actually doing some rather strange things, like doing a "alloc_large_system_hash()" but not unconditionally: it does it in the "late" initialization too, if not done early.inode_init_earlydoes soemthing very similar (ie a _conditional_ early init).So none of this seems to really get a huge advantage from the early init. There seems to be some subtle NUMA issues, but do we really want that? I get the feeling that nobody ever wanted to do it early, and then the NUMA people said "I don't wnt to do this early, but I don't want to touch the non-NUMA case, so I'll do it early for non-numa, and late for numa".
Quoted 5 lines
Quoted 2 lines
SLUB does sysfs setup in
kmem_cache_init()and if I saw some oopses if I don't callvfs_caches_init_early()first. I didn't look too closely, though.Did you also test the NUMA/hashdist case?
vfs_caches_init_earlydoesn't do much in that case.No, I tested UMA only.
Quoted 8 lines
Quoted 3 lines · Nick Piggin
I would say it is much more robust to do sysfs setup later if we move the slab setup so early. Probably it is just quite lucky not to explode in the !numa case because the vfs needs quite a bit of setting up...
That should not be an issue. SLUB already defers sysfs registration until
slab_sysfs_init()initcall has been run. So my patches have zero change in how SLUB interracts with sysfs, actually.
Oh right, I didn't actually look. An initcall should be fine, but I wonder why it is crashing if you move it before vfs_caches_init_early? Those just allocate inode and dentry hash tables...
Quoted 3 lines · Nick Piggin
Oh right, I didn't actually look. An initcall should be fine, but I wonder why it is crashing if you move it before
vfs_caches_init_early? Those just allocate inode and dentry hash tables...
That's because we pass HASH_EARLY to alloc_large_system_hash(). Should be easy enough to fix, though.
Quoted 30 lines · Linus Torvalds
Quoted 14 lines · Pekka J Enberg
init/main.c +6
-0@@ -576,6 +576,22 @@ asmlinkage void __init start_kernel(void)setup_nr_cpu_ids ();smp_prepare_boot_cpu ();/* arch-specific boot-cpu hooks */ +build_all_zonelists (); +page_alloc_init (); + +printk (KERN_NOTICE "Kernel command line: %s\n" , boot_command_line); +parse_early_param (); + parse_args("Booting kernel", static_command_line, __start___param,init/main.c asmlinkage void __init start_kernel(void) · line 594
Quoted 10 lines
init/main.c +10
-0+ __stop___param - __start___param, + &unknown_bootoption); +pidhash_init (); +vmalloc_init (); +vfs_caches_init_early (); +/* +* Set up kernel memory allocators +*/ +mem_init (); +kmem_cache_init ();So what strikes me is a question:
Quoted 5 lines
- why do we want to do
pidhash_initandvfs_caches_init_early()so early?Yes,
pidhash_init()now usesalloc_bootmem. It's an allocation that is not trivially small, but it's not humongous either (max 4096 hash list heads, one pointer each).
It would be nice to use the regular page allocator for pidhash_init; For my case, I have this patch floating around for a long time which can make this (among other things) dynamically resizable without using locking, and avoiding the special case for the bootmem allocated hash would be good.
Quoted 10 lines
And
vfs_caches_init_early()is actually doing some rather strange things, like doing a "alloc_large_system_hash()" but not unconditionally: it does it in the "late" initialization too, if not done early.inode_init_earlydoes soemthing very similar (ie a _conditional_ early init).So none of this seems to really get a huge advantage from the early init. There seems to be some subtle NUMA issues, but do we really want that? I get the feeling that nobody ever wanted to do it early, and then the NUMA people said "I don't wnt to do this early, but I don't want to touch the non-NUMA case, so I'll do it early for non-numa, and late for numa".
vfs_caches_init_early wants to allocate with bootmem so it can get
Quoted 1 line
=
MAX_ORDERcache size in the kernel direct mapping. In the NUMA
case, it is more important to spread the memory usage and utilisation over nodes I guess so they use vmalloc for that. Bootmem and vmalloc are not available at the same time, so it has to be 2 cases.
Quoted 2 lines
I'm also not entirely sure we really need to do
vmalloc_init()that early, but I dunno. It also usesalloc_bootmem().
Probably not. vmalloc doesn't really work without the page allocator and slab allocator already up, so it can probably be moved after them.
Quoted 10 lines · Pekka Enberg
Quoted 9 lines · Nick Piggin
Quoted 5 lines
The bootmem allocations of course are required because some hashes may need to be larger than
MAX_ORDERwithout using vmalloc.
kmem_cache_init(andmem_init, partially) sets up the kernel memory allocators...Which comment are you talking about? The "setup kernel memory allocators" one? Yeah, I should probably move it couple of lines down.
Quoted 1 line · Nick Piggin
Yes, just that one.
And oh, we probably want to do this too while at it. Nick?
Quoted 1 line
From
b211497261670ff6f9af6b8b0fab429e848ccc87Mon Sep 17 00:00:00 2001
From: Pekka Enberg <[email protected]>
Date: Mon, 25 May 2009 15:01:35 +0300
Subject: [PATCH] vmalloc: use kzalloc() instead of alloc_bootmem()
We can call vmalloc_init() after kmem_cache_init() and use kzalloc() instead of the bootmem allocator when initializing vmalloc data structures.
--- init/main.c | 2 +- mm/vmalloc.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-)
init/main.c +1 -1
@@ -585,13 +585,13 @@ asmlinkage void __init start_kernel(void) __stop___param - __start___param, &unknown_bootoption);pidhash_init (); -vmalloc_init ();vfs_caches_init_early ();/* * Set up kernel memory allocators */ mem_init ();kmem_cache_init (); +vmalloc_init ();/* * Set up the scheduler prior starting any interrupts (such as the * timer interrupt). Full topology setup happens at smp_init()
mm/vmalloc.c +1 -2
@@ -23,7 +23,6 @@#include <linux/rbtree.h> #include <linux/radix-tree.h> #include <linux/rcupdate.h> -#include <linux/bootmem.h> #include <linux/pfn.h> #include <asm/atomic.h>
@@ -1032,7 +1031,7 @@ void __init vmalloc_init(void)/* Import existing vmlist entries. */ for (tmp = vmlist; tmp; tmp = tmp->next) { - va =alloc_bootmem (sizeof (struct vmap_area )); + va =kzalloc (sizeof (struct vmap_area ),GFP_NOWAIT ); va->flags = tmp->flags |VM_VM_AREA ; va->va_start = (unsigned long )tmp->addr; va->va_end = va->va_start + tmp->size;
Signature · 1 line
1.6.0.4
Quoted 36 lines · Pekka Enberg
Quoted 10 lines · Pekka Enberg
Quoted 9 lines · Nick Piggin
Quoted 5 lines
The bootmem allocations of course are required because some hashes may need to be larger than
MAX_ORDERwithout using vmalloc.
kmem_cache_init(andmem_init, partially) sets up the kernel memory allocators...Which comment are you talking about? The "setup kernel memory allocators" one? Yeah, I should probably move it couple of lines down.
Quoted 1 line · Nick Piggin
Yes, just that one.
And oh, we probably want to do this too while at it. Nick?
Quoted 1 line
From
b211497261670ff6f9af6b8b0fab429e848ccc87Mon Sep 17 00:00:00 2001From: Pekka Enberg <[email protected]>
Date: Mon, 25 May 2009 15:01:35 +0300
Subject: [PATCH] vmalloc: usekzalloc()instead ofalloc_bootmem()We can call
vmalloc_init()afterkmem_cache_init()and usekzalloc()instead of the bootmem allocator when initializing vmalloc data structures.Signed-off-by Pekka Enberg <[email protected]>--- init/main.c | 2 +- mm/vmalloc.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-)init/main.c +0
-0@@ -585,13 +585,13 @@ asmlinkage void __init start_kernel(void) __stop___param - __start___param,
init/main.c asmlinkage void __init start_kernel(void) · line 594
Quoted 10 lines
init/main.c +1
-1&unknown_bootoption);pidhash_init (); -vmalloc_init ();vfs_caches_init_early ();/* * Set up kernel memory allocators */ mem_init ();kmem_cache_init (); +vmalloc_init ();
vmalloc won't work fully without slab and page allocators anyway, so this is conceptually a bugfix I think. Ack.
Quoted 21 lines · Pekka J Enberg
Quoted 10 lines · Ingo Molnar
Ok, i think this all looks pretty realistic - but there's quite a bit of layering on top of pending changes in the x86 and irq trees. We could do this on top of those topic branches in -tip, and rebase in the merge window. Or delay it to .32.
... plus i think we are _very_ close to being able to remove all of bootmem on x86 (with some compatibility/migration mechanism in place). Which bootmem calls do we have before kmalloc init with Pekka's patch applied? I think it's mostly the page table init code.
Quoted 4 lines
( beyond the page allocator internal use - where we could use
straight e820 based APIs that clip memory off from the beginning of existing e820 RAM ranges - enriched with NUMA/SRAT locality info. )OK, here's a version of the patch with Yinghai's rebasing on top of tip/master. All in-kernel memory allocators boot cleanly now on my configuration (I did not try SLQB but it probably needs fixing). I would appreciate if someone gave SLAB+NUMA configuration a try.
Quoted 5 lines
Ingo, I don't really want to carry this in slab.git so perhaps you could put this and Yinghai's irq init cleanup in a "earlyslab" branch in tip for testing?
And oh, Christoph/Matt, can I have your NAK/ACK for this patch?
Looks ok to me, though I'd probably split it into three parts (slab, slub, init).
For future reference, 0xbaadf00d ought to move into poison.h.
Signature · 1 line
http://selenic.com : development and support for Mercurial and Linux
Quoted 4 lines · Matt Mackall
Quoted 1 line
And oh, Christoph/Matt, can I have your NAK/ACK for this patch?
Looks ok to me, though I'd probably split it into three parts (slab, slub, init).
Yeah, the patch is pretty big but I don't think we can do that. It would break git bisect.
Pekka
Quoted 4 lines · Matt Mackall
Quoted 1 line
And oh, Christoph/Matt, can I have your NAK/ACK for this patch?
Looks ok to me, though I'd probably split it into three parts (slab, slub, init).
Could we document the core boot strap procedure some more? I think we will have slight breakage in various parts if we change the order. If we had some design to guide then it may be easier to bring all the components in order.
Quoted 4 lines · Christoph Lameter
Could we document the core boot strap procedure some more? I think we will have slight breakage in various parts if we change the order. If we had some design to guide then it may be easier to bring all the components in order.
Hmm. I am avoiding all the hard issues by letting all the architecture specific early boot code run before I set up the kernel memory allocators. The only thing that changes order is init_IRQ() which should be fine.
Pekka
Quoted 1 line · Pekka J Enberg
And oh, Christoph/Matt, can I have your NAK/ACK for this patch?
You are dinking with something very sensitive here. There are numerous dependencies on the exact sequence in the boot code and that sequence may vary from arch to arch. It wont be easy to ack.
Argh. Meeting. I hope to get back to this later.
H. Peter Anvin
reviewerQuoted 11 lines · Ingo Molnar
Would it be possible to restructure things to move kmalloc init to before IRQ init as well? We have a couple of uglinesses there too.
Conceptually, memory should be the first thing set up in general, in a kernel. It does not need IRQs, timers, the scheduler or any of the IO facilities and abstractions. All of them need memory though - and as Linux scales to more and more hardware via the same single image, so will we get more and more dynamic concepts like
cpumask_var_tand sparse-irqs, which want to allocate very early.
Quoted 8 lines
setup_arch()is one huge function that sets up all architecture details at once - but if we split a separatesetup_arch_mem()out of it, and left the rest insetup_arch(and moved it further down), we could remove much of bootmem (especially the ugly uses).This might even be doable realistically, and we could thus librarize bootmem and eliminate it from x86 at least. Perhaps.
The only thing that might make sense to set up before memory might be exceptions (as opposed to interrupts), but both of those should be doable very very early.
-hpa
Signature · 2 lines
H. Peter Anvin, Intel Open Source Technology Center I work for Intel. I don't speak on their behalf.
Quoted 17 lines · H. Peter Anvin
Quoted 16 lines · Ingo Molnar
Would it be possible to restructure things to move kmalloc init to before IRQ init as well? We have a couple of uglinesses there too.
Conceptually, memory should be the first thing set up in general, in a kernel. It does not need IRQs, timers, the scheduler or any of the IO facilities and abstractions. All of them need memory though - and as Linux scales to more and more hardware via the same single image, so will we get more and more dynamic concepts like
cpumask_var_tand sparse-irqs, which want to allocate very early.
setup_arch()is one huge function that sets up all architecture details at once - but if we split a separatesetup_arch_mem()out of it, and left the rest insetup_arch(and moved it further down), we could remove much of bootmem (especially the ugly uses).
Quoted 7 lines
Quoted 3 lines
This might even be doable realistically, and we could thus librarize bootmem and eliminate it from x86 at least. Perhaps.
The only thing that might make sense to set up before memory might be exceptions (as opposed to interrupts), but both of those should be doable very very early.
Yeah. And we already have early exception handlers to help debugging so there's practically no dependency on memory init. (other than the full kernel image being executable by the CPU - so some very minimal memory/paging init is necessary)
Ingo
Quoted 16 lines · H. Peter Anvin
Quoted 15 lines · Ingo Molnar
Would it be possible to restructure things to move kmalloc init to before IRQ init as well? We have a couple of uglinesses there too.
Conceptually, memory should be the first thing set up in general, in a kernel. It does not need IRQs, timers, the scheduler or any of the IO facilities and abstractions. All of them need memory though - and as Linux scales to more and more hardware via the same single image, so will we get more and more dynamic concepts like
cpumask_var_tand sparse-irqs, which want to allocate very early.
setup_arch()is one huge function that sets up all architecture details at once - but if we split a separatesetup_arch_mem()out of it, and left the rest insetup_arch(and moved it further down), we could remove much of bootmem (especially the ugly uses).
Quoted 8 lines
Quoted 3 lines
This might even be doable realistically, and we could thus librarize bootmem and eliminate it from x86 at least. Perhaps.
The only thing that might make sense to set up before memory might be exceptions (as opposed to interrupts), but both of those should be doable very very early.
put trap_init() right after setup_arch() in start_kernel()?
YH
Quoted 3 lines · Ingo Molnar
Would it be possible to restructure things to move kmalloc init to before IRQ init as well? We have a couple of uglinesses there too.
Amen :-)
Quoted 7 lines
Conceptually, memory should be the first thing set up in general, in a kernel. It does not need IRQs, timers, the scheduler or any of the IO facilities and abstractions. All of them need memory though - and as Linux scales to more and more hardware via the same single image, so will we get more and more dynamic concepts like
cpumask_var_tand sparse-irqs, which want to allocate very early.
Quoted 7 lines
setup_arch()is one huge function that sets up all architecture details at once - but if we split a separatesetup_arch_mem()out of it, and left the rest insetup_arch(and moved it further down), we could remove much of bootmem (especially the ugly uses).This might even be doable realistically, and we could thus librarize bootmem and eliminate it from x86 at least. Perhaps.
Yup, see my earlier email. Archs like x86 and powerpc already have a low level allocator they can use to allocate the mem_map etc... so bootmem really becomes redundant.
Cheers
Ben.
Quoted 4 lines · Ingo Molnar
setup_arch()is one huge function that sets up all architecture details at once - but if we split a separatesetup_arch_mem()out of it, and left the rest insetup_arch(and moved it further down), we could remove much of bootmem (especially the ugly uses).
cmdline parsing needs to be available even before mem setup. I waded into this earlier and broke almost every arch, but if there's renewed enthusiasm I could dig out those patches.
(Search for "arch_get_boot_cmdline()").
Cheers,
Rusty.
A message that is not in the archive (sent off-list or not yet received): [email protected]
A message that is not in the archive (sent off-list or not yet received): [email protected]
PATCH 1/1 x86: enable_update_mptable should MACRO
instead of declare as one inline function.
because other case that is variable
--- arch/x86/include/asm/mpspec.h | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-)
arch/x86/include/asm/mpspec.h +2 -9
@@ -61,9 +61,11 @@ extern void get_smp_config(void);#ifdef CONFIG_X86_MPPARSE extern void find_smp_config (void );extern void early_reserve_e820_mpc_new (void ); +extern int enable_update_mptable;#else static inline void find_smp_config (void ) { }static inline void early_reserve_e820_mpc_new (void ) { } +#define enable_update_mptable 0#endif void __cpuinit generic_processor_info (int apicid,int version);
@@ -87,15 +89,6 @@ static inline int acpi_probe_gsi(void) }#endif /* CONFIG_ACPI */ -#ifdef CONFIG_X86_MPPARSE -extern int enable_update_mptable; -#else -static inline int enable_update_mptable (void ) -{ -return 0 ; -} -#endif -#define PHYSID_ARRAY_SIZE BITS_TO_LONGS(MAX_APICS)struct physid_mask {
[tip:irq/numa] x86: enable_update_mptable should be a macro
Commit-ID: abfe0af9813153bae8c85d9bac966bafcb8ddab1 Gitweb: http://git.kernel.org/tip/abfe0af9813153bae8c85d9bac966bafcb8ddab1 Author: Yinghai Lu <[email protected]> AuthorDate: Wed, 20 May 2009 00:37:40 -0700 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 28 May 2009 01:59:05 +0200
x86: enable_update_mptable should be a macro
instead of declaring one variant as an inline function... because other case is a variable
LKML-Reference: <[email protected]>
--- arch/x86/include/asm/mpspec.h | 11 ++--------- 1 files changed, 2 insertions(+), 9 deletions(-)
arch/x86/include/asm/mpspec.h +2 -9
@@ -61,9 +61,11 @@ extern void get_smp_config(void);#ifdef CONFIG_X86_MPPARSE extern void find_smp_config (void );extern void early_reserve_e820_mpc_new (void ); +extern int enable_update_mptable;#else static inline void find_smp_config (void ) { }static inline void early_reserve_e820_mpc_new (void ) { } +#define enable_update_mptable 0#endif void __cpuinit generic_processor_info (int apicid,int version);
@@ -87,15 +89,6 @@ static inline int acpi_probe_gsi(void) }#endif /* CONFIG_ACPI */ -#ifdef CONFIG_X86_MPPARSE -extern int enable_update_mptable; -#else -static inline int enable_update_mptable (void ) -{ -return 0 ; -} -#endif -#define PHYSID_ARRAY_SIZE BITS_TO_LONGS(MAX_APICS)struct physid_mask {
[tip:sched/core] sched: properly define the sched_group::cpumask and sched_domain::span fields
In reply to Linus TorvaldsCommit-ID: 4200efd9acda4accf24640f1e77d24fdcdb524df Gitweb: http://git.kernel.org/tip/4200efd9acda4accf24640f1e77d24fdcdb524df Author: Ingo Molnar <[email protected]> AuthorDate: Tue, 19 May 2009 09:22:19 +0200 Committer: Ingo Molnar <[email protected]> CommitDate: Tue, 19 May 2009 09:22:19 +0200
sched: properly define the sched_group::cpumask and sched_domain::span fields
Properly document the variable-size structure tricks we are doing wrt. struct sched_group and sched_domain, and use the field[0] GCC extension instead of defining a vla array.
Dont use unions for this, as pointed out by Linus.
[ Impact: cleanup, un-confuse Sparse and LLVM ]
LKML-Reference: <[email protected]>
--- include/linux/sched.h | 25 ++++++++++++++++++++++--- kernel/sched.c | 5 +++-- 2 files changed, 25 insertions(+), 5 deletions(-)
include/linux/sched.h +22 -3
@@ -839,7 +839,17 @@ struct sched_group {*/ u32 reciprocal_cpu_power; -unsigned long cpumask[]; +/* +* The CPUs this group covers. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +* +* It is also be embedded into static data structures at build +* time. (See 'struct static_sched_group' in kernel/sched.c) +*/ +unsigned long cpumask[0 ]; };static inline struct cpumask *sched_group_cpus (struct sched_group *sg)
@@ -925,8 +935,17 @@ struct sched_domain {char *name;#endif -/* span of all CPUs in this domain */ -unsigned long span[]; +/* +* Span of all CPUs in this domain. +* +* NOTE: this field is variable length. (Allocated dynamically +* by attaching extra space to the end of the structure, +* depending on how many CPUs the kernel has booted up with) +* +* It is also be embedded into static data structures at build +* time. (See 'struct static_sched_domain' in kernel/sched.c) +*/ +unsigned long span[0 ]; };static inline struct cpumask *sched_domain_span (struct sched_domain *sd)
kernel/sched.c +3 -2
@@ -7948,8 +7948,9 @@ int sched_smt_power_savings = 0, sched_mc_power_savings = 0;/* * The cpus mask in sched_group and sched_domain hangs off the end. -* FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space -* for nr_cpu_ids < CONFIG_NR_CPUS. +* +* ( See the the comments in include/linux/sched.h:struct sched_group +* and struct sched_domain. ) */ struct static_sched_group {struct sched_group sg;
Linus,
Please pull the latest sched-fixes-for-linus-2 git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip.git sched-fixes-for-linus-2
This excludes the "sched: avoid flexible array member inside struct (gcc extension)" commit you objected to. Did a test-build and a test-boot of this, just in case...
Thanks,
Ingo
------------------>
Ron (1):
sched: Fix fallback sched_clock()'s offset when using jiffies
kernel/sched_clock.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-)
kernel/sched_clock.c +2 -1
@@ -38,7 +38,8 @@*/ unsigned long long __attribute__((weak))sched_clock (void ) { -return (unsigned long long )jiffies * (NSEC_PER_SEC /HZ ); +return (unsigned long long )(jiffies -INITIAL_JIFFIES ) + * (NSEC_PER_SEC /HZ ); }static __read_mostly int sched_clock_running;