i386: always use 4k stacks
PATCH 1/1 i386: always use 4k stacks
It seems most problems with 4k stacks are already resolved.
I'd like to see this patch to always use 4k stacks in -mm now for finding any remaining problems before submitting this patch for 2.6.16.
---
arch/i386/Kconfig.debug | 10 ---------- arch/i386/kernel/irq.c | 10 ---------- include/asm-i386/irq.h | 11 +++-------- include/asm-i386/module.h | 8 +------- include/asm-i386/thread_info.h | 6 +----- 5 files changed, 5 insertions(+), 40 deletions(-)
arch/i386/Kconfig.debug renamed from arch/i386/Kconfig.debug.old +0 -10
@@ -52,16 +52,6 @@ portion of the kernel code won't be covered by a 2MB TLB anymore. If in doubt, say "N". -config 4KSTACKS - bool "Use 4Kb for kernel stacks instead of 8Kb" - depends on DEBUG_KERNEL - help - If you say Y here the kernel will use a 4Kb stacksize for the - kernel stack attached to each process/thread. This facilitates - running more threads on a system and also reduces the pressure - on the VM subsystem for higher order allocations. This option - will also use IRQ stacks to compensate for the reduced stackspace. - config X86_FIND_SMP_CONFIG bool depends on X86_LOCAL_APIC || X86_VOYAGER
include/asm-i386/irq.h renamed from include/asm-i386/irq.h.old +3 -8
@@ -27,14 +27,9 @@# define ARCH_HAS_NMI_WATCHDOG /* See include/linux/nmi.h */ #endif -#ifdef CONFIG_4KSTACKS -extern void irq_ctx_init (int cpu); -extern void irq_ctx_exit (int cpu); -# define __ARCH_HAS_DO_SOFTIRQ -#else -# define irq_ctx_init (cpu) do { } while (0) -# define irq_ctx_exit (cpu) do { } while (0) -#endif +extern void irq_ctx_init (int cpu); +extern void irq_ctx_exit (int cpu); +#define __ARCH_HAS_DO_SOFTIRQ#ifdef CONFIG_IRQBALANCE extern int irqbalance_disable (char *str);
include/asm-i386/thread_info.h renamed from include/asm-i386/thread_info.h.old +1 -5
@@ -53,11 +53,7 @@#endif #define PREEMPT_ACTIVE 0x10000000 -#ifdef CONFIG_4KSTACKS -#define THREAD_SIZE (4096) -#else -#define THREAD_SIZE (8192) -#endif +#define THREAD_SIZE (4096)#define STACK_WARN (THREAD_SIZE/8)/*
include/asm-i386/module.h renamed from include/asm-i386/module.h.old +1 -7
@@ -64,12 +64,6 @@#define MODULE_REGPARM ""#endif -#ifdef CONFIG_4KSTACKS -#define MODULE_STACKSIZE "4KSTACKS " -#else -#define MODULE_STACKSIZE "" -#endif - -#define MODULE_ARCH_VERMAGIC MODULE_PROC_FAMILY MODULE_REGPARM MODULE_STACKSIZE +#define MODULE_ARCH_VERMAGIC MODULE_PROC_FAMILY MODULE_REGPARM#endif /* _ASM_I386_MODULE_H */
arch/i386/kernel/irq.c renamed from arch/i386/kernel/irq.c.old +0 -10
@@ -33,7 +33,6 @@ }#endif -#ifdef CONFIG_4KSTACKS /* * per-CPU IRQ handling contexts (thread information and stack) */
@@ -44,7 +43,6 @@static union irq_ctx *hardirq_ctx[NR_CPUS ];static union irq_ctx *softirq_ctx[NR_CPUS ]; -#endif /* * do_IRQ handles all normal device IRQ's (the special
@@ -58,10 +56,8 @@#ifdef CONFIG_DEBUG_PREEMPT u32 count =preempt_count () &PREEMPT_MASK ;#endif -#ifdef CONFIG_4KSTACKS union irq_ctx *curctx, *irqctx;u32 *isp; -#endif irq_enter ();#ifdef CONFIG_DEBUG_STACKOVERFLOW
@@ -79,8 +75,6 @@ }#endif -#ifdef CONFIG_4KSTACKS - curctx = (union irq_ctx *)current_thread_info (); irqctx = hardirq_ctx[smp_processor_id ()];
@@ -118,7 +112,6 @@ irqctx->tinfo.preempt_count -= count;#endif }else -#endif __do_IRQ (irq, regs);irq_exit ();
@@ -128,8 +121,6 @@return 1 ; } -#ifdef CONFIG_4KSTACKS -/* * These should really be __section__(".bss.page_aligned") as well, but * gcc's 3.0 and earlier don't handle that correctly.
@@ -228,7 +219,6 @@ }EXPORT_SYMBOL (do_softirq); -#endif /* * Interrupt statistics:
Quoted 8 lines · Adrian Bunk
It seems most problems with 4k stacks are already resolved.
I'd like to see this patch to always use 4k stacks in -mm now for finding any remaining problems before submitting this patch for 2.6.16.
Signed-off-by Adrian Bunk <[email protected]>
Quoted 1 line
Quoted 5 lines · Adrian Bunk
It seems most problems with 4k stacks are already resolved.
I'd like to see this patch to always use 4k stacks in -mm now for finding any remaining problems before submitting this patch for 2.6.16.
Has the block layer been remade to a serial approach? Last time I checked you could exhaust the stack fairly easy by layering a couple of block subsystems on each other (usb+scsi+raid+lvm+xfs+nfs to make things as horrible as possible).
Rgds
Pierre
Quoted 9 lines · Pierre Ossman
Quoted 6 lines · Adrian Bunk
It seems most problems with 4k stacks are already resolved.
I'd like to see this patch to always use 4k stacks in -mm now for finding any remaining problems before submitting this patch for 2.6.16.
Has the block layer been remade to a serial approach?
yes.
Quoted 9 lines · Arjan van de Ven
Quoted 3 lines · Pierre Ossman
Has the block layer been remade to a serial approach?
yes.
Great. Then please excuse my uninformed objections. :)
Rgds
Pierre
Quoted 4 lines · Arjan van de Ven
Quoted 3 lines · Pierre Ossman
Quoted 2 lines · Adrian Bunk
It seems most problems with 4k stacks are already resolved.
Quoted 8 lines
Quoted 6 lines
Quoted 4 lines
I'd like to see this patch to always use 4k stacks in -mm now for finding any remaining problems before submitting this patch for 2.6.16.
Has the block layer been remade to a serial approach?
yes.
Not in mainline it hasn't.
Are there any recent benchmarks of 4kb vs 8kb stacks? Is anyone shipping 4kb stack kernels?
Signature · 1 line
Jens Axboe
Quoted 10 lines · Jens Axboe
Quoted 9 lines · Arjan van de Ven
Quoted 8 lines · Pierre Ossman
Quoted 6 lines · Adrian Bunk
It seems most problems with 4k stacks are already resolved.
I'd like to see this patch to always use 4k stacks in -mm now for finding any remaining problems before submitting this patch for 2.6.16.
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
Has the block layer been remade to a serial approach?
yes.
Not in mainline it hasn't.
well the patch was for -mm ;)
Quoted 2 lines
Are there any recent benchmarks of 4kb vs 8kb stacks?
not sure; I do know that it very much helps java (many more threads possible) and the VM (far less order 1 allocs). In addition the 4Kb allocation can be satisfied with the per cpu list of free 4Kb pages, while obviously an order 1 cannot and has to go global.
Quoted 1 line
Is anyone shipping 4kb stack kernels?
Both Fedora and RHEL are shipping this for all 2.6 based versions (eg FC2 and all later, RHEL4)
Quoted 11 lines · Arjan van de Ven
Quoted 10 lines · Jens Axboe
Quoted 9 lines · Arjan van de Ven
Quoted 8 lines · Pierre Ossman
Quoted 6 lines · Adrian Bunk
It seems most problems with 4k stacks are already resolved.
I'd like to see this patch to always use 4k stacks in -mm now for finding any remaining problems before submitting this patch for 2.6.16.
Quoted 7 lines
Quoted 5 lines
Quoted 3 lines
Quoted 1 line
Has the block layer been remade to a serial approach?
yes.
Not in mainline it hasn't.
well the patch was for -mm ;)
ok :)
Quoted 7 lines
Quoted 2 lines
Are there any recent benchmarks of 4kb vs 8kb stacks?
not sure; I do know that it very much helps java (many more threads possible) and the VM (far less order 1 allocs). In addition the 4Kb allocation can be satisfied with the per cpu list of free 4Kb pages, while obviously an order 1 cannot and has to go global.
I realize it has nice advantages in theory, just wondering if anyone has done a performance analysis of 4kb vs 8kb stacks lately (or at all?).
Quoted 4 lines
Quoted 1 line
Is anyone shipping 4kb stack kernels?
Both Fedora and RHEL are shipping this for all 2.6 based versions (eg FC2 and all later, RHEL4)
Cool, that's the kind of testing coverage I was hoping for.
Signature · 1 line
Jens Axboe
Quoted 8 lines · Jens Axboe
Quoted 5 lines
not sure; I do know that it very much helps java (many more threads possible) and the VM (far less order 1 allocs). In addition the 4Kb allocation can be satisfied with the per cpu list of free 4Kb pages, while obviously an order 1 cannot and has to go global.
I realize it has nice advantages in theory, just wondering if anyone has done a performance analysis of 4kb vs 8kb stacks lately (or at all?).
I don't think at least anyone at RH has done any; the functionality gain was already enough for us. One item I missed: in the many-thread cases, you also save a lot of memory that can now be used for pagecache; this won't of course be visible in a microbenchmark but should help system wide.
Also in the implementation I don't see any way 4Kb stacks could show up in any benchmarks as negative; there are only 4 or 5 extra instructions in any path, and afaics no cache downsides (in fact the same irq stack memory is now reused for irqs instead of threadstack-du-jour, so less footprint/hotter caches)
Quoted 16 lines · Arjan van de Ven
Quoted 8 lines · Jens Axboe
Quoted 5 lines
not sure; I do know that it very much helps java (many more threads possible) and the VM (far less order 1 allocs). In addition the 4Kb allocation can be satisfied with the per cpu list of free 4Kb pages, while obviously an order 1 cannot and has to go global.
I realize it has nice advantages in theory, just wondering if anyone has done a performance analysis of 4kb vs 8kb stacks lately (or at all?).
I don't think at least anyone at RH has done any; the functionality gain was already enough for us. One item I missed: in the many-thread cases, you also save a lot of memory that can now be used for pagecache; this won't of course be visible in a microbenchmark but should help system wide.
Quoted 5 lines
Also in the implementation I don't see any way 4Kb stacks could show up in any benchmarks as negative; there are only 4 or 5 extra instructions in any path, and afaics no cache downsides (in fact the same irq stack memory is now reused for irqs instead of threadstack-du-jour, so less footprint/hotter caches)
The only downside is the potential crashes due to overflowing the stack, I'm not worried about 4kb stacks performing worse.
Signature · 1 line
Jens Axboe
Quoted 11 lines
Quoted 7 lines
Also in the implementation I don't see any way 4Kb stacks could show up in any benchmarks as negative; there are only 4 or 5 extra instructions in any path, and afaics no cache downsides (in fact the same irq stack memory is now reused for irqs instead of threadstack-du-jour, so less footprint/hotter caches)
The only downside is the potential crashes due to overflowing the stack, I'm not worried about 4kb stacks performing worse.
First of all the 8Kb->4Kb change sounds like a big change, but 4Kb really is 4Kb+4Kb (eg interrupt stacks). So the net reduction in available stack for user context is a lot less (IRQ context need at least 2Kb given that this is reentrant code wrt softirq processing), probably in the order of 2Kb (compared to 8Kb 2.6 kernels) to 500 bytes compared to 2.4 kernels (in 2.4 you lost 1.5 to the task struct on the stack).
The experience with Fedora so far is exceptionally good; in early 2.6 there were some reports with XFS stacked on top of DM, but since then XFS has gone on a stack diet... also the -mm patches to do non-recursive IO submission will bury this (mostly theoretical) monster for good.
Quoted 7 lines · Arjan van de Ven
The experience with Fedora so far is exceptionally good; in early 2.6 there were some reports with XFS stacked on top of DM, but since then XFS has gone on a stack diet... also the -mm patches to do non-recursive IO submission will bury this (mostly theoretical) monster for good.
Fedora with their 2.6.12 and raid+xfs+nfs still causes occasional problems for me. Haven't tried their 2.6.14. But until the block layer modifications are mainline I'm sticking with 8 KiB. My heart momentarily stops every time the file server decides to have a kernel panic, so better safe than sorry.
Rgds
Pierre
Quoted 4 lines · Arjan van de Ven
The experience with Fedora so far is exceptionally good; in early 2.6 there were some reports with XFS stacked on top of DM, but since then XFS has gone on a stack diet... also the -mm patches to do non-recursive IO submission will bury this (mostly theoretical) monster for good.
Not theorical for iscsi though. I guess net+block is a little too much.
OG.
Quoted 2 lines · Olivier Galibert
Not theorical for iscsi though. I guess net+block is a little too much.
net + block fits for the simpler stuff at least (nbd, ataoe)
Quoted 11 lines · Olivier Galibert
Quoted 4 lines · Arjan van de Ven
The experience with Fedora so far is exceptionally good; in early 2.6 there were some reports with XFS stacked on top of DM, but since then XFS has gone on a stack diet... also the -mm patches to do non-recursive IO submission will bury this (mostly theoretical) monster for good.
Not theorical for iscsi though. I guess net+block is a little too much.
If you have stack problem with iscsi then you should post it to those lists or send me a pointer offlist. There were problems with iscsi and XFS but they should be fixed in mainline. The XFS + iscsi problems that have been reported have not been stack usage problems though.
Quoted 11 lines · Mike Christie
Quoted 9 lines · Olivier Galibert
Quoted 4 lines · Arjan van de Ven
The experience with Fedora so far is exceptionally good; in early 2.6 there were some reports with XFS stacked on top of DM, but since then XFS has gone on a stack diet... also the -mm patches to do non-recursive IO submission will bury this (mostly theoretical) monster for good.
Quoted 6 lines
Quoted 3 lines
Not theorical for iscsi though. I guess net+block is a little too much.
If you have stack problem with iscsi then you should post it to those lists or send me a pointer offlist. There were problems with iscsi and XFS but they should be fixed in mainline. The XFS + iscsi problems that have been reported have not been stack usage problems though.
I think to hit a iscsi stack problem you will have to have the scsi request_fn get called from __make_request (or one of the functions it calls like __elv_add_request which looks like it could call __generic_unplug_device). And then the iscsi queuecomamnd would have to hit the path that calls iscsi_data_xmit. Have you hit this?
Quoted 4 lines · Mike Christie
If you have stack problem with iscsi then you should post it to those lists or send me a pointer offlist. There were problems with iscsi and XFS but they should be fixed in mainline. The XFS + iscsi problems that have been reported have not been stack usage problems though.
That hasn't been very efficient last time. In any case, on the latest version I tried (0.4-408, I can't blow up the backup machine every other day):
- iscsi-tape is incompatible with tg3 and works with e1000
- iscsi-disk blows after a random time in what seems to be a (irq?) stack explosion on x86 but not on x86-64 (which iirc has bigger stacks). Seems to because the serial console blows too and only writes a handful of characters.
Filesystem is reiser3.
OG.
Quoted 9 lines · Olivier Galibert
Quoted 1 line · Mike Christie
If you have stack problem with iscsi then you should post it to those lists or send me a pointer offlist. There were problems with iscsi and XFS but they should be fixed in mainline. The XFS + iscsi problems that have been reported have not been stack usage problems though.
That hasn't been very efficient last time. In any case, on the latest version I tried (0.4-408, I can't blow up the backup machine every other day):
Quoted 7 lines
- iscsi-tape is incompatible with tg3 and works with e1000
- iscsi-disk blows after a random time in what seems to be a (irq?) stack explosion on x86 but not on x86-64 (which iirc has bigger stacks). Seems to because the serial console blows too and only writes a handful of characters.
And these only occur with 4k stacks? Do you have the traces still?