Put modules into linear mapping
Subsystems (3)
PATCH 1/1 Put modules into linear mapping
Now that kksymoops is in this can be safely merged:
In some cases there is a benchmarkable difference (in the several percent range) between a compiled in driver and a modular driver. The reason is the compiled in driver is mapped in 2/4MB pages in the direct mapping, while modules are mapped wit 4K pages in the vmalloc area. The resulting TLB misses can make that much difference.
This patch fixes it by just putting the modules into the linear mapping if possible. This needs unfragmented memory, but modules tend to be loaded at bootup when there is still plenty of this. When it fails it just falls back to vmalloc.
It has one drawback in that it makes the load address of modules much more random than they were previously: you have a bigger chance that the module is loaded somewhere else after reboot and then ksymoops wouldn't be able to decode an module oops correctly anymore. With kksymoops this shouldn't be a problem anymore, so it can be safely merged.
To avoid too much memory wastage it adds a new interface alloc_exact/free_exact that allocates pages from the page allocator but doesn't round the area size to the next power of two.
Variants of this patch have run successfully for several years in the -aa kernels and in the SuSE kernels.
Patch for 2.5.38. Please apply.
-Andi
include/linux/gfp.h renamed from include/linux/gfp.h-MODULE +3 -0
@@ -85,4 +85,7 @@#define __free_page (page) __free_pages((page), 0)#define free_page (addr) free_pages((addr),0) +extern void *alloc_exact (unsigned int size); +extern void free_exact (void * addr,unsigned int size); +#endif /* __LINUX_GFP_H */
include/asm-i386/module.h renamed from include/asm-i386/module.h-MODULE +24 -2
@@ -3,9 +3,31 @@/* * This file contains the i386 architecture specific module code. */ +#include <linux/vmalloc.h> +#include <linux/highmem.h> +#include <asm/page.h> + +/* +* Try to allocate in the linear large page mapping first to conserve +* TLB entries. +*/ +static inline void *module_map (unsigned long size) +{ +void *p =alloc_exact (size); +if (!p) + p =vmalloc (size); +return p; +} + +static inline void module_unmap (struct module *mod) +{ +unsigned long mptr = (unsigned long )mod; +if (mptr >=VMALLOC_START && mptr+mod->size <=VMALLOC_END ) +vfree (mod); +else +free_exact (mod, mod->size); +} -#define module_map (x) vmalloc(x) -#define module_unmap (x) vfree(x)#define module_arch_init (x) (0)#define arch_init_modules (x) do { } while (0)
mm/page_alloc.c renamed from mm/page_alloc.c-MODULE +32 -0
@@ -464,6 +464,38 @@__free_pages (virt_to_page (addr), order); } } + +void *alloc_exact (unsigned int size) +{ +struct page *p, *w; +int order =get_order (size); + + p =alloc_pages (GFP_KERNEL , order); +if (p) { +struct page *end = p + (1UL << order); +for (w = p+1 ; w < end; ++w) +set_page_count (w,1 ); +for (w = p + (size>>PAGE_SHIFT )+1 ; w < end; ++w) +__free_pages (w,0 ); +return (void *)page_address (p); + } + +return NULL ; +} + +void free_exact (void * addr,unsigned int size) +{ +struct page * w; +unsigned long mptr = (unsigned long ) addr; +int sz; + + w =virt_to_page (addr); +for (sz = size; sz >0 ; sz -=PAGE_SIZE , ++w) { +if (atomic_read (&w->count) !=1 ) +BUG (); +__free_pages (w,0 ); + } +}/* * Total amount of free (allocatable) RAM:
kernel/ksyms.c renamed from kernel/ksyms.c-MODULE +2 -0
@@ -97,6 +97,8 @@EXPORT_SYMBOL (get_zeroed_page);EXPORT_SYMBOL (__page_cache_release);EXPORT_SYMBOL (__free_pages); +EXPORT_SYMBOL (alloc_exact); +EXPORT_SYMBOL (free_exact);EXPORT_SYMBOL (free_pages);EXPORT_SYMBOL (num_physpages);EXPORT_SYMBOL (kmem_find_general_cachep);
Hi,
include/asm-i386/module.h line 20
Quoted 13 lines · Andi Kleen
include/asm-i386/module.h renamed from include/asm-i386/module.h-MODULE +11
-0+/* +* Try to allocate in the linear large page mapping first to conserve +* TLB entries. +*/ +static inline void *module_map (unsigned long size) +{ +void *p =alloc_exact (size); +if (!p) + p =vmalloc (size); +return p; +}
Why is i386 only? This is generic code and other archs will benefit from it as well (or at least it won't hurt).
$ grep module_map include/asm-*/*.h include/asm-alpha/module.h:#define module_map(x) vmalloc(x) include/asm-arm/module.h:#define module_map(x) vmalloc(x) include/asm-cris/module.h:#define module_map(x) vmalloc(x) include/asm-i386/module.h:#define module_map(x) vmalloc(x) include/asm-ia64/module.h:#define module_map(x) vmalloc(x) include/asm-m68k/module.h:#define module_map(x) vmalloc(x) include/asm-mips/module.h:#define module_map(x) vmalloc(x) include/asm-mips64/module.h:#define module_map(x) vmalloc(x) include/asm-parisc/pgtable.h:#define module_map vmalloc include/asm-ppc/module.h:#define module_map(x) vmalloc(x) include/asm-ppc64/module.h:#define module_map(x) vmalloc(x) include/asm-s390/module.h:#define module_map(x) vmalloc(x) include/asm-s390x/module.h:#define module_map(x) vmalloc(x) include/asm-sh/module.h:#define module_map(x) vmalloc(x) include/asm-sparc/module.h:#define module_map(x) vmalloc(x) include/asm-sparc64/module.h:extern void * module_map (unsigned long size); include/asm-x86_64/module.h:extern void *module_map(unsigned long);
The x86_64 version is currently broken, as it uses vmalloc_area_pages which doesn't exist anymore, but it did the same as vmalloc. The sparc64 version also looks like vmalloc reimplementation.
include/asm-i386/module.h
Quoted 7 lines
include/asm-i386/module.h renamed from include/asm-i386/module.h-MODULE +7
-0+ +void *alloc_exact (unsigned int size) +{ +struct page *p, *w; +int order =get_order (size); + + p =alloc_pages (GFP_KERNEL , order);
Wouldn't it be better to add a gfp argument?
bye, Roman
include/asm-i386/module.h line 25
Quoted 2 lines · Andi Kleen
include/asm-i386/module.h renamed from include/asm-i386/module.h-MODULE +2
-0+unsigned long mptr = (unsigned long )mod; +if (mptr >=VMALLOC_START && mptr+mod->size <=VMALLOC_END )
Any chance you could add is is_vmalloc() macro to vmalloc.h for this?
Quoted 5 lines · Andi Kleen
It has one drawback in that it makes the load address of modules much more random than they were previously: you have a bigger chance that the module is loaded somewhere else after reboot and then ksymoops wouldn't be able to decode an module oops correctly anymore. With kksymoops this shouldn't be a problem anymore, so it can be safely merged.
man insmod. See 'ksymoops assistance'. mkdir /var/log/ksymoops.