modules: Use a better scheme for refcounting
PATCH 1/1 modules: Use a better scheme for refcounting
From: Eric Dumazet <[email protected]>
(Linus, please apply. This has become critical with NR_CPUS 4096: we currently use an astonishing 1GB additional space for 2000 modules.)
Current refcounting for modules (done if CONFIG_MODULE_UNLOAD=y) is using a lot of memory.
Each 'struct module' contains an [NR_CPUS] array of full cache lines.
This patch uses existing infrastructure (percpu_modalloc() & percpu_modfree()) to allocate percpu space for the refcount storage.
Instead of wasting NR_CPUS*128 bytes (on i386), we now use nr_cpu_ids*sizeof(local_t) bytes.
On a typical distro, where NR_CPUS=8, shiping 2000 modules, we reduce size of module files by about 2 Mbytes. (1Kb per module)
Instead of having all refcounters in the same memory node - with TLB misses because of vmalloc() - this new implementation permits to have better NUMA properties, since each CPU will use storage on its preferred node, thanks to percpu storage.
--- include/linux/module.h | 25 ++++++++++++++++--------- kernel/module.c | 35 +++++++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 19 deletions(-)
include/linux/module.h +16 -9
@@ -219,11 +219,6 @@ void *__symbol_get_gpl(const char *symbo#endif -struct module_ref -{ -local_t count; -}____cacheline_aligned ; -enum module_state {MODULE_STATE_LIVE ,
@@ -344,8 +339,11 @@ struct module/* Destruction function. */ void (*exit)(void ); -/* Reference counts */ -struct module_ref ref[NR_CPUS ]; +#ifdef CONFIG_SMP +char *refptr; +#else +local_t ref; +#endif #endif };#ifndef MODULE_ARCH_INIT
@@ -395,13 +393,22 @@ void __symbol_put(const char *symbol);#define symbol_put (x) __symbol_put(MODULE_SYMBOL_PREFIX #x)void symbol_put_addr (void *addr); +static inline local_t *__module_ref_addr (struct module *mod,int cpu) +{ +#ifdef CONFIG_SMP +return (local_t *) (mod->refptr +per_cpu_offset (cpu)); +#else +return &mod->ref; +#endif +} +/* Sometimes we know we already have a refcount, and it's easier not to handle the error case (which only happens with rmmod --wait). */ static inline void __module_get (struct module *module) {if (module) {BUG_ON (module_refcount (module) ==0 ); -local_inc (&module->ref[get_cpu ()].count); +local_inc (__module_ref_addr (module,get_cpu ()));put_cpu (); } }
@@ -413,7 +420,7 @@ static inline int try_module_get(structif (module) {unsigned int cpu =get_cpu ();if (likely (module_is_live (module))) -local_inc (&module->ref[cpu].count); +local_inc (__module_ref_addr (module, cpu));else ret =0 ;put_cpu ();
kernel/module.c +25 -10
@@ -573,13 +573,13 @@ static char last_unloaded_module[MODULE_/* Init the unload section of the module. */ static void module_unload_init (struct module *mod) { -unsigned int i; +int cpu;INIT_LIST_HEAD (&mod->modules_which_use_me); -for (i =0 ; i <NR_CPUS ; i++) -local_set (&mod->ref[i].count,0 ); +for_each_possible_cpu (cpu) +local_set (__module_ref_addr (mod, cpu),0 );/* Hold reference count during initialization. */ -local_set (&mod->ref[raw_smp_processor_id ()].count,1 ); +local_set (__module_ref_addr (mod,raw_smp_processor_id ()),1 );/* Backwards compatibility macros put refcount during init. */ mod->waiter = current; }
@@ -717,10 +717,11 @@ static int try_stop_module(struct moduleunsigned int module_refcount (struct module *mod) { -unsigned int i, total =0 ; +unsigned int total =0 ; +int cpu; -for (i =0 ; i <NR_CPUS ; i++) - total +=local_read (&mod->ref[i].count); +for_each_possible_cpu (cpu) + total +=local_read (__module_ref_addr (mod, cpu));return total; }EXPORT_SYMBOL (module_refcount);
@@ -894,7 +895,7 @@ void module_put(struct module *module) {if (module) {unsigned int cpu =get_cpu (); -local_dec (&module->ref[cpu].count); +local_dec (__module_ref_addr (module, cpu));/* Maybe they're waiting for us to drop reference? */ if (unlikely (!module_is_live (module)))wake_up_process (module->waiter);
@@ -1464,7 +1465,10 @@ static void free_module(struct module *mkfree (mod->args);if (mod->percpu)percpu_modfree (mod->percpu); - +#if defined(CONFIG_MODULE_UNLOAD ) && defined(CONFIG_SMP ) +if (mod->refptr) +percpu_modfree (mod->refptr); +#endif /* Free lock-classes: */ lockdep_free_key_range (mod->module_core, mod->core_size);
@@ -2011,6 +2015,14 @@ static noinline struct module *load_moduif (err <0 ) goto free_mod; +#if defined(CONFIG_MODULE_UNLOAD ) && defined(CONFIG_SMP ) + mod->refptr =percpu_modalloc (sizeof (local_t), __alignof__(local_t ), + mod->name); +if (!mod->refptr) { + err = -ENOMEM ; + goto free_mod; + } +#endif if (pcpuindex) {/* We have a special allocation for this section. */ percpu = percpu_modalloc(sechdrs[pcpuindex].sh_size,
@@ -2018,7 +2030,7 @@ static noinline struct module *load_modu mod->name);if (!percpu) { err = -ENOMEM ; - goto free_mod; + goto free_percpu; } sechdrs[pcpuindex].sh_flags &= ~(unsigned long )SHF_ALLOC ; mod->percpu = percpu;
@@ -2282,6 +2294,9 @@ static noinline struct module *load_modu free_percpu:if (percpu)percpu_modfree (percpu); +#if defined(CONFIG_MODULE_UNLOAD ) && defined(CONFIG_SMP ) +percpu_modfree (mod->refptr); +#endif free_mod:kfree (args); free_hdr: