static kmem_cache instances for core caches
Subsystems (6)
Al Viro
maintainerRFC PATCH 1/1 static kmem_cache instances for core caches
kmem_cache_create() and friends create new instances of
struct kmem_cache and return pointers to those. Quite a few things in core kernel are allocated from such caches; each allocation involves dereferencing an assign-once pointer and for sufficiently hot ones that dereferencing does show in profiles.
There had been patches floating around switching some of those
to runtime_constant infrastructure. Unfortunately, it's arch-specific and most of the architectures lack it.
There's an alternative approach applicable to the caches that
are never destroyed, which covers a lot of them. No matter what, runtime_constant for pointers is not going to be faster than plain &, so if we had struct kmem_cache instances with static storage duration, we would be at least no worse off than we are with runtime_constant variants.
There are obstacles to doing that, but they turn out to be easy
to deal with.
1) as it is, struct kmem_cache is opaque for anything outside of a few files in mm/*; that avoids serious headache with header dependencies, etc., and it's not something we want to lose. Solution: struct kmem_cache_static, with the size and alignment identical to struct kmem_cache. Calculation of size and alignment can be done via the same mechanism we use for asm-offsets.h and rq-offsets.h, with build-time check for mismatches. With that done, we get an opaque type defined in linux/slab.h that can be used for declaring those caches.
2) real constructor of kmem_cache needs to be taught to deal with preallocated instances. That turns out to be easy - we already pass an obscene amount of optional arguments via struct kmem_cache_args, so we can stash the pointer to preallocated instance in there. Changes in mm/slab_common.c are very minor - we should treat preallocated caches as unmergable, use the instance passed to us instead of allocating a new one and we should not free them on failure. That's it.
Patch below implements that and converts several caches (mnt_cache, signal_cache and thread_stack_cache) to static allocation to demonstrate the uses. If we use that in mainline, these would obviously get split into separate commits.
It seems to work. There's only one real limitation at the moment
- we should never use kmem_cache_destroy() for such caches; unlike runtime_constant, we can *use* those caches from modules - there's no problem with that, as long as the cache itself is in the kernel proper. The obstacle to use of kmem_cache_destroy() might be possible to lift - the only tricky part is sysfs-related logics in kmem_cache_release(). I hadn't looked into that; there's a plenty of never-destroyed core caches, so that thing doesn't lack applications as it is.
Review and comments would be very welcome.
---
Kbuild +12 -1
@@ -45,13 +45,24 @@ kernel/sched/rq-offsets.s: $(offsets-file) $(rq-offsets-file): kernel/sched/rq-offsets.s FORCE $(call filechk,offsets,__RQ_OFFSETS_H__) +# generate kmem_cache_size.h + +kmem_cache_size-file := include/generated/kmem_cache_size.h + +targets += mm/kmem_cache_size.s + +mm/kmem_cache_size.s: $(rq-offsets-file) + +$(kmem_cache_size-file): mm/kmem_cache_size.s FORCE + $(call filechk,offsets,__KMEM_CACHE_SIZE_H__) + # Check for missing system calls quiet_cmd_syscalls = CALL $< cmd_syscalls = $(CONFIG_SHELL) $< $(CC) $(c_flags) $(missing_syscalls_flags) PHONY += missing-syscalls -missing-syscalls: scripts/checksyscalls.sh $(rq-offsets-file) +missing-syscalls: scripts/checksyscalls.sh $(kmem_cache_size-file) $(call cmd,syscalls) # Check the manual modification of atomic headers
fs/namespace.c +5 -3
@@ -85,7 +85,8 @@ static u64 mnt_id_ctr = MNT_UNIQUE_ID_OFFSET;static struct hlist_head *mount_hashtable__ro_after_init ;static struct hlist_head *mountpoint_hashtable__ro_after_init ; -static struct kmem_cache *mnt_cache__ro_after_init ; +static struct kmem_cache_opaque __mnt_cache; +#define mnt_cache to_kmem_cache(&__mnt_cache)static DECLARE_RWSEM (namespace_sem);static HLIST_HEAD (unmounted);/* protected by namespace_sem */ static LIST_HEAD (ex_mountpoints);/* protected by namespace_sem */
@@ -6016,8 +6017,9 @@ void __init mnt_init(void) {int err; - mnt_cache =kmem_cache_create ("mnt_cache" ,sizeof (struct mount ), -0 ,SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT ,NULL ); +kmem_cache_setup ("mnt_cache" ,sizeof (struct mount ), +0 ,SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT , +NULL , mnt_cache); mount_hashtable = alloc_large_system_hash("Mount-cache", sizeof(struct hlist_head),
include/linux/slab.h +44 -0
73 lines not on this page · raw message
kernel/fork.c +9 -8
@@ -379,7 +379,8 @@ static void free_thread_stack(struct task_struct *tsk)#else /* !(THREAD_SIZE >= PAGE_SIZE) */ -static struct kmem_cache *thread_stack_cache; +static struct kmem_cache_opaque __thread_stack_cache; +#define thread_stack_cache to_kmem_cache(&__thread_stack_cache)static void thread_stack_free_rcu (struct rcu_head *rh) {
@@ -410,17 +411,17 @@ static void free_thread_stack(struct task_struct *tsk)void thread_stack_cache_init (void ) { - thread_stack_cache =kmem_cache_create_usercopy ("thread_stack" , -THREAD_SIZE ,THREAD_SIZE ,0 ,0 , -THREAD_SIZE ,NULL ); -BUG_ON (thread_stack_cache ==NULL ); +kmem_cache_setup_usercopy ("thread_stack" ,THREAD_SIZE ,THREAD_SIZE , +SLAB_PANIC ,0 ,THREAD_SIZE ,NULL , + thread_stack_cache); }#endif /* THREAD_SIZE >= PAGE_SIZE */ #endif /* CONFIG_VMAP_STACK */ /* SLAB cache for signal_struct structures (tsk->signal) */ -static struct kmem_cache *signal_cachep; +static struct kmem_cache_opaque signal_cache; +#define signal_cachep to_kmem_cache(&signal_cache)/* SLAB cache for sighand_struct structures (tsk->sighand) */ struct kmem_cache *sighand_cachep;
@@ -2980,10 +2981,10 @@ void __init proc_caches_init(void)sizeof (struct sighand_struct ),0 ,SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_TYPESAFE_BY_RCU | SLAB_ACCOUNT, sighand_ctor); - signal_cachep =kmem_cache_create ("signal_cache" , +kmem_cache_setup ("signal_cache" ,sizeof (struct signal_struct ),0 ,SLAB_HWCACHE_ALIGN |SLAB_PANIC |SLAB_ACCOUNT , -NULL ); +NULL , signal_cachep); files_cachep = kmem_cache_create("files_cache", sizeof(struct files_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT,
mm/kmem_cache_size.c new file +20 -0
@@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Generate definitions needed by the preprocessor. +* This code generates raw asm output which is post-processed +* to extract and format the required data. +*/ + +#define COMPILE_OFFSETS +#include <linux/kbuild.h> +#include "slab.h" + +int main (void ) +{ +/* The constants to put into include/generated/kmem_cache_size.h */ +DEFINE (KMEM_CACHE_SIZE ,sizeof (struct kmem_cache )); +DEFINE (KMEM_CACHE_ALIGN , __alignof(struct kmem_cache )); +/* End of constants */ + +return 0 ; +}
mm/slab_common.c +15 -15
@@ -224,33 +224,30 @@ static struct kmem_cache *create_cache(const char *name, struct kmem_cache_args *args, slab_flags_t flags) { -struct kmem_cache *s; +struct kmem_cache *s = args->preallocated;int err;/* If a custom freelist pointer is requested make sure it's sane. */ - err = -EINVAL ;if (args->use_freeptr_offset && (args->freeptr_offset >= object_size || !(flags &SLAB_TYPESAFE_BY_RCU ) || !IS_ALIGNED (args->freeptr_offset, __alignof__(freeptr_t )))) - goto out; +return ERR_PTR (-EINVAL ); - err = -ENOMEM ; - s =kmem_cache_zalloc (kmem_cache,GFP_KERNEL ); -if (!s) - goto out; +if (!s) { + s =kmem_cache_zalloc (kmem_cache,GFP_KERNEL ); +if (!s) +return ERR_PTR (-ENOMEM ); + } err =do_kmem_cache_create (s, name, object_size, args, flags); -if (err) - goto out_free_cache; - +if (unlikely (err)) { +if (!args->preallocated) +kmem_cache_free (kmem_cache, s); +return ERR_PTR (err); + } s->refcount =1 ;list_add (&s->list, &slab_caches);return s; - -out_free_cache: -kmem_cache_free (kmem_cache, s); -out: -return ERR_PTR (err); }/**
@@ -324,6 +321,9 @@ struct kmem_cache *__kmem_cache_create_args(const char *name, object_size - args->usersize < args->useroffset)) args->usersize = args->useroffset = 0; +if (args->preallocated) + flags |=SLAB_NO_MERGE ; +if (!args->usersize && !args->sheaf_capacity) s =__kmem_cache_alias (name, object_size, args->align, flags, args->ctor);
mm/slub.c +6 -0
@@ -8465,6 +8465,12 @@ void __init kmem_cache_init(void) boot_kmem_cache_node;int node; +/* verify that kmem_cache_opaque is correct */ +BUILD_BUG_ON (sizeof (struct kmem_cache ) != +sizeof (struct kmem_cache_opaque )); +BUILD_BUG_ON (__alignof(struct kmem_cache ) != + __alignof(struct kmem_cache_opaque )); +if (debug_guardpage_minorder ()) slub_max_order =0 ;