slab: cpucache allocation cleanup
PATCH 1/2 slab: cpucache allocation cleanup
From: Pekka Enberg <[email protected]>
This patch cleans up allocation from the per-CPU cache by separating NUMA and UMA paths.
Signed-off-by Pekka Enberg <[email protected]>
---
mm/slab.c | 69 +++++++++++++++++++++++++++++++++++++------------------------ 1 files changed, 42 insertions(+), 27 deletions(-)
8658c94d24e3b97f2ad747182811713c52dffdcf
mm/slab.c +42 -27
@@ -2855,19 +2855,12 @@ static void *cache_alloc_debugcheck_afte#define cache_alloc_debugcheck_after (a,b,objp,d) (objp)#endif -static inline void *____cache_alloc (struct kmem_cache *cachep,gfp_t flags) +static __always_inline void *__cache_alloc_cpucache(struct kmem_cache *cachep, +gfp_t flags) {void *objp;struct array_cache *ac; -#ifdef CONFIG_NUMA -if (unlikely (current->flags & (PF_SPREAD_SLAB |PF_MEMPOLICY ))) { - objp =alternate_node_alloc (cachep, flags); -if (objp !=NULL ) -return objp; - } -#endif -check_irq_off (); ac =cpu_cache_get (cachep);if (likely (ac->avail)) {
@@ -2881,23 +2874,6 @@ static inline void *____cache_alloc(strureturn objp; } -static __always_inline void *__cache_alloc(struct kmem_cache *cachep, -gfp_t flags,void *caller) -{ -unsigned long save_flags; -void *objp; - -cache_alloc_debugcheck_before (cachep, flags); - -local_irq_save (save_flags); - objp =____cache_alloc (cachep, flags); -local_irq_restore (save_flags); - objp =cache_alloc_debugcheck_after (cachep, flags, objp, - caller); -prefetchw (objp); -return objp; -} -#ifdef CONFIG_NUMA /* * Try allocating on another node if PF_SPREAD_SLAB|PF_MEMPOLICY.
@@ -2982,8 +2958,47 @@ must_grow: done:return obj; } + +static inline void *cache_alloc_cpucache (struct kmem_cache *cache,gfp_t flags) +{ +if (unlikely (current->flags & (PF_SPREAD_SLAB |PF_MEMPOLICY ))) { +void *objp =alternate_node_alloc (cache, flags); +if (objp !=NULL ) +return objp; + } +return __cache_alloc_cpucache (cache, flags); +} + +#else + +/* +* On UMA, we always allocate directly drom the per-CPU cache. +*/ + +static inline void *cache_alloc_cpucache (struct kmem_cache *cache,gfp_t flags) +{ +return __cache_alloc_cpucache (cache, flags); +} +#endif +static __always_inline void *__cache_alloc(struct kmem_cache *cachep, +gfp_t flags,void *caller) +{ +unsigned long save_flags; +void *objp; + +cache_alloc_debugcheck_before (cachep, flags); + +local_irq_save (save_flags); + objp =cache_alloc_cpucache (cachep, flags); +local_irq_restore (save_flags); + objp =cache_alloc_debugcheck_after (cachep, flags, objp, + caller); +prefetchw (objp); +return objp; +} +/* * Caller needs to acquire correct kmem_list's list_lock */
@@ -3229,7 +3244,7 @@ void *kmem_cache_alloc_node(struct kmem_if (nodeid ==-1 || nodeid ==numa_node_id () || !cachep->nodelists[nodeid]) - ptr =____cache_alloc (cachep, flags); + ptr =cache_alloc_cpucache (cachep, flags);else ptr =__cache_alloc_node (cachep, flags, nodeid);local_irq_restore (save_flags);
Signature · 1 line
1.1.3
mm/slab.c static void *cache_alloc_debugcheck_afte · line 2858
Quoted 2 lines · Pekka Enberg
mm/slab.c +1
-1-static inline void *____cache_alloc (struct kmem_cache *cachep,gfp_t flags) +static __always_inline void *__cache_alloc_cpucache(struct kmem_cache *cachep,
The new name is confusing because __cache_alloc_cpucache suggests that we are only allocating from the cpucache and that this be something special. However, we always allocate from the cpucache for local allocations and we refill the cpucache in the __cpucache function from the shared cache and the per node lists. So we do much more there.
Maybe call this __cache_alloc_local ?