slab: implement bulking for SLAB allocator
Subsystems (2)
cover slab: implement bulking for SLAB allocator
This patchset implements bulking for the SLAB allocator. I split the implementation into two patches, the alloc and free "side" for easier review.
(Based on Linus tree at v4.4-rc3-24-g25364a9e54fb)
Normal SLAB fastpath 95 cycles(tsc) 23.852 ns, when compiled without debugging options enabled.
Benchmarked[1] obj size 256 bytes on CPU i7-4790K @ 4.00GHz:
1 - 115 cycles(tsc) 28.812 ns - 42 cycles(tsc) 10.715 ns - improved 63.5%
2 - 103 cycles(tsc) 25.956 ns - 27 cycles(tsc) 6.985 ns - improved 73.8% 3 - 101 cycles(tsc) 25.336 ns - 22 cycles(tsc) 5.733 ns - improved 78.2% 4 - 100 cycles(tsc) 25.147 ns - 21 cycles(tsc) 5.319 ns - improved 79.0% 8 - 98 cycles(tsc) 24.616 ns - 18 cycles(tsc) 4.620 ns - improved 81.6% 16 - 97 cycles(tsc) 24.408 ns - 17 cycles(tsc) 4.344 ns - improved 82.5% 30 - 98 cycles(tsc) 24.641 ns - 16 cycles(tsc) 4.202 ns - improved 83.7% 32 - 98 cycles(tsc) 24.607 ns - 16 cycles(tsc) 4.199 ns - improved 83.7% 34 - 98 cycles(tsc) 24.605 ns - 18 cycles(tsc) 4.579 ns - improved 81.6% 48 - 97 cycles(tsc) 24.463 ns - 17 cycles(tsc) 4.405 ns - improved 82.5% 64 - 97 cycles(tsc) 24.370 ns - 17 cycles(tsc) 4.384 ns - improved 82.5% 128 - 99 cycles(tsc) 24.763 ns - 19 cycles(tsc) 4.755 ns - improved 80.8% 158 - 98 cycles(tsc) 24.708 ns - 18 cycles(tsc) 4.723 ns - improved 81.6% 250 - 101 cycles(tsc) 25.342 ns - 20 cycles(tsc) 5.035 ns - improved 80.2%
[1] https://github.com/netoptimizer/prototype-kernel/blob/master/kernel/mm/slab_bulk_test01.c
---
Jesper Dangaard Brouer (2):
slab: implement bulk alloc in SLAB allocator
slab: implement bulk free in SLAB allocator
mm/slab.c | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 77 insertions(+), 8 deletions(-)
--
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH 1/2 slab: implement bulk alloc in SLAB allocator
(will add more desc after RFC)
--- mm/slab.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-)
mm/slab.c +52 -2
@@ -3420,9 +3420,59 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)EXPORT_SYMBOL (kmem_cache_free_bulk);int kmem_cache_alloc_bulk (struct kmem_cache *s,gfp_t flags,size_t size, -void **p) +void **p) { -return __kmem_cache_alloc_bulk (s, flags, size, p); +size_t i; + + flags &= gfp_allowed_mask; +lockdep_trace_alloc (flags); + +if (slab_should_failslab (s, flags)) +return 0 ; + + s =memcg_kmem_get_cache (s, flags); + +cache_alloc_debugcheck_before (s, flags); + +local_irq_disable (); +for (i =0 ; i < size; i++) { +void *objp =__do_cache_alloc (s, flags); + +// this call could be done outside IRQ disabled section + objp =cache_alloc_debugcheck_after (s, flags, objp, _RET_IP_); + +if (unlikely (!objp)) + goto error; + +prefetchw (objp); + p[i] = objp; + } +local_irq_enable (); + +/* Kmemleak and kmemcheck outside IRQ disabled section */ +for (i =0 ; i < size; i++) { +void *x = p[i]; + +kmemleak_alloc_recursive (x, s->object_size,1 , s->flags, flags); +kmemcheck_slab_alloc (s, flags, x, s->object_size); + } + +/* Clear memory outside IRQ disabled section */ +if (unlikely (flags & __GFP_ZERO)) +for (i =0 ; i < size; i++) +memset (p[i],0 , s->object_size); + +// FIXME: Trace call missing... should we create a bulk variant? +/* Like: +trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, s->size, flags); +*/ +memcg_kmem_put_cache (s); +return size; +error: +local_irq_enable (); +memcg_kmem_put_cache (s); +__kmem_cache_free_bulk (s, i, p); +return 0 ; }EXPORT_SYMBOL (kmem_cache_alloc_bulk);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH 1/2] slab: implement bulk alloc in SLAB allocator
Quoted 20 lines · On Thu, 03 Dec 2015 16:57:31 +0100 Jesper Dangaard Brouer <[email protected]>…
mm/slab.c +9
-2@@ -3420,9 +3420,59 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)EXPORT_SYMBOL (kmem_cache_free_bulk);int kmem_cache_alloc_bulk (struct kmem_cache *s,gfp_t flags,size_t size, -void **p) +void **p) { -return __kmem_cache_alloc_bulk (s, flags, size, p); +size_t i; + + flags &= gfp_allowed_mask; +lockdep_trace_alloc (flags); + +if (slab_should_failslab (s, flags)) +return 0 ; +
mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) · line 3442
Quoted 10 lines
mm/slab.c +10
-0+ s =memcg_kmem_get_cache (s, flags); + +cache_alloc_debugcheck_before (s, flags); + +local_irq_disable (); +for (i =0 ; i < size; i++) { +void *objp =__do_cache_alloc (s, flags); + +// this call could be done outside IRQ disabled section + objp =cache_alloc_debugcheck_after (s, flags, objp, _RET_IP_);
Profiling with SLAB mem debugging on (CONFIG_DEBUG_SLAB=y), this call cache_alloc_debugcheck_after() is the most expensive call, well actually the underlying check_poison_obj() call.
Thus, it might be a good idea to, place it outside the IRQ disabled section? It might make the code look a little strange, but I can try and we can see how ugly that makes the code look (and the compiler still have to be able to remove the code in-case no debugging enabled).
Quoted 21 lines
mm/slab.c +21
-0+ +if (unlikely (!objp)) + goto error; + +prefetchw (objp); + p[i] = objp; + } +local_irq_enable (); + +/* Kmemleak and kmemcheck outside IRQ disabled section */ +for (i =0 ; i < size; i++) { +void *x = p[i]; + +kmemleak_alloc_recursive (x, s->object_size,1 , s->flags, flags); +kmemcheck_slab_alloc (s, flags, x, s->object_size); + } + +/* Clear memory outside IRQ disabled section */ +if (unlikely (flags & __GFP_ZERO)) +for (i =0 ; i < size; i++) +memset (p[i],0 , s->object_size);
[...]
Signature · 11 lines
Best regards, Jesper Dangaard Brouer MSc.CS, Principal Kernel Engineer at Red Hat Author of http://www.iptv-analyzer.org LinkedIn: http://www.linkedin.com/in/brouer -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH 1/2] slab: implement bulk alloc in SLAB allocator
In reply to Jesper Dangaard Brouermm/slab.c void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) · line 3431
Quoted 7 lines · Jesper Dangaard Brouer
mm/slab.c +7
-0+size_t i; + + flags &= gfp_allowed_mask; +lockdep_trace_alloc (flags); + +if (slab_should_failslab (s, flags)) +return 0 ;
Ok here is an overlap with slub;'s pre_alloc_hook() and that stuff is really not allocator specific. Could make it generic and move the hook calls into slab_common.c/slab.h? That also gives you the opportunity to get the array option in there.
Quoted 5 lines
mm/slab.c +5
-0+ s =memcg_kmem_get_cache (s, flags); + +cache_alloc_debugcheck_before (s, flags); + +local_irq_disable ();
mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) · line 3447
Quoted 10 lines
mm/slab.c +10
-0+for (i =0 ; i < size; i++) { +void *objp =__do_cache_alloc (s, flags); + +// this call could be done outside IRQ disabled section + objp =cache_alloc_debugcheck_after (s, flags, objp, _RET_IP_); + +if (unlikely (!objp)) + goto error; + +prefetchw (objp);
Is the prefetch really useful here? Only if these objects are immediately used I would think.
Quoted 6 lines
mm/slab.c +6
-0+ p[i] = objp; + } +local_irq_enable (); + +/* Kmemleak and kmemcheck outside IRQ disabled section */ +for (i =0 ; i < size; i++) {
mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) · line 3463
Quoted 10 lines
mm/slab.c +10
-0+void *x = p[i]; + +kmemleak_alloc_recursive (x, s->object_size,1 , s->flags, flags); +kmemcheck_slab_alloc (s, flags, x, s->object_size); + } + +/* Clear memory outside IRQ disabled section */ +if (unlikely (flags & __GFP_ZERO)) +for (i =0 ; i < size; i++) +memset (p[i],0 , s->object_size);
Maybe make this one loop instead of two?
mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) · line 3468
Quoted 4 lines
mm/slab.c +4
-0+// FIXME: Trace call missing... should we create a bulk variant? +/* Like: +trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, s->size, flags); +*/
That trace call could be created when you do the genericization of the hooks() which also involve debugging stuff.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH 1/2] slab: implement bulk alloc in SLAB allocator
Quoted 14 lines · On Fri, 4 Dec 2015 11:10:05 -0600 (CST) Christoph Lameter <[email protected]> wrote:
mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) · line 3431
Quoted 7 lines · Jesper Dangaard Brouer
mm/slab.c +7
-0+size_t i; + + flags &= gfp_allowed_mask; +lockdep_trace_alloc (flags); + +if (slab_should_failslab (s, flags)) +return 0 ;Ok here is an overlap with slub;'s
pre_alloc_hook()and that stuff is really not allocator specific. Could make it generic and move the hook calls intoslab_common.c/slab.h? That also gives you the opportunity to get the array option in there.
Perhaps we could consolidate some code here. (This would also help code SLAB elimination between slab_alloc_node() and slab_alloc())
A question: SLAB takes the "boot_cache" into account before calling should_failslab(), but SLUB does not. Should we also do so for SLUB?
SLAB code:
static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags)
{
if (unlikely(cachep == kmem_cache))
return false;
return should_failslab(cachep->object_size, flags, cachep->flags);}
Quoted 10 lines
Quoted 10 lines
mm/slab.c +10
-0+ s =memcg_kmem_get_cache (s, flags); + +cache_alloc_debugcheck_before (s, flags); + +local_irq_disable (); +for (i =0 ; i < size; i++) { +void *objp =__do_cache_alloc (s, flags); + +// this call could be done outside IRQ disabled section + objp =cache_alloc_debugcheck_after (s, flags, objp, _RET_IP_);
Quoted 8 lines
mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) · line 3447
Quoted 5 lines
mm/slab.c +5
-0+ +if (unlikely (!objp)) + goto error; + +prefetchw (objp);Is the prefetch really useful here? Only if these objects are immediately used I would think.
I primarily have prefetch here because I'm mimicking the behavior of slab_alloc(). We can remove it here.
Quoted 10 lines
Quoted 10 lines
mm/slab.c +10
-0+ p[i] = objp; + } +local_irq_enable (); + +/* Kmemleak and kmemcheck outside IRQ disabled section */ +for (i =0 ; i < size; i++) { +void *x = p[i]; + +kmemleak_alloc_recursive (x, s->object_size,1 , s->flags, flags); +kmemcheck_slab_alloc (s, flags, x, s->object_size);
Quoted 8 lines
mm/slab.c void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) · line 3463
Quoted 6 lines
mm/slab.c +6
-0+ } + +/* Clear memory outside IRQ disabled section */ +if (unlikely (flags & __GFP_ZERO)) +for (i =0 ; i < size; i++) +memset (p[i],0 , s->object_size);Maybe make this one loop instead of two?
I kept it two loops to get the advantage of only needing to check the __GFP_ZERO flag once. (Plus, in case debugging is enabled, we might get a small advantage of better instruction and pipeline usage, as erms memset rep-stos operations flush the CPU pipeline).
I also wrote it this way to make it more obvious what code I want the compiler to generate. If no debugging is enabled to top loop should be compiled out. If I didn't pullout the flag check, the compiler should be smart enough to realize this optimization itself, but can only realize this in case the other code compiles out (case where loops were combined). Thus, compiler might already do this optimization, but I'm making it explicit.
Besides, maybe we can consolidate first loop and replace it with slab_post_alloc_hook()?
Quoted 7 lines
Quoted 4 lines
mm/slab.c +3
-0+// FIXME: Trace call missing... should we create a bulk variant? +/* Like: +trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
s->size, flags); +*/That trace call could be created when you do the genericization of the
hooks()which also involve debugging stuff.
Should we call trace_kmem_cache_alloc() for each object?
Or should we create trace calls that are specific to bulk'ing? (which would allow us to study/record bulk sizes)
Signature · 11 lines
Best regards, Jesper Dangaard Brouer MSc.CS, Principal Kernel Engineer at Red Hat Author of http://www.iptv-analyzer.org LinkedIn: http://www.linkedin.com/in/brouer -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH 1/2] slab: implement bulk alloc in SLAB allocator
Quoted 2 lines · Jesper Dangaard Brouer
A question: SLAB takes the "
boot_cache" into account before callingshould_failslab(), but SLUB does not. Should we also do so for SLUB?
Not necessary in SLUB.
Quoted 2 lines
Besides, maybe we can consolidate first loop and replace it with
slab_post_alloc_hook()?
Ok.
Quoted 2 lines
Or should we create trace calls that are specific to bulk'ing? (which would allow us to study/record bulk sizes)
I would prefer that.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH 2/2 slab: implement bulk free in SLAB allocator
In reply to Jesper Dangaard Brouer(will add more desc after RFC)
--- mm/slab.c | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-)
mm/slab.c +25 -6
@@ -3413,12 +3413,6 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) }EXPORT_SYMBOL (kmem_cache_alloc); -void kmem_cache_free_bulk (struct kmem_cache *s,size_t size,void **p) -{ -__kmem_cache_free_bulk (s, size, p); -} -EXPORT_SYMBOL (kmem_cache_free_bulk); -int kmem_cache_alloc_bulk (struct kmem_cache *s,gfp_t flags,size_t size,void **p) {
@@ -3619,6 +3613,31 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) }EXPORT_SYMBOL (kmem_cache_free); +void kmem_cache_free_bulk (struct kmem_cache *orig_s,size_t size,void **p) +{ +struct kmem_cache *s; +size_t i; + +local_irq_disable (); +for (i =0 ; i < size; i++) { +void *objp = p[i]; + + s =cache_from_obj (orig_s, objp); + +debug_check_no_locks_freed (objp, s->object_size); +if (!(s->flags &SLAB_DEBUG_OBJECTS )) +debug_check_no_obj_freed (objp, s->object_size); + +__cache_free (s, objp, _RET_IP_); + } +local_irq_enable (); + +// FIXME: tracing +// trace_kmem_cache_free(_RET_IP_, objp); +} +EXPORT_SYMBOL (kmem_cache_free_bulk); + +/** * kfree - free previously allocated memory * @objp: pointer returned by kmalloc.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH 2/2] slab: implement bulk free in SLAB allocator
Quoted 1 line · Jesper Dangaard Brouer
+void
kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
orig_s? Thats strange
mm/slab.c void kmem_cache_free(struct kmem_cache *cachep, void *objp) · line 3618
Quoted 2 lines
mm/slab.c +2
-0+{ +struct kmem_cache *s;
s?
mm/slab.c void kmem_cache_free(struct kmem_cache *cachep, void *objp) · line 3625
Quoted 7 lines
mm/slab.c +7
-0+size_t i; + +local_irq_disable (); +for (i =0 ; i < size; i++) { +void *objp = p[i]; + + s =cache_from_obj (orig_s, objp);
Does this support freeing objects from a set of different caches?
Otherwise there needs to be a check in here that the objects come from the same cache.
mm/slab.c void kmem_cache_free(struct kmem_cache *cachep, void *objp) · line 3631
Quoted 6 lines
mm/slab.c +6
-0+ +debug_check_no_locks_freed (objp, s->object_size); +if (!(s->flags &SLAB_DEBUG_OBJECTS )) +debug_check_no_obj_freed (objp, s->object_size); + +__cache_free (s, objp, _RET_IP_);
The function could be further optimized if you take the code from __cache_free() and move stuff outside of the loop. The alien cache check f.e. and the Pfmemalloc checking may be moved out. The call to virt_to_head page may also be avoided if the objects are on the same page as the last. So you may be able to function calls for the fastpath in the inner loop which may accelerate frees significantly.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH 2/2] slab: implement bulk free in SLAB allocator
Quoted 10 lines · On Fri, 4 Dec 2015 11:17:02 -0600 (CST) Christoph Lameter <[email protected]> wrote:
Quoted 1 line · Jesper Dangaard Brouer
+void
kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
orig_s? Thats strangemm/slab.c void kmem_cache_free(struct kmem_cache *cachep, void *objp) · line 3618
Quoted 2 lines
mm/slab.c +2
-0+{ +struct kmem_cache *s;s?
The "s" comes from the slub.c code uses "struct kmem_cache *s" everywhere.
Quoted 9 lines
mm/slab.c void kmem_cache_free(struct kmem_cache *cachep, void *objp) · line 3625
Quoted 7 lines
mm/slab.c +7
-0+size_t i; + +local_irq_disable (); +for (i =0 ; i < size; i++) { +void *objp = p[i]; + + s =cache_from_obj (orig_s, objp);Does this support freeing objects from a set of different caches?
This is for supporting memcg (CONFIG_MEMCG_KMEM).
Quoting from commit 033745189b1b ("slub: add missing kmem cgroup support to kmem_cache_free_bulk"):
Incoming bulk free objects can belong to different kmem cgroups, and object free call can happen at a later point outside memcg context. Thus, we need to keep the orig kmem_cache, to correctly verify if a memcg object match against its "root_cache" (s->memcg_params.root_cache).
Quoted 5 lines
Quoted 5 lines
mm/slab.c +5
-0+ +debug_check_no_locks_freed (objp, s->object_size); +if (!(s->flags &SLAB_DEBUG_OBJECTS )) +debug_check_no_obj_freed (objp, s->object_size); +
Quoted 8 lines
mm/slab.c void kmem_cache_free(struct kmem_cache *cachep, void *objp) · line 3631
Quoted 1 line
mm/slab.c +1
-0+__cache_free (s, objp, _RET_IP_);The function could be further optimized if you take the code from
__cache_free()and move stuff outside of the loop. The alien cache check f.e. and the Pfmemalloc checking may be moved out. The call tovirt_to_headpage may also be avoided if the objects are on the same page as the last. So you may be able to function calls for the fastpath in the inner loop which may accelerate frees significantly.
Interesting! Maybe we can do a followup patch to pullout last optimization's. Right now I'm mostly interested in correctness and clean code. And we are already looking at a 80% speedup with these patches ;-)
Signature · 11 lines
Best regards, Jesper Dangaard Brouer MSc.CS, Principal Kernel Engineer at Red Hat Author of http://www.iptv-analyzer.org LinkedIn: http://www.linkedin.com/in/brouer -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH 2/2] slab: implement bulk free in SLAB allocator
Quoted 3 lines · Jesper Dangaard Brouer
Quoted 1 line
s?
The "s" comes from the
slub.ccode uses "structkmem_cache*s" everywhere.
Ok then use it. Why is there an orig_s here.
Quoted 11 lines
Quoted 8 lines
mm/slab.c void kmem_cache_free(struct kmem_cache *cachep, void *objp) · line 3625
Quoted 6 lines
mm/slab.c +6
-0+ +local_irq_disable (); +for (i =0 ; i < size; i++) { +void *objp = p[i]; + + s =cache_from_obj (orig_s, objp);Does this support freeing objects from a set of different caches?
This is for supporting memcg (
CONFIG_MEMCG_KMEM).
Quoted 7 lines
Quoting from commit
033745189b1b("slub: add missing kmem cgroup support tokmem_cache_free_bulk"):Incoming bulk free objects can belong to different kmem cgroups, and object free call can happen at a later point outside memcg context. Thus, we need to keep the orig
kmem_cache, to correctly verify if a memcg object match against its "root_cache" (s->memcg_params.root_cache).
Where is that verification? This looks like SLAB would support freeing objects from different caches.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH 2/2] slab: implement bulk free in SLAB allocator
(To Vladimir can you comment on memcg?)
Quoted 16 lines · On Mon, 7 Dec 2015 08:59:25 -0600 (CST) Christoph Lameter <[email protected]> wrote:
Quoted 14 lines · Jesper Dangaard Brouer
Quoted 8 lines
mm/slab.c void kmem_cache_free(struct kmem_cache *cachep, void *objp) · line 3625
Quoted 6 lines
mm/slab.c +6
-0+ +local_irq_disable (); +for (i =0 ; i < size; i++) { +void *objp = p[i]; + + s =cache_from_obj (orig_s, objp);Does this support freeing objects from a set of different caches?
This is for supporting memcg (
CONFIG_MEMCG_KMEM).Quoting from commit
033745189b1b("slub: add missing kmem cgroup support tokmem_cache_free_bulk"):
Quoted 7 lines
Quoted 4 lines
Incoming bulk free objects can belong to different kmem cgroups, and object free call can happen at a later point outside memcg context. Thus, we need to keep the orig
kmem_cache, to correctly verify if a memcg object match against its "root_cache" (s->memcg_params.root_cache).Where is that verification? This looks like SLAB would support freeing objects from different caches.
This is for supporting CONFIG_MEMCG_KMEM, thus I would like Vladimir input on this, as I don't know enough about memcg....
Signature · 11 lines
Best regards, Jesper Dangaard Brouer MSc.CS, Principal Kernel Engineer at Red Hat Author of http://www.iptv-analyzer.org LinkedIn: http://www.linkedin.com/in/brouer -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH 2/2] slab: implement bulk free in SLAB allocator
Quoted 12 lines · Jesper Dangaard Brouer
Quoted 12 lines
Quoted 11 lines
Quoted 1 line
Does this support freeing objects from a set of different caches?
This is for supporting memcg (
CONFIG_MEMCG_KMEM).Quoting from commit
033745189b1b("slub: add missing kmem cgroup support tokmem_cache_free_bulk"):Incoming bulk free objects can belong to different kmem cgroups, and object free call can happen at a later point outside memcg context. Thus, we need to keep the orig
kmem_cache, to correctly verify if a memcg object match against its "root_cache" (s->memcg_params.root_cache).
Quoted 5 lines
Quoted 2 lines
Where is that verification? This looks like SLAB would support freeing objects from different caches.
This is for supporting
CONFIG_MEMCG_KMEM, thus I would like Vladimir input on this, as I don't know enough about memcg....
Oww... So far objects passed to bulk free must be confined to the *same* slab cache. Not the same "root_cache". Are we changing that?
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH 2/2] slab: implement bulk free in SLAB allocator
In reply to Christoph LameterQuoted 22 lines · Christoph Lameter
Quoted 3 lines · Jesper Dangaard Brouer
Quoted 1 line
s?
The "s" comes from the
slub.ccode uses "structkmem_cache*s" everywhere.Ok then use it. Why is there an
orig_shere.Quoted 14 lines
Quoted 8 lines
mm/slab.c void kmem_cache_free(struct kmem_cache *cachep, void *objp) · line 3625
Quoted 6 lines
mm/slab.c +6
-0+ +local_irq_disable (); +for (i =0 ; i < size; i++) { +void *objp = p[i]; + + s =cache_from_obj (orig_s, objp);Does this support freeing objects from a set of different caches?
This is for supporting memcg (
CONFIG_MEMCG_KMEM).Quoting from commit
033745189b1b("slub: add missing kmem cgroup support tokmem_cache_free_bulk"):
Quoted 7 lines
Quoted 4 lines
Incoming bulk free objects can belong to different kmem cgroups, and object free call can happen at a later point outside memcg context. Thus, we need to keep the orig
kmem_cache, to correctly verify if a memcg object match against its "root_cache" (s->memcg_params.root_cache).Where is that verification? This looks like SLAB would support freeing objects from different caches.
As Jesper explained to me in the thread regarding the SLUB version of this API (see http://www.spinics.net/lists/linux-mm/msg96728.html),
objects allocated by kmem_cache_alloc_bulk() will not necessarily be freed by kmem_cache_free_bulk() and vice-versa. For instance, it is possible that a bunch of objects allocated using kmem_cache_alloc() will be freed with a single kmem_cache_free_bulk() call. As a result, we can't prevent users of the API from doing something like this:
1. Multiple producers allocate objects of the same kind using kmem_cache_alloc() and pass them to the consumer
2. The consumer processes the objects and frees as many as possible using kmem_cache_bulk()
If producers are represented by different processes, they can belong to different memory cgroups, so that objects passed to the consumer will come from different kmem caches (per memcg caches), although they are all of the same kind. This means, we must call cache_from_obj() on each object passed to kmem_cache_free_bulk() in order to free each object to the cache it was allocated from.
Thanks,
Vladimir
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH 2/2] slab: implement bulk free in SLAB allocator
Quoted 6 lines · Vladimir Davydov
If producers are represented by different processes, they can belong to different memory cgroups, so that objects passed to the consumer will come from different kmem caches (per memcg caches), although they are all of the same kind. This means, we must call
cache_from_obj()on each object passed tokmem_cache_free_bulk()in order to free each object to the cache it was allocated from.
The we should change the API so that we do not specify kmem_cache on bulk free. Do it like kfree without any cache spec.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH 2/2] slab: implement bulk free in SLAB allocator
Quoted 9 lines · Christoph Lameter
Quoted 6 lines · Vladimir Davydov
If producers are represented by different processes, they can belong to different memory cgroups, so that objects passed to the consumer will come from different kmem caches (per memcg caches), although they are all of the same kind. This means, we must call
cache_from_obj()on each object passed tokmem_cache_free_bulk()in order to free each object to the cache it was allocated from.
Quoted 3 lines
The we should change the API so that we do not specify
kmem_cacheon bulk free. Do it like kfree without any cache spec.
Don't think so, because AFAIU the whole kmem_cache_free_bulk optimization comes from the assumption that objects passed to it are likely to share the same slab page. So they must be of the same kind, otherwise no optimization would be possible and the function wouldn't perform any better than calling kfree directly in a for-loop. By requiring the caller to specify the cache we emphasize this.
Enabling kmemcg might break the assumption and neglect the benefit of using kmem_cache_free_bulk, but it is to be expected, because kmem accounting does not come for free. Callers who do care about the performance and count every cpu cycle will surely disable it, in which case cache_from_obj() will be a no-op and kmem_cache_free_bulk will bear fruit.
Thanks,
Vladimir
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH 2/2] slab: implement bulk free in SLAB allocator
Quoted 14 lines · On Tue, 8 Dec 2015 17:56:35 +0300 Vladimir Davydov <[email protected]>…
Quoted 9 lines · Christoph Lameter
Quoted 6 lines · Vladimir Davydov
If producers are represented by different processes, they can belong to different memory cgroups, so that objects passed to the consumer will come from different kmem caches (per memcg caches), although they are all of the same kind. This means, we must call
cache_from_obj()on each object passed tokmem_cache_free_bulk()in order to free each object to the cache it was allocated from.Quoted 3 lines
The we should change the API so that we do not specify
kmem_cacheon bulk free. Do it like kfree without any cache spec.
Quoted 6 lines
Don't think so, because AFAIU the whole
kmem_cache_free_bulkoptimization comes from the assumption that objects passed to it are likely to share the same slab page. So they must be of the same kind, otherwise no optimization would be possible and the function wouldn't perform any better than calling kfree directly in a for-loop. By requiring the caller to specify the cache we emphasize this.
I agree with Vladimir here. The performance gain for SLUB is especially depended on that objects are likely to share the same slab page. It might not hurt SLAB as much.
Quoted 6 lines
Enabling kmemcg might break the assumption and neglect the benefit of using
kmem_cache_free_bulk, but it is to be expected, because kmem accounting does not come for free. Callers who do care about the performance and count every cpu cycle will surely disable it, in which casecache_from_obj()will be a no-op andkmem_cache_free_bulkwill bear fruit.
True, compiler does realize, when CONFIG_MEMCG_KMEM is disabled, that it can optimize the call to cache_from_obj() away.
Signature · 11 lines
Best regards, Jesper Dangaard Brouer MSc.CS, Principal Kernel Engineer at Red Hat Author of http://www.iptv-analyzer.org LinkedIn: http://www.linkedin.com/in/brouer -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH 2/2] slab: implement bulk free in SLAB allocator
In reply to Vladimir DavydovQuoted 6 lines · Vladimir Davydov
Don't think so, because AFAIU the whole
kmem_cache_free_bulkoptimization comes from the assumption that objects passed to it are likely to share the same slab page. So they must be of the same kind, otherwise no optimization would be possible and the function wouldn't perform any better than calling kfree directly in a for-loop. By requiring the caller to specify the cache we emphasize this.
This is likely but an implementation specific feature that only SLUB can exploit. However, one page can only contain objects from the same slab page. So checking the slab cache too is irrelevant. We could take it out.
If the logic finds two objects that share the same page then they will be from the same slab cache. The checking of the cache is just not necessary and actually increases the code size and therefore reduces performance.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 7 lines · On Thu, 03 Dec 2015 16:56:32 +0100 Jesper Dangaard Brouer <[email protected]>…
Normal SLAB fastpath 95 cycles(tsc) 23.852 ns, when compiled without debugging options enabled.
Benchmarked[1] obj size 256 bytes on CPU i7-4790K @ 4.00GHz:
[1] https://github.com/netoptimizer/prototype-kernel/blob/master/kernel/mm/slab_bulk_test01.c1 - 115 cycles(tsc) 28.812 ns - 42 cycles(tsc) 10.715 ns - improved 63.5%
Quoted 13 lines
2 - 103 cycles(tsc) 25.956 ns - 27 cycles(tsc) 6.985 ns - improved 73.8% 3 - 101 cycles(tsc) 25.336 ns - 22 cycles(tsc) 5.733 ns - improved 78.2% 4 - 100 cycles(tsc) 25.147 ns - 21 cycles(tsc) 5.319 ns - improved 79.0% 8 - 98 cycles(tsc) 24.616 ns - 18 cycles(tsc) 4.620 ns - improved 81.6% 16 - 97 cycles(tsc) 24.408 ns - 17 cycles(tsc) 4.344 ns - improved 82.5% 30 - 98 cycles(tsc) 24.641 ns - 16 cycles(tsc) 4.202 ns - improved 83.7% 32 - 98 cycles(tsc) 24.607 ns - 16 cycles(tsc) 4.199 ns - improved 83.7% 34 - 98 cycles(tsc) 24.605 ns - 18 cycles(tsc) 4.579 ns - improved 81.6% 48 - 97 cycles(tsc) 24.463 ns - 17 cycles(tsc) 4.405 ns - improved 82.5% 64 - 97 cycles(tsc) 24.370 ns - 17 cycles(tsc) 4.384 ns - improved 82.5% 128 - 99 cycles(tsc) 24.763 ns - 19 cycles(tsc) 4.755 ns - improved 80.8% 158 - 98 cycles(tsc) 24.708 ns - 18 cycles(tsc) 4.723 ns - improved 81.6% 250 - 101 cycles(tsc) 25.342 ns - 20 cycles(tsc) 5.035 ns - improved 80.2%
Ups, copy-past mistake, these (above) were the old results, from the old rejected patch [2].
The results from this patchset is:
1 - 112 cycles(tsc) 28.060 ns - 45 cycles(tsc) 11.454 ns - improved 59.8% 2 - 102 cycles(tsc) 25.735 ns - 28 cycles(tsc) 7.038 ns - improved 72.5%
3 - 98 cycles(tsc) 24.666 ns - 22 cycles(tsc) 5.518 ns - improved 77.6% 4 - 97 cycles(tsc) 24.437 ns - 18 cycles(tsc) 4.746 ns - improved 81.4% 8 - 95 cycles(tsc) 23.875 ns - 15 cycles(tsc) 3.782 ns - improved 84.2% 16 - 95 cycles(tsc) 24.002 ns - 14 cycles(tsc) 3.621 ns - improved 85.3% 30 - 95 cycles(tsc) 23.893 ns - 14 cycles(tsc) 3.577 ns - improved 85.3% 32 - 95 cycles(tsc) 23.875 ns - 13 cycles(tsc) 3.402 ns - improved 86.3% 34 - 95 cycles(tsc) 23.794 ns - 13 cycles(tsc) 3.385 ns - improved 86.3% 48 - 94 cycles(tsc) 23.721 ns - 14 cycles(tsc) 3.550 ns - improved 85.1% 64 - 94 cycles(tsc) 23.608 ns - 13 cycles(tsc) 3.427 ns - improved 86.2% 128 - 96 cycles(tsc) 24.045 ns - 15 cycles(tsc) 3.936 ns - improved 84.4% 158 - 95 cycles(tsc) 23.886 ns - 17 cycles(tsc) 4.289 ns - improved 82.1% 250 - 97 cycles(tsc) 24.358 ns - 17 cycles(tsc) 4.329 ns - improved 82.5%
These results are likely better because there is now less instructions in the more tight bulk loops (mostly in kmem_cache_alloc_bulk). SLUB itself might also have improved at bit since old patch.
Signature · 11 lines
Best regards, Jesper Dangaard Brouer MSc.CS, Principal Kernel Engineer at Red Hat Author of http://www.iptv-analyzer.org LinkedIn: http://www.linkedin.com/in/brouer -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
v2 cover slab: cleanup and bulk API for SLAB
In reply to Jesper Dangaard BrouerRFC series for code sharing between SLUB and SLAB, and dedublication of code in SLAB, as Christoph requested. See questions in code.
Plus adding bulk API implemenation for SLAB.
---
Jesper Dangaard Brouer (9):
mm/slab: move SLUB alloc hooks to common mm/slab.h
mm: generalize avoid fault-inject on bootstrap kmem_cache
slab: use slab_pre_alloc_hook in SLAB allocator
mm: kmemcheck skip object if slab allocation failed
slab: use slab_post_alloc_hook in SLAB allocator
slab: implement bulk alloc in SLAB allocator
slab: avoid running debug SLAB code with IRQs disabled for alloc_bulk
slab: implement bulk free in SLAB allocator
slab: annotate code to generate more compact asm code
mm/failslab.c | 2 + mm/kmemcheck.c | 3 + mm/slab.c | 122 +++++++++++++++++++++++++++++++++++--------------------- mm/slab.h | 92 ++++++++++++++++++++++++++++++++++++++++++ mm/slub.c | 54 ------------------------- 5 files changed, 174 insertions(+), 99 deletions(-)
--
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH v2 1/9 mm/slab: move SLUB alloc hooks to common mm/slab.h
First step towards sharing alloc_hook's between SLUB and SLAB allocators. Move the SLUB allocators *_alloc_hook to the common mm/slab.h for internal slab definitions.
--- mm/slab.h | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ mm/slub.c | 54 ---------------------------------------------- 2 files changed, 71 insertions(+), 54 deletions(-)
mm/slab.h +71 -0
@@ -38,6 +38,17 @@ struct kmem_cache {#endif #include <linux/memcontrol.h> +#include <linux/fault-inject.h> +/* Q: Howto handle this nicely? below includes are needed for alloc hooks +* +* e.g. mm/mempool.c and mm/slab_common.c does not include kmemcheck.h +* including it here solves the probem, but should they include it +* themselves? +*/ +#include <linux/kmemcheck.h> +// Below includes are already included in other users of "mm/slab.h" +//#include <linux/kasan.h> +//#include <linux/kmemleak.h> /* * State of the slab allocator.
@@ -319,6 +330,66 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)return s; } +#ifdef CONFIG_SLUB +static inline size_t slab_ksize (const struct kmem_cache *s) +{ +#ifdef CONFIG_SLUB_DEBUG +/* +* Debugging requires use of the padding between object +* and whatever may come after it. +*/ +if (s->flags & (SLAB_RED_ZONE |SLAB_POISON )) +return s->object_size; +#endif +/* +* If we have the need to store the freelist pointer +* back there or track user information then we can +* only use the space before that information. +*/ +if (s->flags & (SLAB_DESTROY_BY_RCU |SLAB_STORE_USER )) +return s->inuse; +/* +* Else we can use all the padding etc for the allocation +*/ +return s->size; +} +#else /* !CONFIG_SLUB */ +static inline size_t slab_ksize (const struct kmem_cache *s) +{ +return s->object_size; +} +#endif + +static inline struct kmem_cache *slab_pre_alloc_hook (struct kmem_cache *s, +gfp_t flags) +{ + flags &= gfp_allowed_mask; +lockdep_trace_alloc (flags); +might_sleep_if (gfpflags_allow_blocking (flags)); + +if (should_failslab (s->object_size, flags, s->flags)) +return NULL ; + +return memcg_kmem_get_cache (s, flags); +} + +static inline void slab_post_alloc_hook (struct kmem_cache *s,gfp_t flags, +size_t size,void **p) +{ +size_t i; + + flags &= gfp_allowed_mask; +for (i =0 ; i < size; i++) { +void *object = p[i]; + +kmemcheck_slab_alloc (s, flags, object,slab_ksize (s)); +kmemleak_alloc_recursive (object, s->object_size,1 , + s->flags, flags); +kasan_slab_alloc (s, object); + } +memcg_kmem_put_cache (s); +} +#ifndef CONFIG_SLOB /* * The slab lists for all objects.
mm/slub.c +0 -54
66 lines not on this page · raw message
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH V2 1/9] mm/slab: move SLUB alloc hooks to common mm/slab.h
Quoted 6 lines · Jesper Dangaard Brouer
+/* Q: Howto handle this nicely? below includes are needed for alloc hooks
+ *
+ * e.g.mm/mempool.candmm/slab_common.cdoes not includekmemcheck.h
+ * including it here solves the probem, but should they include it
+ * themselves?
+ */
Including in mm/slab.h is enough.
Quoted 1 line
+#ifdef
CONFIG_SLUB
Move this into slab_ksize?
Quoted 13 lines
mm/slab.h +13
-0+static inline size_t slab_ksize (const struct kmem_cache *s) +{ +#ifdef CONFIG_SLUB_DEBUG +/* +* Debugging requires use of the padding between object +* and whatever may come after it. +*/ +if (s->flags & (SLAB_RED_ZONE |SLAB_POISON )) +return s->object_size; +#endif +/* +* If we have the need to store the freelist pointer +* back there or track user information then we can
mm/slab.h static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) · line 356
Quoted 10 lines
mm/slab.h +10
-0+* only use the space before that information. +*/ +if (s->flags & (SLAB_DESTROY_BY_RCU |SLAB_STORE_USER )) +return s->inuse; +/* +* Else we can use all the padding etc for the allocation +*/ + return s->size; +} +#else /* !CONFIG_SLUB */
Abnd drop the else branch?
mm/slab.h static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) · line 361
Quoted 5 lines
mm/slab.h +5
-0+static inline size_t slab_ksize (const struct kmem_cache *s) +{ +return s->object_size; +} +#endif
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH v2 2/9 mm: generalize avoid fault-inject on bootstrap kmem_cache
In reply to Jesper Dangaard BrouerMove slab_should_failslab() check from SLAB allocator to generic slab_pre_alloc_hook(). The check guards against slab alloc fault-injects failures for the bootstrap slab that is used for allocating "kmem_cache" objects to the allocator itself.
I'm not really happy with this code...
--- mm/failslab.c | 2 ++ mm/slab.c | 8 -------- mm/slab.h | 23 ++++++++++++++++++++++- 3 files changed, 24 insertions(+), 9 deletions(-)
mm/failslab.c +2 -0
@@ -13,6 +13,8 @@ static struct {bool should_failslab (size_t size,gfp_t gfpflags,unsigned long cache_flags) { +// Should we place bootstrap kmem_cache check here??? +if (gfpflags & __GFP_NOFAIL)return false;
mm/slab.c +0 -8
@@ -2917,14 +2917,6 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,#define cache_alloc_debugcheck_after (a,b,objp,d) (objp)#endif -static bool slab_should_failslab (struct kmem_cache *cachep,gfp_t flags) -{ -if (unlikely (cachep == kmem_cache)) -return false; - -return should_failslab (cachep->object_size, flags, cachep->flags); -} -static inline void *____cache_alloc (struct kmem_cache *cachep,gfp_t flags) {void *objp;
mm/slab.h +22 -1
@@ -360,6 +360,27 @@ static inline size_t slab_ksize(const struct kmem_cache *s) }#endif +/* FIXME: This construct sucks, because this compare+branch needs to +* get removed by compiler then !CONFIG_FAILSLAB (maybe compiler is +* smart enough to realize only "false" can be generated). +* +* Comments please: Pulling out CONFIG_FAILSLAB here looks ugly... +* should we instead change API of should_failslab() ?? +* +* Next question: is the bootstrap cache check okay to add for all +* allocators? (this would be the easiest, else need more ugly ifdef's) +*/ +static inline bool slab_should_failslab (struct kmem_cache *cachep,gfp_t flags) +{ +/* No fault-injection for bootstrap cache */ +#ifdef CONFIG_FAILSLAB +if (unlikely (cachep == kmem_cache)) +return false; +#endif + +return should_failslab (cachep->object_size, flags, cachep->flags); +} +static inline struct kmem_cache *slab_pre_alloc_hook (struct kmem_cache *s,gfp_t flags) {
@@ -367,7 +388,7 @@ static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,lockdep_trace_alloc (flags);might_sleep_if (gfpflags_allow_blocking (flags)); -if (should_failslab (s->object_size, flags, s->flags)) +if (slab_should_failslab (s, flags))return NULL ;return memcg_kmem_get_cache (s, flags);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH V2 2/9] mm: generalize avoid fault-inject on bootstrap kmem_cache
Quoted 77 lines · Jesper Dangaard Brouer
Move
slab_should_failslab()check from SLAB allocator to genericslab_pre_alloc_hook(). The check guards against slab alloc fault-injects failures for the bootstrap slab that is used for allocating "kmem_cache" objects to the allocator itself.I'm not really happy with this code...
--- mm/failslab.c | 2 ++ mm/slab.c | 8 -------- mm/slab.h | 23 ++++++++++++++++++++++- 3 files changed, 24 insertions(+), 9 deletions(-)mm/failslab.c +2
-0@@ -13,6 +13,8 @@ static struct {bool should_failslab (size_t size,gfp_t gfpflags,unsigned long cache_flags) { +// Should we place bootstrap kmem_cache check here??? +if (gfpflags & __GFP_NOFAIL)return false;mm/slab.c +0
-8@@ -2917,14 +2917,6 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,#define cache_alloc_debugcheck_after (a,b,objp,d) (objp)#endif -static bool slab_should_failslab (struct kmem_cache *cachep,gfp_t flags) -{ -if (unlikely (cachep == kmem_cache)) -return false; - -return should_failslab (cachep->object_size, flags, cachep->flags); -} -static inline void *____cache_alloc (struct kmem_cache *cachep,gfp_t flags) {void *objp;mm/slab.h +21
-0@@ -360,6 +360,27 @@ static inline size_t slab_ksize(const struct kmem_cache *s) }#endif +/* FIXME: This construct sucks, because this compare+branch needs to +* get removed by compiler then !CONFIG_FAILSLAB (maybe compiler is +* smart enough to realize only "false" can be generated). +* +* Comments please: Pulling out CONFIG_FAILSLAB here looks ugly... +* should we instead change API of should_failslab() ?? +* +* Next question: is the bootstrap cache check okay to add for all +* allocators? (this would be the easiest, else need more ugly ifdef's) +*/ +static inline bool slab_should_failslab (struct kmem_cache *cachep,gfp_t flags) +{ +/* No fault-injection for bootstrap cache */ +#ifdef CONFIG_FAILSLAB +if (unlikely (cachep == kmem_cache)) +return false; +#endif + +return should_failslab (cachep->object_size, flags, cachep->flags); +} +static inline struct kmem_cache *slab_pre_alloc_hook (struct kmem_cache *s,gfp_t flags) {
mm/slab.h static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, · line 392
Quoted 6 lines
mm/slab.h +1
-1@@ -367,7 +388,7 @@ static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,lockdep_trace_alloc (flags);might_sleep_if (gfpflags_allow_blocking (flags)); -if (should_failslab (s->object_size, flags, s->flags)) +if (slab_should_failslab (s, flags))return NULL ;
It'd be better to remove slab_should_failslab() and insert following code snippet here.
if (IS_ENABLED(CONFIG_FAILSLAB) &&
cachep != kmem_cache &&
should_failslab())
return NULL;Thanks.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH v2 3/9 slab: use slab_pre_alloc_hook in SLAB allocator
In reply to Jesper Dangaard BrouerDedublicate code in slab_alloc() and slab_alloc_node() by using the slab_pre_alloc_hook() call.
--- mm/slab.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-)
mm/slab.c +4 -14
@@ -3140,15 +3140,10 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,void *ptr;int slab_node =numa_mem_id (); - flags &= gfp_allowed_mask; - -lockdep_trace_alloc (flags); - -if (slab_should_failslab (cachep, flags)) + cachep =slab_pre_alloc_hook (cachep, flags); +if (!cachep)return NULL ; - cachep =memcg_kmem_get_cache (cachep, flags); -cache_alloc_debugcheck_before (cachep, flags);local_irq_save (save_flags);
@@ -3228,15 +3223,10 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)unsigned long save_flags;void *objp; - flags &= gfp_allowed_mask; - -lockdep_trace_alloc (flags); - -if (slab_should_failslab (cachep, flags)) + cachep =slab_pre_alloc_hook (cachep, flags); +if (!cachep)return NULL ; - cachep =memcg_kmem_get_cache (cachep, flags); -cache_alloc_debugcheck_before (cachep, flags);local_irq_save (save_flags); objp =__do_cache_alloc (cachep, flags);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH v2 4/9 mm: kmemcheck skip object if slab allocation failed
In reply to Jesper Dangaard BrouerIn the SLAB allocator kmemcheck_slab_alloc() is guarded against being called in case the object is NULL. In SLUB allocator this NULL pointer invocation can happen, which seems like an oversight.
Move the NULL pointer check into kmemcheck code (kmemcheck_slab_alloc) so the check gets moved out of the fastpath, when not compiled with CONFIG_KMEMCHECK.
--- mm/kmemcheck.c | 3 +++ 1 file changed, 3 insertions(+)
mm/kmemcheck.c +3 -0
@@ -60,6 +60,9 @@ void kmemcheck_free_shadow(struct page *page, int order)void kmemcheck_slab_alloc (struct kmem_cache *s,gfp_t gfpflags,void *object,size_t size) { +if (unlikely (!object))/* Skip object if allocation failed */ +return ; +/* * Has already been memset(), which initializes the shadow for us * as well.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH v2 5/9 slab: use slab_post_alloc_hook in SLAB allocator
In reply to Jesper Dangaard BrouerReviewers notice that the order in slab_post_alloc_hook() of kmemcheck_slab_alloc() and kmemleak_alloc_recursive() gets swapped compared to slab.c.
Also notice memset now occurs before calling kmemcheck_slab_alloc() and kmemleak_alloc_recursive().
--- mm/slab.c | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-)
mm/slab.c +6 -16
@@ -3172,16 +3172,11 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, out:local_irq_restore (save_flags); ptr =cache_alloc_debugcheck_after (cachep, flags, ptr, caller); -kmemleak_alloc_recursive (ptr, cachep->object_size,1 , cachep->flags, - flags); -if (likely (ptr)) { -kmemcheck_slab_alloc (cachep, flags, ptr, cachep->object_size); -if (unlikely (flags & __GFP_ZERO)) -memset (ptr,0 , cachep->object_size); - } +if (unlikely (flags & __GFP_ZERO) && ptr) +memset (ptr,0 , cachep->object_size); -memcg_kmem_put_cache (cachep); +slab_post_alloc_hook (cachep, flags,1 , &ptr);return ptr; }
@@ -3232,17 +3227,12 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller) objp =__do_cache_alloc (cachep, flags);local_irq_restore (save_flags); objp =cache_alloc_debugcheck_after (cachep, flags, objp, caller); -kmemleak_alloc_recursive (objp, cachep->object_size,1 , cachep->flags, - flags);prefetchw (objp); -if (likely (objp)) { -kmemcheck_slab_alloc (cachep, flags, objp, cachep->object_size); -if (unlikely (flags & __GFP_ZERO)) -memset (objp,0 , cachep->object_size); - } +if (unlikely (flags & __GFP_ZERO) && objp) +memset (objp,0 , cachep->object_size); -memcg_kmem_put_cache (cachep); +slab_post_alloc_hook (cachep, flags,1 , &objp);return objp; }
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH v2 6/9 slab: implement bulk alloc in SLAB allocator
In reply to Jesper Dangaard Brouer--- mm/slab.c | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-)
mm/slab.c +35 -2
@@ -3392,9 +3392,42 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)EXPORT_SYMBOL (kmem_cache_free_bulk);int kmem_cache_alloc_bulk (struct kmem_cache *s,gfp_t flags,size_t size, -void **p) +void **p) { -return __kmem_cache_alloc_bulk (s, flags, size, p); +size_t i; + + s =slab_pre_alloc_hook (s, flags); +if (!s) +return 0 ; + +cache_alloc_debugcheck_before (s, flags); + +local_irq_disable (); +for (i =0 ; i < size; i++) { +void *objp =__do_cache_alloc (s, flags); + +/* this call could be done outside IRQ disabled section */ + objp =cache_alloc_debugcheck_after (s, flags, objp, _RET_IP_); + +if (unlikely (!objp)) + goto error; + p[i] = objp; + } +local_irq_enable (); + +/* Clear memory outside IRQ disabled section */ +if (unlikely (flags & __GFP_ZERO)) +for (i =0 ; i < size; i++) +memset (p[i],0 , s->object_size); + +slab_post_alloc_hook (s, flags, size, p); +/* FIXME: Trace call missing. Christoph would like a bulk variant */ +return size; +error: +local_irq_enable (); +slab_post_alloc_hook (s, flags, i, p); +__kmem_cache_free_bulk (s, i, p); +return 0 ; }EXPORT_SYMBOL (kmem_cache_alloc_bulk);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH v2 7/9 slab: avoid running debug SLAB code with IRQs disabled for alloc_bulk
In reply to Jesper Dangaard BrouerMove the call to cache_alloc_debugcheck_after() outside the IRQ disabled section in kmem_cache_alloc_bulk().
When CONFIG_DEBUG_SLAB is disabled the compiler should remove this code.
--- mm/slab.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-)
mm/slab.c +13 -3
@@ -3391,6 +3391,16 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p) }EXPORT_SYMBOL (kmem_cache_free_bulk); +static __always_inline void +cache_alloc_debugcheck_after_bulk (struct kmem_cache *s,gfp_t flags, +size_t size,void **p,unsigned long caller) +{ +size_t i; + +for (i =0 ; i < size; i++) + p[i] =cache_alloc_debugcheck_after (s, flags, p[i], _RET_IP_); +} +int kmem_cache_alloc_bulk (struct kmem_cache *s,gfp_t flags,size_t size,void **p) {
@@ -3406,15 +3416,14 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,for (i =0 ; i < size; i++) {void *objp =__do_cache_alloc (s, flags); -/* this call could be done outside IRQ disabled section */ - objp =cache_alloc_debugcheck_after (s, flags, objp, _RET_IP_); -if (unlikely (!objp)) goto error; p[i] = objp; }local_irq_enable (); +cache_alloc_debugcheck_after_bulk (s, flags, size, p, _RET_IP_); +/* Clear memory outside IRQ disabled section */ if (unlikely (flags & __GFP_ZERO))for (i =0 ; i < size; i++)
@@ -3425,6 +3434,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,return size; error:local_irq_enable (); +cache_alloc_debugcheck_after_bulk (s, flags, i, p, _RET_IP_);slab_post_alloc_hook (s, flags, i, p);__kmem_cache_free_bulk (s, i, p);return 0 ;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH v2 8/9 slab: implement bulk free in SLAB allocator
In reply to Jesper Dangaard Brouer--- mm/slab.c | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-)
mm/slab.c +23 -6
@@ -3385,12 +3385,6 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) }EXPORT_SYMBOL (kmem_cache_alloc); -void kmem_cache_free_bulk (struct kmem_cache *s,size_t size,void **p) -{ -__kmem_cache_free_bulk (s, size, p); -} -EXPORT_SYMBOL (kmem_cache_free_bulk); - static __always_inline voidcache_alloc_debugcheck_after_bulk (struct kmem_cache *s,gfp_t flags,size_t size,void **p,unsigned long caller)
@@ -3584,6 +3578,29 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) }EXPORT_SYMBOL (kmem_cache_free); +void kmem_cache_free_bulk (struct kmem_cache *orig_s,size_t size,void **p) +{ +struct kmem_cache *s; +size_t i; + +local_irq_disable (); +for (i =0 ; i < size; i++) { +void *objp = p[i]; + + s =cache_from_obj (orig_s, objp); + +debug_check_no_locks_freed (objp, s->object_size); +if (!(s->flags &SLAB_DEBUG_OBJECTS )) +debug_check_no_obj_freed (objp, s->object_size); + +__cache_free (s, objp, _RET_IP_); + } +local_irq_enable (); + +/* FIXME: add tracing */ +} +EXPORT_SYMBOL (kmem_cache_free_bulk); +/** * kfree - free previously allocated memory * @objp: pointer returned by kmalloc.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH V2 8/9] slab: implement bulk free in SLAB allocator
Quoted 1 line · Jesper Dangaard Brouer
+void
kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
Drop orig_s as a parameter? This makes the function have less code and makes it more universally useful for freeing large amount of objects.
Review
Quoted 9 lines
(quoted diff) +9
-0+{ + struct kmem_cache *s; + size_t i; + + local_irq_disable(); + for (i = 0; i < size; i++) { + void *objp = p[i]; + + s = cache_from_obj(orig_s, objp);
And just use the cache the object belongs to.
s = virt_to_head_page(objp)->slab_cache;
Review
Quoted 12 lines
(quoted diff) +12
-0+ + debug_check_no_locks_freed(objp, s->object_size); + if (!(s->flags & SLAB_DEBUG_OBJECTS)) + debug_check_no_obj_freed(objp, s->object_size); + + __cache_free(s, objp, _RET_IP_); + } + local_irq_enable(); + + /* FIXME: add tracing */ +} +EXPORT_SYMBOL(kmem_cache_free_bulk);
Could we do the following API change patch before this series so that kmem_cache_free_bulk is properly generalized?
From: Christoph Lameter <[email protected]>
Subject: slab bulk api: Remove the kmem_cache parameter from kmem_cache_bulk_free()
It is desirable and necessary to free objects from different kmem_caches. It is required in order to support memcg object freeing across different5 cgroups.
So drop the pointless parameter and allow freeing of arbitrary lists of slab allocated objects.
This patch also does the proper compound page handling so that arbitrary objects allocated via kmalloc() can be handled by kmem_cache_bulk_free().
include/linux/slab.h +1 -1
@@ -315,7 +315,7 @@ void kmem_cache_free(struct kmem_cache ** * Note that interrupts must be enabled when calling these functions. */ -void kmem_cache_free_bulk (struct kmem_cache *,size_t ,void **); +void kmem_cache_free_bulk (size_t ,void **);int kmem_cache_alloc_bulk (struct kmem_cache *,gfp_t ,size_t ,void **);#ifdef CONFIG_NUMA
mm/slab.c +2 -2
@@ -3413,9 +3413,9 @@ void *kmem_cache_alloc(struct kmem_cache }EXPORT_SYMBOL (kmem_cache_alloc); -void kmem_cache_free_bulk (struct kmem_cache *s,size_t size,void **p) +void kmem_cache_free_bulk (size_t size,void **p) { -__kmem_cache_free_bulk (s, size, p); +__kmem_cache_free_bulk (size, p); }EXPORT_SYMBOL (kmem_cache_free_bulk);
mm/slab.h +2 -2
@@ -166,10 +166,10 @@ ssize_t slabinfo_write(struct file *file/* * Generic implementation of bulk operations * These are useful for situations in which the allocator cannot -* perform optimizations. In that case segments of the objecct listed +* perform optimizations. In that case segments of the object listed * may be allocated or freed using these operations. */ -void __kmem_cache_free_bulk (struct kmem_cache *,size_t ,void **); +void __kmem_cache_free_bulk (size_t ,void **);int __kmem_cache_alloc_bulk (struct kmem_cache *,gfp_t ,size_t ,void **);#ifdef CONFIG_MEMCG_KMEM
mm/slub.c +14 -7
@@ -2887,23 +2887,30 @@ static int build_detached_freelist(struc/* Note that interrupts must be enabled when calling this function. */ -void kmem_cache_free_bulk (struct kmem_cache *orig_s,size_t size,void **p) +void kmem_cache_free_bulk (size_t size,void **p) {if (WARN_ON (!size))return ;do {struct detached_freelist df; -struct kmem_cache *s; +struct page *page; -/* Support for memcg */ - s =cache_from_obj (orig_s, p[size -1 ]); + page =virt_to_head_page (p[size -1 ]); - size =build_detached_freelist (s, size, p, &df); +if (unlikely (!PageSlab (page))) { +BUG_ON (!PageCompound (page)); +kfree_hook (p[size -1 ]); +__free_kmem_pages (page,compound_order (page)); + p[--size] =NULL ; +continue ; + } + + size =build_detached_freelist (page->slab_cache, size, p, &df);if (unlikely (!df.page))continue ; -slab_free (s, df.page, df.freelist, df.tail, df.cnt, _RET_IP_); +slab_free (page->slab_cache, df.page, df.freelist, df.tail, df.cnt, _RET_IP_); }while (likely (size)); }EXPORT_SYMBOL (kmem_cache_free_bulk);
@@ -2963,7 +2970,7 @@ int kmem_cache_alloc_bulk(struct kmem_ca error:local_irq_enable ();slab_post_alloc_hook (s, flags, i, p); -__kmem_cache_free_bulk (s, i, p); +__kmem_cache_free_bulk (i, p);return 0 ; }EXPORT_SYMBOL (kmem_cache_alloc_bulk);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH V2 8/9] slab: implement bulk free in SLAB allocator
Quoted 6 lines · On Wed, 9 Dec 2015 10:06:39 -0600 (CST) Christoph Lameter <[email protected]> wrote:
Quoted 1 line · Jesper Dangaard Brouer
+void
kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)Drop
orig_sas a parameter? This makes the function have less code and makes it more universally useful for freeing large amount of objects.
I really like the idea of making it able to free kmalloc'ed objects. But I hate to change the API again... (I do have a use-case in the network stack where I could use this feature).
Quoted 2 lines
Could we do the following API change patch before this series so that
kmem_cache_free_bulkis properly generalized?
I'm traveling (to Sweden) Thursday (afternoon) and will be back late Friday (and have to play Viking in the weekend), thus to be realistic I'll start working on this Monday, so we can get some benchmark numbers to guide this decision.
Quoted 74 lines
From: Christoph Lameter <[email protected]>
Subject: slab bulk api: Remove thekmem_cacheparameter fromkmem_cache_bulk_free()It is desirable and necessary to free objects from different
kmem_caches. It is required in order to support memcg object freeing across different5 cgroups.So drop the pointless parameter and allow freeing of arbitrary lists of slab allocated objects.
This patch also does the proper compound page handling so that arbitrary objects allocated via
kmalloc()can be handled bykmem_cache_bulk_free().Signed-off-by Christoph Lameter <[email protected]>Index:
linux/include/linux/slab.h
===================================================================include/linux/slab.h +1
-1@@ -315,7 +315,7 @@ void kmem_cache_free(struct kmem_cache ** * Note that interrupts must be enabled when calling these functions. */ -void kmem_cache_free_bulk (struct kmem_cache *,size_t ,void **); +void kmem_cache_free_bulk (size_t ,void **);int kmem_cache_alloc_bulk (struct kmem_cache *,gfp_t ,size_t ,void **);#ifdef CONFIG_NUMA Index:
linux/mm/slab.c
===================================================================mm/slab.c +2
-2@@ -3413,9 +3413,9 @@ void *kmem_cache_alloc(struct kmem_cache }EXPORT_SYMBOL (kmem_cache_alloc); -void kmem_cache_free_bulk (struct kmem_cache *s,size_t size,void **p) +void kmem_cache_free_bulk (size_t size,void **p) { -__kmem_cache_free_bulk (s, size, p); +__kmem_cache_free_bulk (size, p); }EXPORT_SYMBOL (kmem_cache_free_bulk);Index:
linux/mm/slab.h
===================================================================mm/slab.h +2
-2@@ -166,10 +166,10 @@ ssize_t slabinfo_write(struct file *file/* * Generic implementation of bulk operations * These are useful for situations in which the allocator cannot -* perform optimizations. In that case segments of the objecct listed +* perform optimizations. In that case segments of the object listed * may be allocated or freed using these operations. */ -void __kmem_cache_free_bulk (struct kmem_cache *,size_t ,void **); +void __kmem_cache_free_bulk (size_t ,void **);int __kmem_cache_alloc_bulk (struct kmem_cache *,gfp_t ,size_t ,void **);#ifdef CONFIG_MEMCG_KMEM Index:
linux/mm/slub.c
===================================================================mm/slub.c +1
-1@@ -2887,23 +2887,30 @@ static int build_detached_freelist(struc/* Note that interrupts must be enabled when calling this function. */ -void kmem_cache_free_bulk (struct kmem_cache *orig_s,size_t size,void **p) +void kmem_cache_free_bulk (size_t size,void **p) {if (WARN_ON (!size))
mm/slub.c static int build_detached_freelist(struc · line 2899
Quoted 10 lines
mm/slub.c +2
-3return ; do { struct detached_freelist df; - struct kmem_cache *s; + struct page *page; - /* Support for memcg */ - s = cache_from_obj(orig_s, p[size - 1]); + page = virt_to_head_page(p[size - 1]);
Think we can drop this.
mm/slub.c static int build_detached_freelist(struc · line 2907
Quoted 8 lines
mm/slub.c +7
-1- size =build_detached_freelist (s, size, p, &df); +if (unlikely (!PageSlab (page))) { +BUG_ON (!PageCompound (page)); +kfree_hook (p[size -1 ]); +__free_kmem_pages (page,compound_order (page)); + p[--size] =NULL ; +continue ; + }
and move above into build_detached_freelist() and make it a little more pretty code wise (avoiding the p[size -1] juggling).
mm/slub.c static int build_detached_freelist(struc · line 2909
Quoted 2 lines
mm/slub.c +2
-0+ + size =build_detached_freelist (page->slab_cache, size, p, &df);
also think we should be able to drop the kmem_cache param "page->slab_cache".
Quoted 17 lines
mm/slub.c +2
-2if (unlikely (!df.page))continue ; -slab_free (s, df.page, df.freelist, df.tail, df.cnt, _RET_IP_); +slab_free (page->slab_cache, df.page, df.freelist, df.tail, df.cnt, _RET_IP_); }while (likely (size)); }EXPORT_SYMBOL (kmem_cache_free_bulk);@@ -2963,7 +2970,7 @@ int kmem_cache_alloc_bulk(struct kmem_ca error:local_irq_enable ();slab_post_alloc_hook (s, flags, i, p); -__kmem_cache_free_bulk (s, i, p); +__kmem_cache_free_bulk (i, p);return 0 ; }EXPORT_SYMBOL (kmem_cache_alloc_bulk);
Signature · 11 lines
Best regards, Jesper Dangaard Brouer MSc.CS, Principal Kernel Engineer at Red Hat Author of http://www.iptv-analyzer.org LinkedIn: http://www.linkedin.com/in/brouer -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH V2 8/9] slab: implement bulk free in SLAB allocator
Quoted 3 lines · Jesper Dangaard Brouer
I really like the idea of making it able to free kmalloc'ed objects. But I hate to change the API again... (I do have a use-case in the network stack where I could use this feature).
Now is the time to fix the API since its not that much in use yet if at all.
Quoted 4 lines
I'm traveling (to Sweden) Thursday (afternoon) and will be back late Friday (and have to play Viking in the weekend), thus to be realistic I'll start working on this Monday, so we can get some benchmark numbers to guide this decision.
Ok great.
Quoted 8 lines
mm/slub.c static int build_detached_freelist(struc · line 2899
Quoted 6 lines
mm/slub.c +2
-3-struct kmem_cache *s; +struct page *page; -/* Support for memcg */ - s =cache_from_obj (orig_s, p[size -1 ]); + page =virt_to_head_page (p[size -1 ]);Think we can drop this.
Well then you wont be able to check for a compound page. And you do not want this check in build detached freelist.
Quoted 12 lines
mm/slub.c static int build_detached_freelist(struc · line 2907
Quoted 9 lines
mm/slub.c +7
-1- size =build_detached_freelist (s, size, p, &df); +if (unlikely (!PageSlab (page))) { +BUG_ON (!PageCompound (page)); +kfree_hook (p[size -1 ]); +__free_kmem_pages (page,compound_order (page)); + p[--size] =NULL ; +continue ; + }and move above into
build_detached_freelist()and make it a little more pretty code wise (avoiding the p[size -1] juggling).
If we do this check here then we wont be needing it in build_detached_freelist.
Quoted 4 lines
mm/slub.c static int build_detached_freelist(struc · line 2909
Quoted 2 lines
mm/slub.c +2
-0+ + size =build_detached_freelist (page->slab_cache, size, p, &df);also think we should be able to drop the
kmem_cacheparam "page->slab_cache".
Yep.
Quoted 7 lines
mm/slub.c static int build_detached_freelist(struc · line 2913
Quoted 7 lines
mm/slub.c +1
-1if (unlikely (!df.page))continue ; -slab_free (s, df.page, df.freelist, df.tail, df.cnt, _RET_IP_); +slab_free (page->slab_cache, df.page, df.freelist, df.tail, df.cnt, _RET_IP_);
Then we need df.slab_cache or something.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH V2 8/9] slab: implement bulk free in SLAB allocator
Quoted 8 lines · On Wed, 9 Dec 2015 13:41:07 -0600 (CST) Christoph Lameter <[email protected]> wrote:
Quoted 3 lines · Jesper Dangaard Brouer
I really like the idea of making it able to free kmalloc'ed objects. But I hate to change the API again... (I do have a use-case in the network stack where I could use this feature).
Now is the time to fix the API since its not that much in use yet if at all.
True. I was just so close submitting the network use-case to DaveM. Guess, that will have to wait if we choose this API change (and I'll have to wait another 3 month before the trees are in sync again).
Quoted 6 lines
Quoted 4 lines
I'm traveling (to Sweden) Thursday (afternoon) and will be back late Friday (and have to play Viking in the weekend), thus to be realistic I'll start working on this Monday, so we can get some benchmark numbers to guide this decision.
Ok great.
I'm actually very eager to see if this works out :-)
Quoted 25 lines
Quoted 8 lines
mm/slub.c static int build_detached_freelist(struc · line 2899
Quoted 6 lines
mm/slub.c +2
-3-struct kmem_cache *s; +struct page *page; -/* Support for memcg */ - s =cache_from_obj (orig_s, p[size -1 ]); + page =virt_to_head_page (p[size -1 ]);Think we can drop this.
Well then you wont be able to check for a compound page. And you do not want this check in build detached freelist.
Quoted 12 lines
mm/slub.c static int build_detached_freelist(struc · line 2907
Quoted 9 lines
mm/slub.c +7
-1- size =build_detached_freelist (s, size, p, &df); +if (unlikely (!PageSlab (page))) { +BUG_ON (!PageCompound (page)); +kfree_hook (p[size -1 ]); +__free_kmem_pages (page,compound_order (page)); + p[--size] =NULL ; +continue ; + }and move above into
build_detached_freelist()and make it a little more pretty code wise (avoiding the p[size -1] juggling).
Quoted 2 lines
If we do this check here then we wont be needing it in
build_detached_freelist.
I'll try see what looks best coding style wise...
Quoted 9 lines
Quoted 4 lines
mm/slub.c static int build_detached_freelist(struc · line 2909
Quoted 2 lines
mm/slub.c +2
-0+ + size =build_detached_freelist (page->slab_cache, size, p, &df);also think we should be able to drop the
kmem_cacheparam "page->slab_cache".Yep.
Quoted 2 lines
Quoted 7 lines
Quoted 5 lines
mm/slub.c static int build_detached_freelist(struc · line 2913
Quoted 5 lines
mm/slub.c +1
-1if (unlikely (!df.page))continue ; -slab_free (s, df.page, df.freelist, df.tail, df.cnt, _RET_IP_); +slab_free (page->slab_cache, df.page, df.freelist, df.tail, df.cnt, _RET_IP_);Then we need df.slab_cache or something.
What about df.page->slab_cache (?)
Signature · 11 lines
Best regards, Jesper Dangaard Brouer MSc.CS, Principal Kernel Engineer at Red Hat Author of http://www.iptv-analyzer.org LinkedIn: http://www.linkedin.com/in/brouer -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH V2 8/9] slab: implement bulk free in SLAB allocator
Quoted 3 lines · Jesper Dangaard Brouer
True. I was just so close submitting the network use-case to DaveM. Guess, that will have to wait if we choose this API change (and I'll have to wait another 3 month before the trees are in sync again).
why? Andrew can push that to next pretty soon and DaveM could carry that patch too.
Quoted 3 lines
Quoted 1 line
Then we need df.slab_cache or something.
What about
df.page->slab_cache(?)
Fine come up with a patch.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH V2 8/9] slab: implement bulk free in SLAB allocator
In reply to Christoph LameterQuoted 8 lines · On Wed, 9 Dec 2015 13:41:07 -0600 (CST) Christoph Lameter <[email protected]> wrote:
Quoted 3 lines · Jesper Dangaard Brouer
I really like the idea of making it able to free kmalloc'ed objects. But I hate to change the API again... (I do have a use-case in the network stack where I could use this feature).
Now is the time to fix the API since its not that much in use yet if at all.
Lets start the naming thread/flame (while waiting for my flight ;-))
If we drop the "kmem_cache *s" parameter from kmem_cache_free_bulk(), and also make it handle kmalloc'ed objects. Why should we name it "kmem_cache_free_bulk"? ... what about naming it kfree_bulk() ???
Or should we keep the name to have a symmetric API kmem_cache_{alloc,free}_bulk() call convention?
I'm undecided...
Signature · 11 lines
Best regards, Jesper Dangaard Brouer MSc.CS, Principal Kernel Engineer at Red Hat Author of http://www.iptv-analyzer.org LinkedIn: http://www.linkedin.com/in/brouer -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH V2 8/9] slab: implement bulk free in SLAB allocator
Quoted 3 lines · Jesper Dangaard Brouer
If we drop the "
kmem_cache*s" parameter fromkmem_cache_free_bulk(), and also make it handle kmalloc'ed objects. Why should we name it "kmem_cache_free_bulk"? ... what about naming itkfree_bulk()???
Yes makes sense.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH V2 8/9] slab: implement bulk free in SLAB allocator
Quoted 7 lines · Christoph Lameter
Quoted 3 lines · Jesper Dangaard Brouer
If we drop the "
kmem_cache*s" parameter fromkmem_cache_free_bulk(), and also make it handle kmalloc'ed objects. Why should we name it "kmem_cache_free_bulk"? ... what about naming itkfree_bulk()???Yes makes sense.
IMHO kmem_cache_alloc_bulk/kfree_bulk looks awkward, especially taking into account the fact that we pair kmem_cache_alloc/kmem_cache_free and kmalloc/kfree, but never kmem_cache_alloc/kfree.
So I'd vote for kmem_cache_free_bulk taking a kmem_cache as an argument, but I'm not a potential user of this API, so please don't count my vote :-)
Thanks,
Vladimir
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH V2 8/9] slab: implement bulk free in SLAB allocator
Quoted 7 lines · Vladimir Davydov
IMHO kmem_cache_alloc_bulk/kfree_bulk looks awkward, especially taking into account the fact that we pair kmem_cache_alloc/kmem_cache_free and kmalloc/kfree, but never kmem_cache_alloc/kfree.
So I'd vote for
kmem_cache_free_bulktaking akmem_cacheas an argument, but I'm not a potential user of this API, so please don't count my vote :-)
One way to have it less awkward is to keep naming it kmem_cache_free_bulk but omit the kmem_cache parameter like what I did initially.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH V2 8/9] slab: implement bulk free in SLAB allocator
In reply to Christoph LameterQuoted 7 lines · On Wed, 9 Dec 2015 10:06:39 -0600 (CST) Christoph Lameter <[email protected]> wrote:
From: Christoph Lameter <[email protected]>
Subject: slab bulk api: Remove thekmem_cacheparameter fromkmem_cache_bulk_free()It is desirable and necessary to free objects from different
kmem_caches. It is required in order to support memcg object freeing across different5 cgroups.
Quoted 8 lines
So drop the pointless parameter and allow freeing of arbitrary lists of slab allocated objects.
This patch also does the proper compound page handling so that arbitrary objects allocated via
kmalloc()can be handled bykmem_cache_bulk_free().Signed-off-by Christoph Lameter <[email protected]>
I've modified this patch, to instead introduce a kfree_bulk() and keep the old behavior of kmem_cache_free_bulk(). This allow us to easier compare the two impl. approaches.
[...]
mm/slub.c static int build_detached_freelist(struc · line 2890
Quoted 10 lines
Index:
linux/mm/slub.c
===================================================================mm/slub.c +1
-1@@ -2887,23 +2887,30 @@ static int build_detached_freelist(struc/* Note that interrupts must be enabled when calling this function. */ -void kmem_cache_free_bulk (struct kmem_cache *orig_s,size_t size,void **p) +void kmem_cache_free_bulk (size_t size,void **p)
Renamed to kfree_bulk(size_t size, void **p)
Quoted 22 lines
mm/slub.c +10
-4{if (WARN_ON (!size))return ; do { struct detached_freelist df; - struct kmem_cache *s; + struct page *page; - /* Support for memcg */ - s = cache_from_obj(orig_s, p[size - 1]); + page = virt_to_head_page(p[size - 1]); - size = build_detached_freelist(s, size, p, &df); + if (unlikely(!PageSlab(page))) { + BUG_ON(!PageCompound(page)); + kfree_hook(p[size - 1]); + __free_kmem_pages(page, compound_order(page)); + p[--size] = NULL; + continue; + } +
mm/slub.c static int build_detached_freelist(struc · line 2916
Quoted 10 lines
mm/slub.c +3
-1+ size =build_detached_freelist (page->slab_cache, size, p, &df);if (unlikely (!df.page))continue ; -slab_free (s, df.page, df.freelist, df.tail, df.cnt, _RET_IP_); +slab_free (page->slab_cache, df.page, df.freelist, + df.tail, df.cnt, _RET_IP_); }while (likely (size)); }EXPORT_SYMBOL (kmem_cache_free_bulk);
This specific implementation was too slow, mostly because we call virt_to_head_page() both in this function and inside build_detached_freelist().
After integrating this directly into build_detached_freelist() I'm getting more comparative results.
Results with disabled CONFIG_MEMCG_KMEM, and SLUB (and slab_nomerge for more accurate results between runs):
bulk- fallback - kmem_cache_free_bulk - kfree_bulk
1 - 58 cycles 14.735 ns - 55 cycles 13.880 ns - 59 cycles 14.843 ns 2 - 53 cycles 13.298 ns - 32 cycles 8.037 ns - 34 cycles 8.592 ns 3 - 51 cycles 12.837 ns - 25 cycles 6.442 ns - 27 cycles 6.794 ns 4 - 50 cycles 12.514 ns - 23 cycles 5.952 ns - 23 cycles 5.958 ns 8 - 48 cycles 12.097 ns - 20 cycles 5.160 ns - 22 cycles 5.505 ns 16 - 47 cycles 11.888 ns - 19 cycles 4.900 ns - 19 cycles 4.969 ns 30 - 47 cycles 11.793 ns - 18 cycles 4.688 ns - 18 cycles 4.682 ns 32 - 47 cycles 11.926 ns - 18 cycles 4.674 ns - 18 cycles 4.702 ns 34 - 95 cycles 23.823 ns - 24 cycles 6.068 ns - 24 cycles 6.058 ns 48 - 81 cycles 20.258 ns - 21 cycles 5.360 ns - 21 cycles 5.338 ns 64 - 73 cycles 18.414 ns - 20 cycles 5.160 ns - 20 cycles 5.140 ns 128 - 90 cycles 22.563 ns - 27 cycles 6.765 ns - 27 cycles 6.801 ns 158 - 99 cycles 24.831 ns - 30 cycles 7.625 ns - 30 cycles 7.720 ns
250 - 104 cycles 26.173 ns - 37 cycles 9.271 ns - 37 cycles 9.371 ns
As can been seen the old kmem_cache_free_bulk() is faster than the new kfree_bulk() (which omits the kmem_cache pointer and need to derive it from the page->slab_cache). The base (bulk=1) extra cost is 4 cycles, which then gets amortized as build_detached_freelist() combines objects belonging to same page.
This is likely because the compiler, with disabled CONFIG_MEMCG_KMEM=n, can optimize and avoid doing the lookup of the kmem_cache structure.
I'll start doing testing with CONFIG_MEMCG_KMEM enabled...
Signature · 11 lines
Best regards, Jesper Dangaard Brouer MSc.CS, Principal Kernel Engineer at Red Hat Author of http://www.iptv-analyzer.org LinkedIn: http://www.linkedin.com/in/brouer -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [RFC PATCH V2 8/9] slab: implement bulk free in SLAB allocator
Quoted 100 lines · On Mon, 14 Dec 2015 16:19:58 +0100 Jesper Dangaard Brouer <[email protected]>…
Quoted 7 lines · On Wed, 9 Dec 2015 10:06:39 -0600 (CST) Christoph Lameter <[email protected]> wrote:
From: Christoph Lameter <[email protected]>
Subject: slab bulk api: Remove thekmem_cacheparameter fromkmem_cache_bulk_free()It is desirable and necessary to free objects from different
kmem_caches. It is required in order to support memcg object freeing across different5 cgroups.Quoted 8 lines
So drop the pointless parameter and allow freeing of arbitrary lists of slab allocated objects.
This patch also does the proper compound page handling so that arbitrary objects allocated via
kmalloc()can be handled bykmem_cache_bulk_free().Signed-off-by Christoph Lameter <[email protected]>I've modified this patch, to instead introduce a
kfree_bulk()and keep the old behavior ofkmem_cache_free_bulk(). This allow us to easier compare the two impl. approaches.[...]
mm/slub.c static int build_detached_freelist(struc · line 2890
Quoted 10 lines
Index:
linux/mm/slub.c
===================================================================mm/slub.c +1
-1@@ -2887,23 +2887,30 @@ static int build_detached_freelist(struc/* Note that interrupts must be enabled when calling this function. */ -void kmem_cache_free_bulk (struct kmem_cache *orig_s,size_t size,void **p) +void kmem_cache_free_bulk (size_t size,void **p)Renamed to
kfree_bulk(size_t size, void **p)Quoted 22 lines
mm/slub.c +10
-4{if (WARN_ON (!size))return ; do { struct detached_freelist df; - struct kmem_cache *s; + struct page *page; - /* Support for memcg */ - s = cache_from_obj(orig_s, p[size - 1]); + page = virt_to_head_page(p[size - 1]); - size = build_detached_freelist(s, size, p, &df); + if (unlikely(!PageSlab(page))) { + BUG_ON(!PageCompound(page)); + kfree_hook(p[size - 1]); + __free_kmem_pages(page, compound_order(page)); + p[--size] = NULL; + continue; + } +mm/slub.c static int build_detached_freelist(struc
Quoted 10 lines
mm/slub.c +3
-1+ size =build_detached_freelist (page->slab_cache, size, p, &df);if (unlikely (!df.page))continue ; -slab_free (s, df.page, df.freelist, df.tail, df.cnt, _RET_IP_); +slab_free (page->slab_cache, df.page, df.freelist, + df.tail, df.cnt, _RET_IP_); }while (likely (size)); }EXPORT_SYMBOL (kmem_cache_free_bulk);This specific implementation was too slow, mostly because we call
virt_to_head_page()both in this function and insidebuild_detached_freelist().After integrating this directly into
build_detached_freelist()I'm getting more comparative results.Results with disabled
CONFIG_MEMCG_KMEM, and SLUB (andslab_nomergefor more accurate results between runs):bulk- fallback -
kmem_cache_free_bulk-kfree_bulk1 - 58 cycles 14.735 ns - 55 cycles 13.880 ns - 59 cycles 14.843 ns 2 - 53 cycles 13.298 ns - 32 cycles 8.037 ns - 34 cycles 8.592 ns 3 - 51 cycles 12.837 ns - 25 cycles 6.442 ns - 27 cycles 6.794 ns 4 - 50 cycles 12.514 ns - 23 cycles 5.952 ns - 23 cycles 5.958 ns 8 - 48 cycles 12.097 ns - 20 cycles 5.160 ns - 22 cycles 5.505 ns 16 - 47 cycles 11.888 ns - 19 cycles 4.900 ns - 19 cycles 4.969 ns 30 - 47 cycles 11.793 ns - 18 cycles 4.688 ns - 18 cycles 4.682 ns 32 - 47 cycles 11.926 ns - 18 cycles 4.674 ns - 18 cycles 4.702 ns 34 - 95 cycles 23.823 ns - 24 cycles 6.068 ns - 24 cycles 6.058 ns 48 - 81 cycles 20.258 ns - 21 cycles 5.360 ns - 21 cycles 5.338 ns 64 - 73 cycles 18.414 ns - 20 cycles 5.160 ns - 20 cycles 5.140 ns 128 - 90 cycles 22.563 ns - 27 cycles 6.765 ns - 27 cycles 6.801 ns 158 - 99 cycles 24.831 ns - 30 cycles 7.625 ns - 30 cycles 7.720 ns250 - 104 cycles 26.173 ns - 37 cycles 9.271 ns - 37 cycles 9.371 ns
As can been seen the old
kmem_cache_free_bulk()is faster than the newkfree_bulk()(which omits thekmem_cachepointer and need to derive it from thepage->slab_cache). The base (bulk=1) extra cost is 4 cycles, which then gets amortized asbuild_detached_freelist()combines objects belonging to same page.
Quoted 4 lines
This is likely because the compiler, with disabled
CONFIG_MEMCG_KMEM=n, can optimize and avoid doing the lookup of thekmem_cachestructure.I'll start doing testing with
CONFIG_MEMCG_KMEMenabled...
Enabling CONFIG_MEMCG_KMEM didn't change the performance, likely because memcg_kmem_enabled() uses a static_key_false() construct.
Implemented kfree_bulk() as an inline function just calling kmem_cache_free_bulk(NULL, size, p) run with CONFIG_MEMCG_KMEM=y (but no "user" of memcg):
bulk- fallback - kmem_cache_free_bulk - kmem_cache_free_bulk(s=NULL)
1 - 60 cycles 15.232 ns - 52 cycles 13.124 ns - 58 cycles 14.562 ns 2 - 54 cycles 13.678 ns - 31 cycles 7.875 ns - 34 cycles 8.509 ns 3 - 52 cycles 13.112 ns - 25 cycles 6.406 ns - 28 cycles 7.045 ns 4 - 51 cycles 12.833 ns - 24 cycles 6.036 ns - 24 cycles 6.076 ns 8 - 49 cycles 12.367 ns - 21 cycles 5.404 ns - 22 cycles 5.522 ns 16 - 48 cycles 12.144 ns - 19 cycles 4.934 ns - 19 cycles 4.967 ns 30 - 50 cycles 12.542 ns - 18 cycles 4.685 ns - 18 cycles 4.687 ns 32 - 48 cycles 12.234 ns - 18 cycles 4.661 ns - 18 cycles 4.662 ns 34 - 96 cycles 24.204 ns - 24 cycles 6.068 ns - 24 cycles 6.067 ns 48 - 82 cycles 20.553 ns - 21 cycles 5.410 ns - 21 cycles 5.433 ns 64 - 74 cycles 18.743 ns - 20 cycles 5.171 ns - 20 cycles 5.179 ns 128 - 91 cycles 22.791 ns - 26 cycles 6.601 ns - 26 cycles 6.600 ns 158 - 100 cycles 25.191 ns - 30 cycles 7.569 ns - 30 cycles 7.617 ns 250 - 106 cycles 26.535 ns - 37 cycles 9.426 ns - 37 cycles 9.427 ns
As can be seen, it is still more costly to get the kmem_cache pointer via the object->slab_cache, even when compiled with CONFIG_MEMCG_KMEM.
But what happen when enabling a "user" of kmemcg, which modify the static_key_false() in function call memcg_kmem_enabled().
First notice normal kmem fastpath cost increased: 70 cycles(tsc) 17.640 ns
bulk- fallback - kmem_cache_free_bulk - kmem_cache_free_bulk(s=NULL)
1 - 85 cycles 21.351 ns - 76 cycles 19.086 ns - 74 cycles 18.701 ns 2 - 79 cycles 19.907 ns - 43 cycles 10.848 ns - 41 cycles 10.467 ns 3 - 77 cycles 19.310 ns - 32 cycles 8.132 ns - 31 cycles 7.880 ns 4 - 76 cycles 19.017 ns - 28 cycles 7.195 ns - 28 cycles 7.050 ns 8 - 77 cycles 19.443 ns - 23 cycles 5.838 ns - 23 cycles 5.782 ns 16 - 75 cycles 18.815 ns - 20 cycles 5.174 ns - 20 cycles 5.127 ns 30 - 74 cycles 18.719 ns - 19 cycles 4.870 ns - 19 cycles 4.816 ns 32 - 75 cycles 18.917 ns - 19 cycles 4.850 ns - 19 cycles 4.800 ns 34 - 119 cycles 29.878 ns - 25 cycles 6.312 ns - 25 cycles 6.282 ns 48 - 106 cycles 26.536 ns - 21 cycles 5.471 ns - 21 cycles 5.448 ns 64 - 98 cycles 24.635 ns - 20 cycles 5.239 ns - 20 cycles 5.224 ns 128 - 114 cycles 28.586 ns - 27 cycles 6.768 ns - 26 cycles 6.736 ns 158 - 124 cycles 31.173 ns - 30 cycles 7.668 ns - 30 cycles 7.611 ns 250 - 129 cycles 32.336 ns - 37 cycles 9.460 ns - 37 cycles 9.468 ns
With kmemcg runtime-enabled, I was expecting to see a bigger win for the case of kmem_cache_free_bulk(s=NULL).
Implemented a function kfree_bulk_export(), which inlines build_detached_freelist() and which avoids including the code path for memcg. But result is almost the same:
bulk- fallback - kmem_cache_free_bulk - kfree_bulk_export
1 - 84 cycles 21.221 ns - 77 cycles 19.324 ns - 74 cycles 18.515 ns 2 - 79 cycles 19.963 ns - 44 cycles 11.120 ns - 41 cycles 10.329 ns 3 - 77 cycles 19.471 ns - 33 cycles 8.291 ns - 31 cycles 7.771 ns 4 - 76 cycles 19.191 ns - 28 cycles 7.100 ns - 27 cycles 6.765 ns 8 - 78 cycles 19.525 ns - 23 cycles 5.781 ns - 22 cycles 5.737 ns 16 - 75 cycles 18.922 ns - 20 cycles 5.152 ns - 20 cycles 5.092 ns 30 - 75 cycles 18.812 ns - 19 cycles 4.864 ns - 19 cycles 4.833 ns 32 - 75 cycles 18.807 ns - 19 cycles 4.852 ns - 23 cycles 5.896 ns 34 - 119 cycles 29.989 ns - 25 cycles 6.258 ns - 25 cycles 6.411 ns 48 - 106 cycles 26.677 ns - 28 cycles 7.182 ns - 22 cycles 5.651 ns 64 - 98 cycles 24.716 ns - 20 cycles 5.245 ns - 21 cycles 5.425 ns 128 - 115 cycles 28.905 ns - 26 cycles 6.748 ns - 27 cycles 6.787 ns 158 - 124 cycles 31.039 ns - 30 cycles 7.604 ns - 30 cycles 7.629 ns 250 - 129 cycles 32.372 ns - 37 cycles 9.475 ns - 37 cycles 9.325 ns
I took a closer look with perf, and it looks like the extra cost from enabling kmemcg originates from the alloc code path. Specifically the function call get_mem_cgroup_from_mm(9.95%) is the costly part, plus __memcg_kmem_get_cache(4.84%) and __memcg_kmem_put_cache(1.35%)
(Added current code as a patch below signature)
- -
Best regards,
Jesper Dangaard Brouer
MSc.CS, Principal Kernel Engineer at Red Hat
Author of http://www.iptv-analyzer.org
LinkedIn: http://www.linkedin.com/in/brouer
[PATCH] New API kfree_bulk()
--- include/linux/slab.h | 14 +++++++++++++ mm/slab_common.c | 8 ++++++-- mm/slub.c | 52 +++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 67 insertions(+), 7 deletions(-)
include/linux/slab.h +14 -0
@@ -317,6 +317,20 @@ void kmem_cache_free(struct kmem_cache *, void *);*/ void kmem_cache_free_bulk (struct kmem_cache *,size_t ,void **);int kmem_cache_alloc_bulk (struct kmem_cache *,gfp_t ,size_t ,void **); +void kfree_bulk_export (size_t ,void **); + +static __always_inline void kfree_bulk(size_t size, void **p) +{ +/* Reusing call to kmem_cache_free_bulk() allow kfree_bulk to +* use same code icache +*/ +kmem_cache_free_bulk (NULL , size, p); +/* +* Inlining in kfree_bulk_export() generate smaller code size +* than more generic kmem_cache_free_bulk(). +*/ +// kfree_bulk_export(size, p); +}#ifdef CONFIG_NUMA void *__kmalloc_node (size_t size,gfp_t flags,int node) __assume_kmalloc_alignment;
mm/slab_common.c +6 -2
@@ -108,8 +108,12 @@ void __kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p) {size_t i; -for (i =0 ; i < nr; i++) -kmem_cache_free (s, p[i]); +for (i =0 ; i < nr; i++) { +if (s) +kmem_cache_free (s, p[i]); +else +kfree (p[i]); + } } int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr,
mm/slub.c +47 -5
82 lines not on this page · raw message
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
RFC PATCH v2 9/9 slab: annotate code to generate more compact asm code
In reply to Jesper Dangaard BrouerPremature optimizations for CONFIG_NUMA case...
--- mm/slab.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
mm/slab.c +2 -2
@@ -3185,7 +3185,7 @@ __do_cache_alloc(struct kmem_cache *cache, gfp_t flags) {void *objp; -if (current->mempolicy ||cpuset_do_slab_mem_spread ()) { +if (unlikely (current->mempolicy ||cpuset_do_slab_mem_spread ())) { objp =alternate_node_alloc (cache, flags);if (objp) goto out;
@@ -3196,7 +3196,7 @@ __do_cache_alloc(struct kmem_cache *cache, gfp_t flags)* We may just have run out of memory on the local node. * ____cache_alloc_node() knows how to locate memory on other nodes */ -if (!objp) +if (unlikely (!objp)) objp =____cache_alloc_node (cache, flags,numa_mem_id ()); out:
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>