memcg: simple cleanup of stats update functions
Subsystems (3)
Shakeel Butt
maintainerPATCH v2 1/1 memcg: simple cleanup of stats update functions
mod_memcg_lruvec_state() is never called from outside of memcontrol.c and with always irq disabled. So, replace it with the irq disabled version and add an assert that irq is disabled in the caller.
Similarly mod_objcg_state() is not called from outside of memcontrol.c, so simply make it static and change it's name to __mod_objcg_state().
---
Change since v1:
- Change mod_objcg_state to __mod_objcg_state as per the naming convention (suggested by Johannes).
include/linux/memcontrol.h | 17 ----------------- mm/memcontrol.c | 31 +++++++++++++++---------------- mm/slab.h | 2 -- 3 files changed, 15 insertions(+), 35 deletions(-)
include/linux/memcontrol.h +0 -17
@@ -1077,8 +1077,6 @@ static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,void mem_cgroup_flush_stats (struct mem_cgroup *memcg);void mem_cgroup_flush_stats_ratelimited (struct mem_cgroup *memcg); -void __mod_memcg_lruvec_state (struct lruvec *lruvec,enum node_stat_item idx, -int val);void __mod_lruvec_kmem_state (void *p,enum node_stat_item idx,int val); static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx,
@@ -1091,16 +1089,6 @@ static inline void mod_lruvec_kmem_state(void *p, enum node_stat_item idx,local_irq_restore (flags); } -static inline void mod_memcg_lruvec_state (struct lruvec *lruvec, -enum node_stat_item idx,int val) -{ -unsigned long flags; - -local_irq_save (flags); -__mod_memcg_lruvec_state (lruvec, idx, val); -local_irq_restore (flags); -} -void __count_memcg_events (struct mem_cgroup *memcg,enum vm_event_item idx,unsigned long count);
@@ -1594,11 +1582,6 @@ static inline void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg) { } -static inline void __mod_memcg_lruvec_state (struct lruvec *lruvec, -enum node_stat_item idx,int val) -{ -} -static inline void __mod_lruvec_kmem_state (void *p,enum node_stat_item idx,int val) {
mm/memcontrol.c +15 -16
@@ -837,8 +837,9 @@ static unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx)return x; } -void __mod_memcg_lruvec_state (struct lruvec *lruvec,enum node_stat_item idx, -int val) +static void __mod_memcg_lruvec_state (struct lruvec *lruvec, +enum node_stat_item idx, +int val) {struct mem_cgroup_per_node *pn;struct mem_cgroup *memcg;
@@ -2983,21 +2984,19 @@ void mem_cgroup_commit_charge(struct folio *folio, struct mem_cgroup *memcg)#ifdef CONFIG_MEMCG_KMEM -/* -* mod_objcg_mlstate() may be called with irq enabled, so -* mod_memcg_lruvec_state() should be used. -*/ -static inline void mod_objcg_mlstate (struct obj_cgroup *objcg, -struct pglist_data *pgdat, -enum node_stat_item idx,int nr) +static inline void __mod_objcg_mlstate (struct obj_cgroup *objcg, +struct pglist_data *pgdat, +enum node_stat_item idx,int nr) {struct mem_cgroup *memcg;struct lruvec *lruvec; +lockdep_assert_irqs_disabled (); +rcu_read_lock (); memcg =obj_cgroup_memcg (objcg); lruvec =mem_cgroup_lruvec (memcg, pgdat); -mod_memcg_lruvec_state (lruvec, idx, nr); +__mod_memcg_lruvec_state (lruvec, idx, nr);rcu_read_unlock (); }
@@ -3317,7 +3316,7 @@ void __memcg_kmem_uncharge_page(struct page *page, int order)obj_cgroup_put (objcg); } -void mod_objcg_state (struct obj_cgroup *objcg,struct pglist_data *pgdat, +static void mod_objcg_state (struct obj_cgroup *objcg,struct pglist_data *pgdat,enum node_stat_item idx,int nr) {struct memcg_stock_pcp *stock;
@@ -3345,12 +3344,12 @@ void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat,struct pglist_data *oldpg = stock->cached_pgdat;if (stock->nr_slab_reclaimable_b) { -mod_objcg_mlstate (objcg, oldpg,NR_SLAB_RECLAIMABLE_B , +__mod_objcg_mlstate (objcg, oldpg,NR_SLAB_RECLAIMABLE_B , stock->nr_slab_reclaimable_b); stock->nr_slab_reclaimable_b =0 ; }if (stock->nr_slab_unreclaimable_b) { -mod_objcg_mlstate (objcg, oldpg,NR_SLAB_UNRECLAIMABLE_B , +__mod_objcg_mlstate (objcg, oldpg,NR_SLAB_UNRECLAIMABLE_B , stock->nr_slab_unreclaimable_b); stock->nr_slab_unreclaimable_b =0 ; }
@@ -3376,7 +3375,7 @@ void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, } }if (nr) -mod_objcg_mlstate (objcg, pgdat, idx, nr); +__mod_objcg_mlstate (objcg, pgdat, idx, nr);local_unlock_irqrestore (&memcg_stock.stock_lock, flags);obj_cgroup_put (old);
@@ -3442,13 +3441,13 @@ static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock)*/ if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) {if (stock->nr_slab_reclaimable_b) { -mod_objcg_mlstate (old, stock->cached_pgdat, +__mod_objcg_mlstate (old, stock->cached_pgdat,NR_SLAB_RECLAIMABLE_B , stock->nr_slab_reclaimable_b); stock->nr_slab_reclaimable_b =0 ; }if (stock->nr_slab_unreclaimable_b) { -mod_objcg_mlstate (old, stock->cached_pgdat, +__mod_objcg_mlstate (old, stock->cached_pgdat,NR_SLAB_UNRECLAIMABLE_B , stock->nr_slab_unreclaimable_b); stock->nr_slab_unreclaimable_b =0 ;
mm/slab.h +0 -2
@@ -578,8 +578,6 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, gfp_t flags, size_t size, void **p);void __memcg_slab_free_hook (struct kmem_cache *s,struct slab *slab,void **p,int objects,struct slabobj_ext *obj_exts); -void mod_objcg_state (struct obj_cgroup *objcg,struct pglist_data *pgdat, -enum node_stat_item idx,int nr);#endif size_t __ksize (const void *objp);
Signature · 1 line
2.43.0
Quoted 10 lines · Shakeel Butt
mod_memcg_lruvec_state()is never called from outside ofmemcontrol.cand with always irq disabled. So, replace it with the irq disabled version and add an assert that irq is disabled in the caller.Similarly
mod_objcg_state()is not called from outside ofmemcontrol.c, so simply make it static and change it's name to__mod_objcg_state().Signed-off-by Shakeel Butt <[email protected]>Acked-by Johannes Weiner <[email protected]>
Quoted 3 lines · Shakeel Butt
mod_memcg_lruvec_state()is never called from outside ofmemcontrol.cand with always irq disabled. So, replace it with the irq disabled version and add an assert that irq is disabled in the caller.
unless PREEMPT_RT is enabled. In that case IRQs are not disabled as part of local_lock_irqsave(&memcg_stock.stock_lock, …) leading to:
| ------------[ cut here ]------------ | WARNING: CPU: 0 PID: 1 at mm/memcontrol.c:3150 __mod_objcg_mlstate+0xc2/0x110 | CPU: 0 PID: 1 Comm: systemd Not tainted 6.10.0-rc1-rt0+ #17 | Call Trace: | <TASK> | mod_objcg_state+0x2b3/0x320 | __memcg_slab_post_alloc_hook+0x13c/0x340 | kmem_cache_alloc_lru_noprof+0x2bd/0x2e0 | alloc_inode+0x59/0xc0 | iget_locked+0xf0/0x290
suggestions?
Sebastian
Vlastimil Babka (SUSE)
maintainerQuoted 7 lines · Sebastian Andrzej Siewior
Quoted 3 lines · Shakeel Butt
mod_memcg_lruvec_state()is never called from outside ofmemcontrol.cand with always irq disabled. So, replace it with the irq disabled version and add an assert that irq is disabled in the caller.unless
PREEMPT_RTis enabled. In that case IRQs are not disabled as part oflocal_lock_irqsave(&memcg_stock.stock_lock, …)leading to:
But then the "interrupts are handled by a kernel thread that can sleep" part of RT also means it's ok to just have the stock_lock taken with no interrupts disabled as no actual raw interrupt handler will interrupt the holder and deadlock, right?
Quoted 11 lines
| ------------[ cut here ]------------ | WARNING: CPU: 0 PID: 1 at mm/memcontrol.c:3150 __mod_objcg_mlstate+0xc2/0x110 | CPU: 0 PID: 1 Comm: systemd Not tainted 6.10.0-rc1-rt0+ #17 | Call Trace: | <TASK> | mod_objcg_state+0x2b3/0x320 | __memcg_slab_post_alloc_hook+0x13c/0x340 | kmem_cache_alloc_lru_noprof+0x2bd/0x2e0 | alloc_inode+0x59/0xc0 | iget_locked+0xf0/0x290
Quoted 1 line
suggestions?
So in that case the appropriate thing would be to replace the assert with lockdep_assert_held(&memcg_stock.stock_lock);
?
It seems all the code paths leading here have that one.
Quoted 2 lines
Sebastian
Shakeel Butt
maintainerQuoted 8 lines · Vlastimil Babka (SUSE)
Quoted 7 lines · Sebastian Andrzej Siewior
Quoted 3 lines · Shakeel Butt
mod_memcg_lruvec_state()is never called from outside ofmemcontrol.cand with always irq disabled. So, replace it with the irq disabled version and add an assert that irq is disabled in the caller.unless
PREEMPT_RTis enabled. In that case IRQs are not disabled as part oflocal_lock_irqsave(&memcg_stock.stock_lock, …)leading to:
Sorry about that and thanks for the report.
Quoted 6 lines
But then the "interrupts are handled by a kernel thread that can sleep" part of RT also means it's ok to just have the
stock_locktaken with no interrupts disabled as no actual raw interrupt handler will interrupt the holder and deadlock, right?
Thanks Vlastimil for jolting my memory on RT reasoning.
Quoted 11 lines
Quoted 11 lines
| ------------[ cut here ]------------ | WARNING: CPU: 0 PID: 1 at mm/memcontrol.c:3150 __mod_objcg_mlstate+0xc2/0x110 | CPU: 0 PID: 1 Comm: systemd Not tainted 6.10.0-rc1-rt0+ #17 | Call Trace: | <TASK> | mod_objcg_state+0x2b3/0x320 | __memcg_slab_post_alloc_hook+0x13c/0x340 | kmem_cache_alloc_lru_noprof+0x2bd/0x2e0 | alloc_inode+0x59/0xc0 | iget_locked+0xf0/0x290
Quoted 8 lines
Quoted 1 line
suggestions?
So in that case the appropriate thing would be to replace the assert with
lockdep_assert_held(&memcg_stock.stock_lock);
?It seems all the code paths leading here have that one.
Yeah this seems right and reasonable. Should I send a fix or you want to send it?
Quoted 11 lines · Shakeel Butt
Quoted 8 lines · Vlastimil Babka (SUSE)
Quoted 7 lines · Sebastian Andrzej Siewior
Quoted 3 lines · Shakeel Butt
mod_memcg_lruvec_state()is never called from outside ofmemcontrol.cand with always irq disabled. So, replace it with the irq disabled version and add an assert that irq is disabled in the caller.unless
PREEMPT_RTis enabled. In that case IRQs are not disabled as part oflocal_lock_irqsave(&memcg_stock.stock_lock, …)leading to:Sorry about that and thanks for the report.
no worries.
Quoted 5 lines
Quoted 5 lines
But then the "interrupts are handled by a kernel thread that can sleep" part of RT also means it's ok to just have the
stock_locktaken with no interrupts disabled as no actual raw interrupt handler will interrupt the holder and deadlock, right?
I *don't* know why the interrupts-disabled check is here. The memcg_stock.stock_lock is acquired on RT with interrupts enabled and never disables interrupts. The lock is never acquired in an hard interrupt (not threaded interrupt) context so there is never a deadlock.
Originally the interrupts were disabled in mod_memcg_lruvec_state() because the counter, it operates on, is per-CPU and relies on disabled interrupts because the operation is not atomic and the code can be run in interrupts context (on !RT). The __mod_memcg_lruvec_state() variant of it relied on interrupts being disabled by the caller. This "rely on" was part of a spinlock_t lock (or invoked from an interrupt handler, the memory is fading slowly away) which does not disable interrupts on PREEMPT_RT.
So for that reason we ended up with __memcg_stats_lock() which disables preemption only on PREEMPT_RT to achieve the same level of "atomic" update.
Quoted 8 lines
Thanks Vlastimil for jolting my memory on RT reasoning.
Quoted 6 lines
Quoted 1 line
suggestions?
So in that case the appropriate thing would be to replace the assert with
lockdep_assert_held(&memcg_stock.stock_lock);
?
Quoted 5 lines
Quoted 2 lines
It seems all the code paths leading here have that one.
Yeah this seems right and reasonable. Should I send a fix or you want to send it?
I don't mind sending a patch. I'm just not sure if the lock is the right thing to do. However it should ensure that interrupts are disabled on !RT for the sake of the counter update (if observed in IRQ context).
Yeah, let me prepare something.
Sebastian
Vlastimil Babka (SUSE)
maintainerQuoted 49 lines · Sebastian Andrzej Siewior
Quoted 11 lines · Shakeel Butt
Quoted 8 lines · Vlastimil Babka (SUSE)
Quoted 7 lines · Sebastian Andrzej Siewior
Quoted 3 lines · Shakeel Butt
mod_memcg_lruvec_state()is never called from outside ofmemcontrol.cand with always irq disabled. So, replace it with the irq disabled version and add an assert that irq is disabled in the caller.unless
PREEMPT_RTis enabled. In that case IRQs are not disabled as part oflocal_lock_irqsave(&memcg_stock.stock_lock, …)leading to:Sorry about that and thanks for the report.
no worries.
Quoted 5 lines
Quoted 5 lines
But then the "interrupts are handled by a kernel thread that can sleep" part of RT also means it's ok to just have the
stock_locktaken with no interrupts disabled as no actual raw interrupt handler will interrupt the holder and deadlock, right?I *don't* know why the interrupts-disabled check is here. The memcg_stock.stock_lock is acquired on RT with interrupts enabled and never disables interrupts. The lock is never acquired in an hard interrupt (not threaded interrupt) context so there is never a deadlock.
Originally the interrupts were disabled in
mod_memcg_lruvec_state()because the counter, it operates on, is per-CPU and relies on disabled interrupts because the operation is not atomic and the code can be run in interrupts context (on !RT). The__mod_memcg_lruvec_state()variant of it relied on interrupts being disabled by the caller. This "rely on" was part of aspinlock_tlock (or invoked from an interrupt handler, the memory is fading slowly away) which does not disable interrupts onPREEMPT_RT.
So for that reason we ended up with__memcg_stats_lock()which disables preemption only onPREEMPT_RTto achieve the same level of "atomic" update.Quoted 11 lines
Thanks Vlastimil for jolting my memory on RT reasoning.
Quoted 8 lines
Quoted 1 line
suggestions?
So in that case the appropriate thing would be to replace the assert with
lockdep_assert_held(&memcg_stock.stock_lock);
?It seems all the code paths leading here have that one.
Quoted 6 lines
Quoted 2 lines
Yeah this seems right and reasonable. Should I send a fix or you want to send it?
I don't mind sending a patch. I'm just not sure if the lock is the right thing to do. However it should ensure that interrupts are disabled on !RT for the sake of the counter update (if observed in IRQ context).
Looks like some places there use VM_WARN_ON_IRQS_ENABLED() that's turned off for PREEMPT_RT, so maybe that's what should replace the current lockdep_assert, perhaps together with lockdep_assert_held(this_cpu_ptr(&memcg_stock.stock_lock));
But also __mod_memcg_lruvec_state() already has that VM_WARN_ON.
Quoted 3 lines
Yeah, let me prepare something.
Sebastian
Quoted 10 lines · Vlastimil Babka (SUSE)
Quoted 3 lines
I don't mind sending a patch. I'm just not sure if the lock is the right thing to do. However it should ensure that interrupts are disabled on !RT for the sake of the counter update (if observed in IRQ context).
Looks like some places there use
VM_WARN_ON_IRQS_ENABLED()that's turned off forPREEMPT_RT, so maybe that's what should replace the currentlockdep_assert, perhaps together with lockdep_assert_held(this_cpu_ptr(&memcg_stock.stock_lock));But also
__mod_memcg_lruvec_state()already has thatVM_WARN_ON.
This "VM_WARN_ON_IRQS_ENABLED" is the initial assert for "interrupts must be disabled while change the counter".
You want to replace it with lockdep? Part of its requirement was that it yells with lockdep disabled.
Currently I am leaning towards removing the lockdep_assert_irqs_disabled() from __mod_objcg_mlstate(). Nothing but the counter need it and they have their own check. So?
Sebastian
Vlastimil Babka (SUSE)
maintainerQuoted 10 lines · Sebastian Andrzej Siewior
Quoted 9 lines · Vlastimil Babka (SUSE)
Quoted 3 lines
I don't mind sending a patch. I'm just not sure if the lock is the right thing to do. However it should ensure that interrupts are disabled on !RT for the sake of the counter update (if observed in IRQ context).
Looks like some places there use
VM_WARN_ON_IRQS_ENABLED()that's turned off forPREEMPT_RT, so maybe that's what should replace the currentlockdep_assert, perhaps together with lockdep_assert_held(this_cpu_ptr(&memcg_stock.stock_lock));
Quoted 6 lines
Quoted 1 line
But also
__mod_memcg_lruvec_state()already has thatVM_WARN_ON.This "
VM_WARN_ON_IRQS_ENABLED" is the initial assert for "interrupts must be disabled while change the counter".
You want to replace it with lockdep? Part of its requirement was that it yells with lockdep disabled.
No I meant the other way around, that we'de use VM_WARN_ON_IRQS_ENABLED instead of the current lockdep_assert() that doesn't work on RT.
Quoted 3 lines
Currently I am leaning towards removing the
lockdep_assert_irqs_disabled()from__mod_objcg_mlstate(). Nothing but the counter need it and they have their own check. So?
Yeah that might be sufficient.
Quoted 1 line
Sebastian