cgroup: improve cgroup_file_notify() scalability
Subsystems (4)
v2 cover cgroup: improve cgroup_file_notify() scalability
Jakub Kicinski reported that after the commit d929525c2e30 ("memcg:
net: track network throttling due to memcg memory pressure"), in Meta fleet, the network intensive workloads started seeing extensive spinlock contention in the rx path in the function cgroup_file_notify().
cgroup_file_notify() uses a single global spinlock to protect the cfile->kn pointer across all cgroups. On systems with many cgroups under memory pressure, this becomes a bottleneck as multiple CPUs in reclaim call __memcg_memory_event(), which walks the cgroup hierarchy calling cgroup_file_notify() at each level -- all serialized on the same lock.
This series reduces the lock hold time by moving kernfs_notify() outside the critical section, adds lockless fast-path checks to avoid the lock entirely in common cases (torn-down files and rate-limited bursts), and replaces the global lock with a per-cgroup_file lock to eliminate cross-cgroup contention.
Changes since v1:
- Second patch moves rate-limiting code out of lock
Shakeel Butt (3):
cgroup: reduce cgroup_file_kn_lock hold time in cgroup_file_notify()
cgroup: add lockless fast-path checks to cgroup_file_notify()
cgroup: replace global cgroup_file_kn_lock with per-cgroup_file lock
include/linux/cgroup-defs.h | 1 + kernel/cgroup/cgroup.c | 53 ++++++++++++++++++++----------------- 2 files changed, 29 insertions(+), 25 deletions(-)
Signature · 1 line
2.52.0
PATCH v2 1/3 cgroup: reduce cgroup_file_kn_lock hold time in cgroup_file_notify()
cgroup_file_notify() calls kernfs_notify() while holding the global cgroup_file_kn_lock. kernfs_notify() does non-trivial work including wake_up_interruptible() and acquisition of a second global spinlock (kernfs_notify_lock), inflating the hold time.
Take a kernfs_get() reference under the lock and call kernfs_notify() after dropping it, following the pattern from cgroup_file_show().
---
Changes since v1:
- N/A
kernel/cgroup/cgroup.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
kernel/cgroup/cgroup.c +8 -1
@@ -4686,6 +4686,7 @@ int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)void cgroup_file_notify (struct cgroup_file *cfile) {unsigned long flags; +struct kernfs_node *kn =NULL ;spin_lock_irqsave (&cgroup_file_kn_lock, flags);if (cfile->kn) {
@@ -4695,11 +4696,17 @@ void cgroup_file_notify(struct cgroup_file *cfile)if (time_in_range (jiffies, last, next)) {timer_reduce (&cfile->notify_timer, next); }else { -kernfs_notify (cfile->kn); + kn = cfile->kn; +kernfs_get (kn); cfile->notified_at = jiffies; } }spin_unlock_irqrestore (&cgroup_file_kn_lock, flags); + +if (kn) { +kernfs_notify (kn); +kernfs_put (kn); + } }EXPORT_SYMBOL_GPL (cgroup_file_notify);
Signature · 1 line
2.52.0
PATCH v2 2/3 cgroup: add lockless fast-path checks to cgroup_file_notify()
In reply to Shakeel ButtAdd lockless checks before acquiring cgroup_file_kn_lock:
1. READ_ONCE(cfile->kn) NULL check to skip torn-down files.
2. READ_ONCE(cfile->notified_at) rate-limit check to skip when within the notification interval. If within the interval, arm the deferred timer via timer_reduce() and confirm it is pending before returning -- if the timer fired in between, fall through to the lock path so the notification is not lost.
Both checks have safe error directions -- a stale read can only cause unnecessary lock acquisition, never a missed notification.
The critical section is simplified to just taking a kernfs_get() reference and updating notified_at.
Annotate cfile->kn and cfile->notified_at write sites with WRITE_ONCE() to pair with the lockless readers.
---
Changes since v1:
- Moves the timer arming and rate limiting out of lock.
kernel/cgroup/cgroup.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-)
kernel/cgroup/cgroup.c +17 -13
@@ -1749,7 +1749,7 @@ static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)struct cgroup_file *cfile = (void *)css + cft->file_offset;spin_lock_irq (&cgroup_file_kn_lock); - cfile->kn =NULL ; +WRITE_ONCE (cfile->kn,NULL );spin_unlock_irq (&cgroup_file_kn_lock);timer_delete_sync (&cfile->notify_timer);
@@ -4429,7 +4429,7 @@ static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp,timer_setup (&cfile->notify_timer, cgroup_file_notify_timer,0 );spin_lock_irq (&cgroup_file_kn_lock); - cfile->kn = kn; +WRITE_ONCE (cfile->kn, kn);spin_unlock_irq (&cgroup_file_kn_lock); }
@@ -4685,21 +4685,25 @@ int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts)*/ void cgroup_file_notify (struct cgroup_file *cfile) { -unsigned long flags; +unsigned long flags, last, next;struct kernfs_node *kn =NULL ; +if (!READ_ONCE (cfile->kn)) +return ; + + last =READ_ONCE (cfile->notified_at); + next = last +CGROUP_FILE_NOTIFY_MIN_INTV ; +if (time_in_range (jiffies, last, next)) { +timer_reduce (&cfile->notify_timer, next); +if (timer_pending (&cfile->notify_timer)) +return ; + } +spin_lock_irqsave (&cgroup_file_kn_lock, flags);if (cfile->kn) { -unsigned long last = cfile->notified_at; -unsigned long next = last +CGROUP_FILE_NOTIFY_MIN_INTV ; - -if (time_in_range (jiffies, last, next)) { -timer_reduce (&cfile->notify_timer, next); - }else { - kn = cfile->kn; -kernfs_get (kn); - cfile->notified_at = jiffies; - } + kn = cfile->kn; +kernfs_get (kn); +WRITE_ONCE (cfile->notified_at, jiffies); }spin_unlock_irqrestore (&cgroup_file_kn_lock, flags);
Signature · 1 line
2.52.0
PATCH v2 3/3 cgroup: replace global cgroup_file_kn_lock with per-cgroup_file lock
In reply to Shakeel ButtReplace the global cgroup_file_kn_lock with a per-cgroup_file spinlock to eliminate cross-cgroup contention as it is not really protecting data shared between different cgroups.
The lock is initialized in cgroup_add_file() alongside timer_setup(). No lock acquisition is needed during initialization since the cgroup directory is being populated under cgroup_mutex and no concurrent accessors exist at that point.
---
Changes since v1:
- N/A
include/linux/cgroup-defs.h | 1 + kernel/cgroup/cgroup.c | 24 ++++++++---------------- 2 files changed, 9 insertions(+), 16 deletions(-)
include/linux/cgroup-defs.h +1 -0
@@ -167,6 +167,7 @@ struct cgroup_file {struct kernfs_node *kn;unsigned long notified_at;struct timer_list notify_timer; +spinlock_t lock; };/*
kernel/cgroup/cgroup.c +8 -16
@@ -107,12 +107,6 @@ static bool cgroup_debug __read_mostly;*/ static DEFINE_SPINLOCK (cgroup_idr_lock); -/* -* Protects cgroup_file->kn for !self csses. It synchronizes notifications -* against file removal/re-creation across css hiding. -*/ -static DEFINE_SPINLOCK (cgroup_file_kn_lock); -DEFINE_PERCPU_RWSEM (cgroup_threadgroup_rwsem);#define cgroup_assert_mutex_or_rcu_locked () \
@@ -1748,9 +1742,9 @@ static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)struct cgroup_subsys_state *css =cgroup_css (cgrp, cft->ss);struct cgroup_file *cfile = (void *)css + cft->file_offset; -spin_lock_irq (&cgroup_file_kn_lock); +spin_lock_irq (&cfile->lock);WRITE_ONCE (cfile->kn,NULL ); -spin_unlock_irq (&cgroup_file_kn_lock); +spin_unlock_irq (&cfile->lock);timer_delete_sync (&cfile->notify_timer); }
@@ -4427,10 +4421,8 @@ static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp,struct cgroup_file *cfile = (void *)css + cft->file_offset;timer_setup (&cfile->notify_timer, cgroup_file_notify_timer,0 ); - -spin_lock_irq (&cgroup_file_kn_lock); -WRITE_ONCE (cfile->kn, kn); -spin_unlock_irq (&cgroup_file_kn_lock); +spin_lock_init (&cfile->lock); + cfile->kn = kn; }return 0 ;
@@ -4699,13 +4691,13 @@ void cgroup_file_notify(struct cgroup_file *cfile)return ; } -spin_lock_irqsave (&cgroup_file_kn_lock, flags); +spin_lock_irqsave (&cfile->lock, flags);if (cfile->kn) { kn = cfile->kn;kernfs_get (kn);WRITE_ONCE (cfile->notified_at, jiffies); } -spin_unlock_irqrestore (&cgroup_file_kn_lock, flags); +spin_unlock_irqrestore (&cfile->lock, flags);if (kn) {kernfs_notify (kn);
@@ -4723,10 +4715,10 @@ void cgroup_file_show(struct cgroup_file *cfile, bool show) {struct kernfs_node *kn; -spin_lock_irq (&cgroup_file_kn_lock); +spin_lock_irq (&cfile->lock); kn = cfile->kn;kernfs_get (kn); -spin_unlock_irq (&cgroup_file_kn_lock); +spin_unlock_irq (&cfile->lock);if (kn)kernfs_show (kn, show);
Signature · 1 line
2.52.0
Tejun Heo
maintainerQuoted 4 lines
Shakeel Butt (3):
cgroup: reducecgroup_file_kn_lockhold time incgroup_file_notify()
cgroup: add lockless fast-path checks tocgroup_file_notify()
cgroup: replace globalcgroup_file_kn_lockwith per-cgroup_file lock
Applied 1-3 to cgroup/for-7.1.
Thanks.
--
tejun