mm-list_lru-split-the-lock-to-per-cgroup-scope.patch removed from -mm tree
The quilt patch titled
Subject: mm/list_lru: split the lock to per-cgroup scope
has been removed from the -mm tree. Its filename was
mm-list_lru-split-the-lock-to-per-cgroup-scope.patch
This patch was dropped because it was merged into the mm-stable branch of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
------------------------------------------------------
From: Kairui Song <[email protected]>
Subject: mm/list_lru: split the lock to per-cgroup scope
Date: Tue, 5 Nov 2024 01:52:56 +0800
Currently, every list_lru has a per-node lock that protects adding, deletion, isolation, and reparenting of all list_lru_one instances belonging to this list_lru on this node. This lock contention is heavy when multiple cgroups modify the same list_lru.
This lock can be split into per-cgroup scope to reduce contention.
To achieve this, we need a stable list_lru_one for every cgroup. This commit adds a lock to each list_lru_one and introduced a helper function lock_list_lru_of_memcg, making it possible to pin the list_lru of a memcg. Then reworked the reparenting process.
Reparenting will switch the list_lru_one instances one by one. By locking each instance and marking it dead using the nr_items counter, reparenting ensures that all items in the corresponding cgroup (on-list or not, because items have a stable cgroup, see below) will see the list_lru_one switch synchronously.
Objcg reparent is also moved after list_lru reparent so items will have a stable mem cgroup until all list_lru_one instances are drained.
The only caller that doesn't work the *_obj interfaces are direct calls to list_lru_{add,del}. But it's only used by zswap and that's also based on objcg, so it's fine.
This also changes the bahaviour of the isolation function when LRU_RETRY or LRU_REMOVED_RETRY is returned, because now releasing the lock could unblock reparenting and free the list_lru_one, isolation function will have to return withoug re-lock the lru.
prepare() {
mkdir /tmp/test-fs
modprobe brd rd_nr=1 rd_size=33554432
mkfs.xfs -f /dev/ram0
mount -t xfs /dev/ram0 /tmp/test-fs
for i in $(seq 1 512); do
mkdir "/tmp/test-fs/$i"
for j in $(seq 1 10240); do
echo TEST-CONTENT > "/tmp/test-fs/$i/$j"
done &
done; wait
}do_test() {
read_worker() {
sleep 1
tar -cv "$1" &>/dev/null
}
read_in_all() {
cd "/tmp/test-fs" && ls
for i in $(seq 1 512); do
(exec sh -c 'echo "$PPID"') > "/sys/fs/cgroup/benchmark/$i/cgroup.procs"
read_worker "$i" &
done; wait
}
for i in $(seq 1 512); do
mkdir -p "/sys/fs/cgroup/benchmark/$i"
done
echo +memory > /sys/fs/cgroup/benchmark/cgroup.subtree_control
echo 512M > /sys/fs/cgroup/benchmark/memory.max
echo 3 > /proc/sys/vm/drop_caches
time read_in_all
}Above script simulates compression of small files in multiple cgroups with memory pressure. Run prepare() then do_test for 6 times:
Before:
real 0m7.762s user 0m11.340s sys 3m11.224s real 0m8.123s user 0m11.548s sys 3m2.549s real 0m7.736s user 0m11.515s sys 3m11.171s real 0m8.539s user 0m11.508s sys 3m7.618s real 0m7.928s user 0m11.349s sys 3m13.063s real 0m8.105s user 0m11.128s sys 3m14.313s
After this commit (about ~15% faster):
real 0m6.953s user 0m11.327s sys 2m42.912s real 0m7.453s user 0m11.343s sys 2m51.942s real 0m6.916s user 0m11.269s sys 2m43.957s real 0m6.894s user 0m11.528s sys 2m45.346s real 0m6.911s user 0m11.095s sys 2m43.168s real 0m6.773s user 0m11.518s sys 2m40.774s
---
drivers/android/binder_alloc.c | 1 fs/inode.c | 1 fs/xfs/xfs_qm.c | 1 include/linux/list_lru.h | 6 mm/list_lru.c | 216 +++++++++++++++++-------------- mm/memcontrol.c | 7 - mm/workingset.c | 1 mm/zswap.c | 5 8 files changed, 135 insertions(+), 103 deletions(-)
drivers/android/binder_alloc.c renamed from drivers/android/binder_alloc.c~mm-list_lru-split-the-lock-to-per-cgroup-scope +0 -1
@@ -1106,7 +1106,6 @@ enum lru_status binder_alloc_free_page(smmput_async (mm);__free_page (page_to_free); -spin_lock (lock);return LRU_REMOVED_RETRY ; err_invalid_vma:
fs/inode.c renamed from fs/inode.c~mm-list_lru-split-the-lock-to-per-cgroup-scope +0 -1
@@ -934,7 +934,6 @@ static enum lru_status inode_lru_isolatemm_account_reclaimed_pages (reap); }inode_unpin_lru_isolating (inode); -spin_lock (lru_lock);return LRU_RETRY ; }
fs/xfs/xfs_qm.c renamed from fs/xfs/xfs_qm.c~mm-list_lru-split-the-lock-to-per-cgroup-scope +0 -1
@@ -496,7 +496,6 @@ out_unlock_dirty:trace_xfs_dqreclaim_busy (dqp);XFS_STATS_INC (dqp->q_mount, xs_qm_dqreclaim_misses);xfs_dqunlock (dqp); -spin_lock (lru_lock);return LRU_RETRY ; }
include/linux/list_lru.h renamed from include/linux/list_lru.h~mm-list_lru-split-the-lock-to-per-cgroup-scope +3 -3
@@ -32,6 +32,8 @@ struct list_lru_one {struct list_head list;/* may become negative during memcg reparenting */ long nr_items; +/* protects all fields above */ +spinlock_t lock; };struct list_lru_memcg {
@@ -41,11 +43,9 @@ struct list_lru_memcg { };struct list_lru_node { -/* protects all lists on the node, including per cgroup */ -spinlock_t lock;/* global list, used for the root cgroup in cgroup aware lrus */ struct list_lru_one lru; -long nr_items; +atomic_long_t nr_items; }____cacheline_aligned_in_smp ;struct list_lru {
mm/list_lru.c renamed from mm/list_lru.c~mm-list_lru-split-the-lock-to-per-cgroup-scope +124 -92
382 lines not on this page · raw message
mm/memcontrol.c renamed from mm/memcontrol.c~mm-list_lru-split-the-lock-to-per-cgroup-scope +6 -1
@@ -3110,8 +3110,13 @@ static void memcg_offline_kmem(struct meif (!parent) parent = root_mem_cgroup; -memcg_reparent_objcgs (memcg, parent);memcg_reparent_list_lrus (memcg, parent); + +/* +* Objcg's reparenting must be after list_lru's, make sure list_lru +* helpers won't use parent's list_lru until child is drained. +*/ +memcg_reparent_objcgs (memcg, parent); }#ifdef CONFIG_CGROUP_WRITEBACK
mm/workingset.c renamed from mm/workingset.c~mm-list_lru-split-the-lock-to-per-cgroup-scope +0 -1
@@ -767,7 +767,6 @@ out_invalid: ret =LRU_REMOVED_RETRY ; out:cond_resched (); -spin_lock_irq (lru_lock);return ret; }
mm/zswap.c renamed from mm/zswap.c~mm-list_lru-split-the-lock-to-per-cgroup-scope +2 -3
@@ -711,9 +711,9 @@ static void zswap_lru_add(struct list_lr* Note that it is safe to use rcu_read_lock() here, even in the face of * concurrent memcg offlining: * -* 1. list_lru_add() is called before list_lru_memcg is erased. The +* 1. list_lru_add() is called before list_lru_one is dead. The * new entry will be reparented to memcg's parent's list_lru. -* 2. list_lru_add() is called after list_lru_memcg is erased. The +* 2. list_lru_add() is called after list_lru_one is dead. The * new entry will be added directly to memcg's parent's list_lru. * * Similar reasoning holds for list_lru_del().
@@ -1179,7 +1179,6 @@ static enum lru_status shrink_memcg_cb(s zswap_written_back_pages++; } -spin_lock (lock);return ret; }
_
Patches currently in -mm which might be from [email protected] are