Skip to contentSkip to conversation

mm-list_lru-split-the-lock-to-per-cgroup-scope.patch removed from -mm tree

merged mm-stable Andrew Morton 1 participant 1 message Nov 12, 2024

lore
Share & tools
lore

Andrew Morton

author tree notice Applied
#0 lore raw

The quilt patch titled
Subject: mm/list_lru: split the lock to per-cgroup scope
has been removed from the -mm tree. Its filename was
mm-list_lru-split-the-lock-to-per-cgroup-scope.patch

This patch was dropped because it was merged into the mm-stable branch of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm

------------------------------------------------------
From: Kairui Song <[email protected]>
Subject: mm/list_lru: split the lock to per-cgroup scope
Date: Tue, 5 Nov 2024 01:52:56 +0800

Currently, every list_lru has a per-node lock that protects adding, deletion, isolation, and reparenting of all list_lru_one instances belonging to this list_lru on this node. This lock contention is heavy when multiple cgroups modify the same list_lru.

This lock can be split into per-cgroup scope to reduce contention.

To achieve this, we need a stable list_lru_one for every cgroup. This commit adds a lock to each list_lru_one and introduced a helper function lock_list_lru_of_memcg, making it possible to pin the list_lru of a memcg. Then reworked the reparenting process.

Reparenting will switch the list_lru_one instances one by one. By locking each instance and marking it dead using the nr_items counter, reparenting ensures that all items in the corresponding cgroup (on-list or not, because items have a stable cgroup, see below) will see the list_lru_one switch synchronously.

Objcg reparent is also moved after list_lru reparent so items will have a stable mem cgroup until all list_lru_one instances are drained.

The only caller that doesn't work the *_obj interfaces are direct calls to list_lru_{add,del}. But it's only used by zswap and that's also based on objcg, so it's fine.

This also changes the bahaviour of the isolation function when LRU_RETRY or LRU_REMOVED_RETRY is returned, because now releasing the lock could unblock reparenting and free the list_lru_one, isolation function will have to return withoug re-lock the lru.

prepare() {
    mkdir /tmp/test-fs
    modprobe brd rd_nr=1 rd_size=33554432
    mkfs.xfs -f /dev/ram0
    mount -t xfs /dev/ram0 /tmp/test-fs
    for i in $(seq 1 512); do
        mkdir "/tmp/test-fs/$i"
        for j in $(seq 1 10240); do
            echo TEST-CONTENT > "/tmp/test-fs/$i/$j"
        done &
    done; wait
}
do_test() {
    read_worker() {
        sleep 1
        tar -cv "$1" &>/dev/null
    }
    read_in_all() {
        cd "/tmp/test-fs" && ls
        for i in $(seq 1 512); do
            (exec sh -c 'echo "$PPID"') > "/sys/fs/cgroup/benchmark/$i/cgroup.procs"
            read_worker "$i" &
        done; wait
    }
    for i in $(seq 1 512); do
        mkdir -p "/sys/fs/cgroup/benchmark/$i"
    done
    echo +memory > /sys/fs/cgroup/benchmark/cgroup.subtree_control
    echo 512M > /sys/fs/cgroup/benchmark/memory.max
    echo 3 > /proc/sys/vm/drop_caches
    time read_in_all
}

Above script simulates compression of small files in multiple cgroups with memory pressure. Run prepare() then do_test for 6 times:

Before:

real      0m7.762s user      0m11.340s sys       3m11.224s
real      0m8.123s user      0m11.548s sys       3m2.549s
real      0m7.736s user      0m11.515s sys       3m11.171s
real      0m8.539s user      0m11.508s sys       3m7.618s
real      0m7.928s user      0m11.349s sys       3m13.063s
real      0m8.105s user      0m11.128s sys       3m14.313s

After this commit (about ~15% faster):

real      0m6.953s user      0m11.327s sys       2m42.912s
real      0m7.453s user      0m11.343s sys       2m51.942s
real      0m6.916s user      0m11.269s sys       2m43.957s
real      0m6.894s user      0m11.528s sys       2m45.346s
real      0m6.911s user      0m11.095s sys       2m43.168s
real      0m6.773s user      0m11.518s sys       2m40.774s
Cc Chengming Zhou <[email protected]>
Cc Johannes Weiner <[email protected]>
Cc Matthew Wilcox (Oracle) <[email protected]>
Cc Michal Hocko <[email protected]>
Cc Muchun Song <[email protected]>
Cc Qi Zheng <[email protected]>
Cc Roman Gushchin <[email protected]>
Cc Shakeel Butt <[email protected]>
Cc Waiman Long <[email protected]>

---

 drivers/android/binder_alloc.c |    1 
 fs/inode.c                     |    1 
 fs/xfs/xfs_qm.c                |    1 
 include/linux/list_lru.h       |    6 
 mm/list_lru.c                  |  216 +++++++++++++++++--------------
 mm/memcontrol.c                |    7 -
 mm/workingset.c                |    1 
 mm/zswap.c                     |    5 
 8 files changed, 135 insertions(+), 103 deletions(-)
drivers/android/binder_alloc.c renamed from drivers/android/binder_alloc.c~mm-list_lru-split-the-lock-to-per-cgroup-scope +0 -1
@@ -1106,7 +1106,6 @@ enum lru_status binder_alloc_free_page(s
 	mmput_async(mm);
 	__free_page(page_to_free);
 
-	spin_lock(lock);
 	return LRU_REMOVED_RETRY;
 
 err_invalid_vma:
fs/inode.c renamed from fs/inode.c~mm-list_lru-split-the-lock-to-per-cgroup-scope +0 -1
@@ -934,7 +934,6 @@ static enum lru_status inode_lru_isolate
 			mm_account_reclaimed_pages(reap);
 		}
 		inode_unpin_lru_isolating(inode);
-		spin_lock(lru_lock);
 		return LRU_RETRY;
 	}
 
fs/xfs/xfs_qm.c renamed from fs/xfs/xfs_qm.c~mm-list_lru-split-the-lock-to-per-cgroup-scope +0 -1
@@ -496,7 +496,6 @@ out_unlock_dirty:
 	trace_xfs_dqreclaim_busy(dqp);
 	XFS_STATS_INC(dqp->q_mount, xs_qm_dqreclaim_misses);
 	xfs_dqunlock(dqp);
-	spin_lock(lru_lock);
 	return LRU_RETRY;
 }
 
include/linux/list_lru.h renamed from include/linux/list_lru.h~mm-list_lru-split-the-lock-to-per-cgroup-scope +3 -3
@@ -32,6 +32,8 @@ struct list_lru_one {
 	struct list_head	list;
 	/* may become negative during memcg reparenting */
 	long			nr_items;
+	/* protects all fields above */
+	spinlock_t		lock;
 };
 
 struct list_lru_memcg {
@@ -41,11 +43,9 @@ struct list_lru_memcg {
 };
 
 struct list_lru_node {
-	/* protects all lists on the node, including per cgroup */
-	spinlock_t		lock;
 	/* global list, used for the root cgroup in cgroup aware lrus */
 	struct list_lru_one	lru;
-	long			nr_items;
+	atomic_long_t		nr_items;
 } ____cacheline_aligned_in_smp;
 
 struct list_lru {
mm/list_lru.c renamed from mm/list_lru.c~mm-list_lru-split-the-lock-to-per-cgroup-scope +124 -92

382 lines not on this page · raw message

mm/memcontrol.c renamed from mm/memcontrol.c~mm-list_lru-split-the-lock-to-per-cgroup-scope +6 -1
@@ -3110,8 +3110,13 @@ static void memcg_offline_kmem(struct me
 	if (!parent)
 		parent = root_mem_cgroup;
 
-	memcg_reparent_objcgs(memcg, parent);
 	memcg_reparent_list_lrus(memcg, parent);
+
+	/*
+	 * Objcg's reparenting must be after list_lru's, make sure list_lru
+	 * helpers won't use parent's list_lru until child is drained.
+	 */
+	memcg_reparent_objcgs(memcg, parent);
 }
 
 #ifdef CONFIG_CGROUP_WRITEBACK
mm/workingset.c renamed from mm/workingset.c~mm-list_lru-split-the-lock-to-per-cgroup-scope +0 -1
@@ -767,7 +767,6 @@ out_invalid:
 	ret = LRU_REMOVED_RETRY;
 out:
 	cond_resched();
-	spin_lock_irq(lru_lock);
 	return ret;
 }
 
mm/zswap.c renamed from mm/zswap.c~mm-list_lru-split-the-lock-to-per-cgroup-scope +2 -3
@@ -711,9 +711,9 @@ static void zswap_lru_add(struct list_lr
 	 * Note that it is safe to use rcu_read_lock() here, even in the face of
 	 * concurrent memcg offlining:
 	 *
-	 * 1. list_lru_add() is called before list_lru_memcg is erased. The
+	 * 1. list_lru_add() is called before list_lru_one is dead. The
 	 *    new entry will be reparented to memcg's parent's list_lru.
-	 * 2. list_lru_add() is called after list_lru_memcg is erased. The
+	 * 2. list_lru_add() is called after list_lru_one is dead. The
 	 *    new entry will be added directly to memcg's parent's list_lru.
 	 *
 	 * Similar reasoning holds for list_lru_del().
@@ -1179,7 +1179,6 @@ static enum lru_status shrink_memcg_cb(s
 		zswap_written_back_pages++;
 	}
 
-	spin_lock(lock);
 	return ret;
 }
 

_

Patches currently in -mm which might be from [email protected] are